Merge pull request #16 from ryanontheinside/refactor/phase-3
Refactor/phase 3
This commit is contained in:
+33
-27
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
import importlib
|
||||
import inspect
|
||||
import os
|
||||
import re
|
||||
import traceback
|
||||
|
||||
@@ -11,14 +11,15 @@ NODE_DISPLAY_NAME_MAPPINGS = {}
|
||||
|
||||
print("\033[93m" + "[ComfyUI_RealtimeNodes] Loading nodes...")
|
||||
|
||||
|
||||
def load_nodes_from_directory(package_path):
|
||||
"""Dynamically loads nodes from Python files in a directory."""
|
||||
loaded_nodes = {}
|
||||
loaded_nodes = {}
|
||||
loaded_display_names = {}
|
||||
|
||||
|
||||
# Calculate absolute path to the package directory
|
||||
package_abs_path = os.path.join(os.path.dirname(__file__), *package_path.split('.'))
|
||||
|
||||
package_abs_path = os.path.join(os.path.dirname(__file__), *package_path.split("."))
|
||||
|
||||
if not os.path.isdir(package_abs_path):
|
||||
print(f"\033[91m" + f"[ComfyUI_RealtimeNodes] Warning: Directory not found: {package_abs_path}")
|
||||
return loaded_nodes, loaded_display_names
|
||||
@@ -26,59 +27,63 @@ def load_nodes_from_directory(package_path):
|
||||
print(f"\033[92m" + f"[ComfyUI_RealtimeNodes] Searching for nodes in: {package_abs_path}")
|
||||
|
||||
for filename in os.listdir(package_abs_path):
|
||||
if filename.endswith(".py") and not filename.startswith('__'):
|
||||
if filename.endswith(".py") and not filename.startswith("__"):
|
||||
module_name = filename[:-3]
|
||||
full_module_path = f".{package_path}.{module_name}" # Relative import path
|
||||
|
||||
full_module_path = f".{package_path}.{module_name}" # Relative import path
|
||||
|
||||
try:
|
||||
module = importlib.import_module(full_module_path, package=__name__)
|
||||
|
||||
|
||||
# Find classes defined in the module
|
||||
for name, obj in inspect.getmembers(module):
|
||||
if inspect.isclass(obj):
|
||||
# Check if it's a ComfyUI node (heuristic: has INPUT_TYPES)
|
||||
if hasattr(obj, 'INPUT_TYPES') and callable(obj.INPUT_TYPES):
|
||||
# Exclude base classes/mixins if they are explicitly named or lack a CATEGORY
|
||||
if not name.endswith("Base") and not name.endswith("Mixin") and hasattr(obj, 'CATEGORY'):
|
||||
if hasattr(obj, "INPUT_TYPES") and callable(obj.INPUT_TYPES):
|
||||
# Exclude base classes/mixins if they are explicitly named or lack a CATEGORY
|
||||
if not name.endswith("Base") and not name.endswith("Mixin") and hasattr(obj, "CATEGORY"):
|
||||
loaded_nodes[name] = obj
|
||||
|
||||
|
||||
# Generate display name (similar to original logic)
|
||||
display_name = ' '.join(word.capitalize() for word in re.findall(r'[A-Z]?[a-z]+|[A-Z]+(?=[A-Z][a-z]|\d|\W|$)|\d+', name))
|
||||
display_name = " ".join(
|
||||
word.capitalize() for word in re.findall(r"[A-Z]?[a-z]+|[A-Z]+(?=[A-Z][a-z]|\d|\W|$)|\d+", name)
|
||||
)
|
||||
suffix = " 🕒🅡🅣🅝"
|
||||
if not display_name.endswith(suffix):
|
||||
display_name += suffix
|
||||
loaded_display_names[name] = display_name
|
||||
print(f"\033[94m" + f"[ComfyUI_RealtimeNodes] - Loaded node: {name} -> {display_name}")
|
||||
|
||||
|
||||
except ImportError as e:
|
||||
print(f"\033[91m" + f"[ComfyUI_RealtimeNodes] Error importing module {full_module_path}: {e}")
|
||||
traceback.print_exc()
|
||||
except Exception as e:
|
||||
print(f"\033[91m" + f"[ComfyUI_RealtimeNodes] Error processing module {full_module_path}: {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
return loaded_nodes, loaded_display_names
|
||||
|
||||
|
||||
# --- Main Loading Logic ---
|
||||
|
||||
# Get base path for the node_wrappers directory
|
||||
base_dir_path = os.path.join(os.path.dirname(__file__), WRAPPER_DIR)
|
||||
|
||||
|
||||
# Recursively scan all directories and subdirectories
|
||||
def scan_for_modules(base_path, base_package):
|
||||
"""Recursively scan directories for module files"""
|
||||
try:
|
||||
for item in os.listdir(base_path):
|
||||
item_path = os.path.join(base_path, item)
|
||||
|
||||
|
||||
# Skip __pycache__ and files/dirs starting with __
|
||||
if item.startswith('__'):
|
||||
if item.startswith("__"):
|
||||
continue
|
||||
|
||||
|
||||
if os.path.isdir(item_path):
|
||||
# Handle subdirectory
|
||||
subdir_package = f"{base_package}.{item}"
|
||||
|
||||
|
||||
# First load .py files in this directory
|
||||
try:
|
||||
nodes, display_names = load_nodes_from_directory(subdir_package)
|
||||
@@ -87,7 +92,7 @@ def scan_for_modules(base_path, base_package):
|
||||
except Exception as e:
|
||||
print(f"\033[91m" + f"[ComfyUI_RealtimeNodes] Error processing directory {subdir_package}: {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
# Then recursively process subdirectories
|
||||
subdir_path = os.path.join(base_path, item)
|
||||
scan_for_modules(subdir_path, subdir_package)
|
||||
@@ -95,6 +100,7 @@ def scan_for_modules(base_path, base_package):
|
||||
print(f"\033[91m" + f"[ComfyUI_RealtimeNodes] Error scanning directory {base_path}: {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
# Start recursive scanning from the base directory
|
||||
print(f"\033[92m" + f"[ComfyUI_RealtimeNodes] Scanning node_wrappers directory for modules...")
|
||||
scan_for_modules(base_dir_path, WRAPPER_DIR)
|
||||
@@ -104,19 +110,19 @@ suffix = " 🕒🅡🅣🅝"
|
||||
for node_name in NODE_CLASS_MAPPINGS.keys():
|
||||
# Convert camelCase or snake_case to Title Case
|
||||
if node_name not in NODE_DISPLAY_NAME_MAPPINGS:
|
||||
display_name = ' '.join(word.capitalize() for word in re.findall(r'[A-Z]?[a-z]+|[A-Z]+(?=[A-Z][a-z]|\d|\W|$)|\d+', node_name))
|
||||
display_name = " ".join(word.capitalize() for word in re.findall(r"[A-Z]?[a-z]+|[A-Z]+(?=[A-Z][a-z]|\d|\W|$)|\d+", node_name))
|
||||
else:
|
||||
display_name = NODE_DISPLAY_NAME_MAPPINGS[node_name]
|
||||
|
||||
|
||||
# Add the suffix if it's not already present
|
||||
if not display_name.endswith(suffix):
|
||||
display_name += suffix
|
||||
|
||||
|
||||
# Assign the final display name to the mappings
|
||||
NODE_DISPLAY_NAME_MAPPINGS[node_name] = display_name
|
||||
# --- Original Web Directory and Export ---
|
||||
WEB_DIRECTORY = "./web/js" # Adjusted path if needed
|
||||
WEB_DIRECTORY = "./web/js" # Adjusted path if needed
|
||||
|
||||
__all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS']
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
|
||||
|
||||
print("\033[92m" + f"[ComfyUI_RealtimeNodes] Loaded {len(NODE_CLASS_MAPPINGS)} nodes.")
|
||||
print("\033[92m" + f"[ComfyUI_RealtimeNodes] Loaded {len(NODE_CLASS_MAPPINGS)} nodes.")
|
||||
|
||||
@@ -1 +1 @@
|
||||
# This file intentionally left blank
|
||||
# This file intentionally left blank
|
||||
|
||||
@@ -4,7 +4,3 @@ Coordinate system node wrappers for ComfyUI.
|
||||
This package provides node wrappers for the coordinate system functionality,
|
||||
allowing users to convert between coordinate spaces and create coordinate objects.
|
||||
"""
|
||||
|
||||
from .conversion_nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
|
||||
|
||||
__all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS']
|
||||
@@ -4,18 +4,18 @@ Node wrappers for coordinate conversion in ComfyUI.
|
||||
These nodes provide user-friendly interfaces for the coordinate system functionality.
|
||||
"""
|
||||
|
||||
import torch
|
||||
import logging
|
||||
from typing import Union, List, Tuple
|
||||
|
||||
from ...src.coordinates import CoordinateSystem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CoordinateConverterNode:
|
||||
"""
|
||||
Fast coordinate conversion between different coordinate spaces.
|
||||
"""
|
||||
|
||||
CATEGORY = "Realtime Nodes/Coordinates"
|
||||
FUNCTION = "convert_coordinates"
|
||||
RETURN_TYPES = ("FLOAT", "FLOAT")
|
||||
@@ -42,20 +42,20 @@ class CoordinateConverterNode:
|
||||
try:
|
||||
# Get dimensions from image
|
||||
dimensions = CoordinateSystem.get_dimensions_from_tensor(image_for_dimensions)
|
||||
|
||||
|
||||
# Map string inputs to CoordinateSystem constants
|
||||
space_map = {
|
||||
"pixel": CoordinateSystem.PIXEL,
|
||||
"normalized": CoordinateSystem.NORMALIZED,
|
||||
}
|
||||
|
||||
|
||||
from_space_const = space_map[from_space]
|
||||
to_space_const = space_map[to_space]
|
||||
|
||||
|
||||
# Convert coordinates
|
||||
x_out = CoordinateSystem.convert(x, dimensions[0], from_space_const, to_space_const)
|
||||
y_out = CoordinateSystem.convert(y, dimensions[1], from_space_const, to_space_const)
|
||||
|
||||
|
||||
return (x_out, y_out)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in coordinate conversion: {e}")
|
||||
@@ -66,6 +66,7 @@ class Point2DNode:
|
||||
"""
|
||||
Creates a 2D point with coordinate space awareness.
|
||||
"""
|
||||
|
||||
CATEGORY = "Realtime Nodes/Coordinates"
|
||||
FUNCTION = "create_point"
|
||||
RETURN_TYPES = ("POINT",)
|
||||
@@ -80,26 +81,29 @@ class Point2DNode:
|
||||
"space": (["normalized", "pixel"],),
|
||||
},
|
||||
"optional": {
|
||||
"image_for_dimensions": ("IMAGE", {"tooltip": "Reference image for dimensions (required for pixel space)"}),
|
||||
}
|
||||
"image_for_dimensions": (
|
||||
"IMAGE",
|
||||
{"tooltip": "Reference image for dimensions (required for pixel space)"},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
def create_point(self, x, y, space, image_for_dimensions=None):
|
||||
"""Create a Point object."""
|
||||
from ...src.coordinates import Point
|
||||
|
||||
|
||||
space_map = {
|
||||
"pixel": CoordinateSystem.PIXEL,
|
||||
"normalized": CoordinateSystem.NORMALIZED,
|
||||
}
|
||||
|
||||
|
||||
space_const = space_map[space]
|
||||
|
||||
|
||||
# Validate dimensions if using pixel space
|
||||
if space == "pixel" and image_for_dimensions is None:
|
||||
logger.warning("Pixel space selected but no image provided for dimensions. Using normalized space.")
|
||||
space_const = CoordinateSystem.NORMALIZED
|
||||
|
||||
|
||||
return (Point(x, y, None, space_const),)
|
||||
|
||||
|
||||
@@ -107,6 +111,7 @@ class PointListNode:
|
||||
"""
|
||||
Creates a list of points from coordinate lists.
|
||||
"""
|
||||
|
||||
CATEGORY = "Realtime Nodes/Coordinates"
|
||||
FUNCTION = "create_point_list"
|
||||
RETURN_TYPES = ("POINT_LIST",)
|
||||
@@ -116,32 +121,44 @@ class PointListNode:
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"x_coords": ("FLOAT", {"default": [0.25, 0.5, 0.75], "forceInput": True, "tooltip": "List of X coordinates"}),
|
||||
"y_coords": ("FLOAT", {"default": [0.25, 0.5, 0.75], "forceInput": True, "tooltip": "List of Y coordinates"}),
|
||||
"x_coords": (
|
||||
"FLOAT",
|
||||
{"default": [0.25, 0.5, 0.75], "forceInput": True, "tooltip": "List of X coordinates"},
|
||||
),
|
||||
"y_coords": (
|
||||
"FLOAT",
|
||||
{"default": [0.25, 0.5, 0.75], "forceInput": True, "tooltip": "List of Y coordinates"},
|
||||
),
|
||||
"space": (["normalized", "pixel"],),
|
||||
},
|
||||
"optional": {
|
||||
"z_coords": ("FLOAT", {"default": None, "forceInput": True, "tooltip": "List of Z coordinates (optional)"}),
|
||||
"image_for_dimensions": ("IMAGE", {"tooltip": "Reference image for dimensions (required for pixel space)"}),
|
||||
}
|
||||
"z_coords": (
|
||||
"FLOAT",
|
||||
{"default": None, "forceInput": True, "tooltip": "List of Z coordinates (optional)"},
|
||||
),
|
||||
"image_for_dimensions": (
|
||||
"IMAGE",
|
||||
{"tooltip": "Reference image for dimensions (required for pixel space)"},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
def create_point_list(self, x_coords, y_coords, space, z_coords=None, image_for_dimensions=None):
|
||||
"""Create a PointList object."""
|
||||
from ...src.coordinates import PointList
|
||||
|
||||
|
||||
space_map = {
|
||||
"pixel": CoordinateSystem.PIXEL,
|
||||
"normalized": CoordinateSystem.NORMALIZED,
|
||||
}
|
||||
|
||||
|
||||
space_const = space_map[space]
|
||||
|
||||
|
||||
# Validate dimensions if using pixel space
|
||||
if space == "pixel" and image_for_dimensions is None:
|
||||
logger.warning("Pixel space selected but no image provided for dimensions. Using normalized space.")
|
||||
space_const = CoordinateSystem.NORMALIZED
|
||||
|
||||
|
||||
# Ensure inputs are lists
|
||||
if not isinstance(x_coords, list):
|
||||
x_coords = [x_coords]
|
||||
@@ -149,7 +166,7 @@ class PointListNode:
|
||||
y_coords = [y_coords]
|
||||
if z_coords is not None and not isinstance(z_coords, list):
|
||||
z_coords = [z_coords]
|
||||
|
||||
|
||||
return (PointList.from_coordinates(x_coords, y_coords, z_coords, space_const),)
|
||||
|
||||
|
||||
@@ -165,4 +182,4 @@ NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"CoordinateConverter": "Coordinate Converter",
|
||||
"Point2D": "Create 2D Point",
|
||||
"PointList": "Create Point List",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,17 +1,19 @@
|
||||
import torch
|
||||
import logging
|
||||
from typing import Union, List
|
||||
from typing import List, Union
|
||||
|
||||
from ...src.coordinates import CoordinateSystem
|
||||
from ...src.coordinates import DrawingEngine
|
||||
import torch
|
||||
|
||||
from ...src.coordinates import CoordinateSystem, DrawingEngine
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RTDrawPointsNode:
|
||||
"""
|
||||
High-performance point drawing node optimized for real-time applications.
|
||||
Draws points (circles) on images with minimal overhead.
|
||||
"""
|
||||
|
||||
CATEGORY = "Realtime Nodes/MediaPipe Vision/Common/Drawing"
|
||||
FUNCTION = "draw_points"
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
@@ -22,40 +24,64 @@ class RTDrawPointsNode:
|
||||
return {
|
||||
"required": {
|
||||
"image": ("IMAGE",),
|
||||
"x": ("FLOAT", {"default": 0.5, "forceInput": True, "tooltip": "X coordinate(s) (0.0-1.0 normalized by default)"}),
|
||||
"y": ("FLOAT", {"default": 0.5, "forceInput": True, "tooltip": "Y coordinate(s) (0.0-1.0 normalized by default)"}),
|
||||
"x": (
|
||||
"FLOAT",
|
||||
{"default": 0.5, "forceInput": True, "tooltip": "X coordinate(s) (0.0-1.0 normalized by default)"},
|
||||
),
|
||||
"y": (
|
||||
"FLOAT",
|
||||
{"default": 0.5, "forceInput": True, "tooltip": "Y coordinate(s) (0.0-1.0 normalized by default)"},
|
||||
),
|
||||
"radius": ("INT", {"default": 5, "min": 1, "max": 100, "tooltip": "Radius of points in pixels"}),
|
||||
"color_hex": ("STRING", {"default": "#FF0000", "tooltip": "Color in #RRGGBB format"}),
|
||||
},
|
||||
"optional": {
|
||||
"is_normalized": ("BOOLEAN", {"default": True, "tooltip": "Whether coordinates are normalized (0-1) or pixel values"}),
|
||||
"batch_mapping": (["broadcast", "one-to-one", "all-on-first"], {"default": "broadcast", "tooltip": "How to map coords to batch images"})
|
||||
}
|
||||
"is_normalized": (
|
||||
"BOOLEAN",
|
||||
{"default": True, "tooltip": "Whether coordinates are normalized (0-1) or pixel values"},
|
||||
),
|
||||
"batch_mapping": (
|
||||
["broadcast", "one-to-one", "all-on-first"],
|
||||
{"default": "broadcast", "tooltip": "How to map coords to batch images"},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
def draw_points(self, image: torch.Tensor, x: Union[float, List[float]], y: Union[float, List[float]],
|
||||
radius: int, color_hex: str, is_normalized: bool = True, batch_mapping: str = "broadcast"):
|
||||
def draw_points(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
x: Union[float, List[float]],
|
||||
y: Union[float, List[float]],
|
||||
radius: int,
|
||||
color_hex: str,
|
||||
is_normalized: bool = True,
|
||||
batch_mapping: str = "broadcast",
|
||||
):
|
||||
"""Draw points efficiently using the DrawingEngine."""
|
||||
# Use the coordinate system to ensure proper handling
|
||||
space = CoordinateSystem.NORMALIZED if is_normalized else CoordinateSystem.PIXEL
|
||||
dimensions = CoordinateSystem.get_dimensions_from_tensor(image)
|
||||
|
||||
|
||||
drawing_engine = DrawingEngine()
|
||||
return (drawing_engine.draw_points(
|
||||
image=image,
|
||||
points_x=x,
|
||||
points_y=y,
|
||||
radius=radius,
|
||||
color=color_hex,
|
||||
is_normalized=is_normalized,
|
||||
batch_mapping=batch_mapping
|
||||
),)
|
||||
return (
|
||||
drawing_engine.draw_points(
|
||||
image=image,
|
||||
points_x=x,
|
||||
points_y=y,
|
||||
radius=radius,
|
||||
color=color_hex,
|
||||
is_normalized=is_normalized,
|
||||
batch_mapping=batch_mapping,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class RTDrawLinesNode:
|
||||
"""
|
||||
High-performance line drawing node optimized for real-time applications.
|
||||
Draws lines on images with minimal overhead.
|
||||
"""
|
||||
|
||||
CATEGORY = "Realtime Nodes/MediaPipe Vision/Common/Drawing"
|
||||
FUNCTION = "draw_lines"
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
@@ -66,59 +92,110 @@ class RTDrawLinesNode:
|
||||
return {
|
||||
"required": {
|
||||
"image": ("IMAGE",),
|
||||
"x1": ("FLOAT", {"default": 0.25, "forceInput": True, "tooltip": "Start X coordinate(s) (0.0-1.0 normalized by default)"}),
|
||||
"y1": ("FLOAT", {"default": 0.25, "forceInput": True, "tooltip": "Start Y coordinate(s) (0.0-1.0 normalized by default)"}),
|
||||
"x2": ("FLOAT", {"default": 0.75, "forceInput": True, "tooltip": "End X coordinate(s) (0.0-1.0 normalized by default)"}),
|
||||
"y2": ("FLOAT", {"default": 0.75, "forceInput": True, "tooltip": "End Y coordinate(s) (0.0-1.0 normalized by default)"}),
|
||||
"x1": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.25,
|
||||
"forceInput": True,
|
||||
"tooltip": "Start X coordinate(s) (0.0-1.0 normalized by default)",
|
||||
},
|
||||
),
|
||||
"y1": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.25,
|
||||
"forceInput": True,
|
||||
"tooltip": "Start Y coordinate(s) (0.0-1.0 normalized by default)",
|
||||
},
|
||||
),
|
||||
"x2": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.75,
|
||||
"forceInput": True,
|
||||
"tooltip": "End X coordinate(s) (0.0-1.0 normalized by default)",
|
||||
},
|
||||
),
|
||||
"y2": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.75,
|
||||
"forceInput": True,
|
||||
"tooltip": "End Y coordinate(s) (0.0-1.0 normalized by default)",
|
||||
},
|
||||
),
|
||||
"thickness": ("INT", {"default": 2, "min": 1, "max": 20, "tooltip": "Line thickness in pixels"}),
|
||||
"color_hex": ("STRING", {"default": "#00FF00", "tooltip": "Color in #RRGGBB format"}),
|
||||
},
|
||||
"optional": {
|
||||
"is_normalized": ("BOOLEAN", {"default": True, "tooltip": "Whether coordinates are normalized (0-1) or pixel values"}),
|
||||
"is_normalized": (
|
||||
"BOOLEAN",
|
||||
{"default": True, "tooltip": "Whether coordinates are normalized (0-1) or pixel values"},
|
||||
),
|
||||
"batch_mapping": (["broadcast", "one-to-one", "all-on-first"], {"default": "broadcast"}),
|
||||
"draw_endpoints": ("BOOLEAN", {"default": False, "tooltip": "Draw circles at line endpoints"}),
|
||||
"point_radius": ("INT", {"default": 3, "min": 1, "max": 20, "tooltip": "Radius of endpoint circles in pixels"}),
|
||||
"point_radius": (
|
||||
"INT",
|
||||
{"default": 3, "min": 1, "max": 20, "tooltip": "Radius of endpoint circles in pixels"},
|
||||
),
|
||||
"draw_label": ("BOOLEAN", {"default": False, "tooltip": "Draw text label on line"}),
|
||||
"label_text": ("STRING", {"default": "Line", "tooltip": "Text to display as label"}),
|
||||
"label_position": (["Midpoint", "Start", "End"], {"default": "Midpoint"}),
|
||||
"font_scale": ("FLOAT", {"default": 0.5, "min": 0.1, "max": 2.0, "step": 0.1})
|
||||
}
|
||||
"font_scale": ("FLOAT", {"default": 0.5, "min": 0.1, "max": 2.0, "step": 0.1}),
|
||||
},
|
||||
}
|
||||
|
||||
def draw_lines(self, image: torch.Tensor,
|
||||
x1: Union[float, List[float]], y1: Union[float, List[float]],
|
||||
x2: Union[float, List[float]], y2: Union[float, List[float]],
|
||||
thickness: int, color_hex: str,
|
||||
is_normalized: bool = True, batch_mapping: str = "broadcast",
|
||||
draw_endpoints: bool = False, point_radius: int = 3,
|
||||
draw_label: bool = False, label_text: str = "Line",
|
||||
label_position: str = "Midpoint", font_scale: float = 0.5):
|
||||
def draw_lines(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
x1: Union[float, List[float]],
|
||||
y1: Union[float, List[float]],
|
||||
x2: Union[float, List[float]],
|
||||
y2: Union[float, List[float]],
|
||||
thickness: int,
|
||||
color_hex: str,
|
||||
is_normalized: bool = True,
|
||||
batch_mapping: str = "broadcast",
|
||||
draw_endpoints: bool = False,
|
||||
point_radius: int = 3,
|
||||
draw_label: bool = False,
|
||||
label_text: str = "Line",
|
||||
label_position: str = "Midpoint",
|
||||
font_scale: float = 0.5,
|
||||
):
|
||||
"""Draw lines efficiently using the DrawingEngine."""
|
||||
# Use the coordinate system to ensure proper handling
|
||||
space = CoordinateSystem.NORMALIZED if is_normalized else CoordinateSystem.PIXEL
|
||||
dimensions = CoordinateSystem.get_dimensions_from_tensor(image)
|
||||
|
||||
|
||||
drawing_engine = DrawingEngine()
|
||||
return (drawing_engine.draw_lines(
|
||||
image=image,
|
||||
x1=x1, y1=y1, x2=x2, y2=y2,
|
||||
thickness=thickness,
|
||||
color=color_hex,
|
||||
is_normalized=is_normalized,
|
||||
batch_mapping=batch_mapping,
|
||||
draw_endpoints=draw_endpoints,
|
||||
point_radius=point_radius,
|
||||
draw_label=draw_label,
|
||||
label_text=label_text,
|
||||
label_position=label_position,
|
||||
font_scale=font_scale
|
||||
),)
|
||||
return (
|
||||
drawing_engine.draw_lines(
|
||||
image=image,
|
||||
x1=x1,
|
||||
y1=y1,
|
||||
x2=x2,
|
||||
y2=y2,
|
||||
thickness=thickness,
|
||||
color=color_hex,
|
||||
is_normalized=is_normalized,
|
||||
batch_mapping=batch_mapping,
|
||||
draw_endpoints=draw_endpoints,
|
||||
point_radius=point_radius,
|
||||
draw_label=draw_label,
|
||||
label_text=label_text,
|
||||
label_position=label_position,
|
||||
font_scale=font_scale,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class RTDrawPolygonNode:
|
||||
"""
|
||||
High-performance polygon drawing node optimized for real-time applications.
|
||||
Draws polygons on images with minimal overhead.
|
||||
"""
|
||||
|
||||
CATEGORY = "Realtime Nodes/MediaPipe Vision/Common/Drawing"
|
||||
FUNCTION = "draw_polygon"
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
@@ -129,58 +206,91 @@ class RTDrawPolygonNode:
|
||||
return {
|
||||
"required": {
|
||||
"image": ("IMAGE",),
|
||||
"points_x": ("FLOAT", {"default": [0.25, 0.75, 0.5], "forceInput": True, "tooltip": "List of X coordinates for polygon vertices"}),
|
||||
"points_y": ("FLOAT", {"default": [0.25, 0.25, 0.75], "forceInput": True, "tooltip": "List of Y coordinates for polygon vertices"}),
|
||||
"points_x": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": [0.25, 0.75, 0.5],
|
||||
"forceInput": True,
|
||||
"tooltip": "List of X coordinates for polygon vertices",
|
||||
},
|
||||
),
|
||||
"points_y": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": [0.25, 0.25, 0.75],
|
||||
"forceInput": True,
|
||||
"tooltip": "List of Y coordinates for polygon vertices",
|
||||
},
|
||||
),
|
||||
"color_hex": ("STRING", {"default": "#0000FF", "tooltip": "Color in #RRGGBB format"}),
|
||||
"thickness": ("INT", {"default": 2, "min": 1, "max": 20, "tooltip": "Line thickness for polygon outline"}),
|
||||
"thickness": (
|
||||
"INT",
|
||||
{"default": 2, "min": 1, "max": 20, "tooltip": "Line thickness for polygon outline"},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"fill": ("BOOLEAN", {"default": False, "tooltip": "Whether to fill the polygon"}),
|
||||
"is_normalized": ("BOOLEAN", {"default": True, "tooltip": "Whether coordinates are normalized (0-1) or pixel values"}),
|
||||
"is_normalized": (
|
||||
"BOOLEAN",
|
||||
{"default": True, "tooltip": "Whether coordinates are normalized (0-1) or pixel values"},
|
||||
),
|
||||
"batch_mapping": (["broadcast", "one-to-one", "all-on-first"], {"default": "broadcast"}),
|
||||
"draw_vertices": ("BOOLEAN", {"default": False, "tooltip": "Draw points at polygon vertices"}),
|
||||
"vertex_radius": ("INT", {"default": 3, "min": 1, "max": 20, "tooltip": "Radius of vertex points"}),
|
||||
"draw_label": ("BOOLEAN", {"default": False, "tooltip": "Draw text label on polygon"}),
|
||||
"label_text": ("STRING", {"default": "Polygon", "tooltip": "Text to display as label"}),
|
||||
"font_scale": ("FLOAT", {"default": 0.5, "min": 0.1, "max": 2.0, "step": 0.1})
|
||||
}
|
||||
"font_scale": ("FLOAT", {"default": 0.5, "min": 0.1, "max": 2.0, "step": 0.1}),
|
||||
},
|
||||
}
|
||||
|
||||
def draw_polygon(self, image: torch.Tensor,
|
||||
points_x: List[float], points_y: List[float],
|
||||
color_hex: str, thickness: int,
|
||||
fill: bool = False, is_normalized: bool = True,
|
||||
batch_mapping: str = "broadcast", draw_vertices: bool = False,
|
||||
vertex_radius: int = 3, draw_label: bool = False,
|
||||
label_text: str = "Polygon", font_scale: float = 0.5):
|
||||
def draw_polygon(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
points_x: List[float],
|
||||
points_y: List[float],
|
||||
color_hex: str,
|
||||
thickness: int,
|
||||
fill: bool = False,
|
||||
is_normalized: bool = True,
|
||||
batch_mapping: str = "broadcast",
|
||||
draw_vertices: bool = False,
|
||||
vertex_radius: int = 3,
|
||||
draw_label: bool = False,
|
||||
label_text: str = "Polygon",
|
||||
font_scale: float = 0.5,
|
||||
):
|
||||
"""Draw polygon efficiently using the DrawingEngine."""
|
||||
# Use the coordinate system to ensure proper handling
|
||||
space = CoordinateSystem.NORMALIZED if is_normalized else CoordinateSystem.PIXEL
|
||||
dimensions = CoordinateSystem.get_dimensions_from_tensor(image)
|
||||
|
||||
|
||||
drawing_engine = DrawingEngine()
|
||||
return (drawing_engine.draw_polygon(
|
||||
image=image,
|
||||
points_x=points_x,
|
||||
points_y=points_y,
|
||||
color=color_hex,
|
||||
thickness=thickness,
|
||||
fill=fill,
|
||||
is_normalized=is_normalized,
|
||||
batch_mapping=batch_mapping,
|
||||
draw_vertices=draw_vertices,
|
||||
vertex_radius=vertex_radius,
|
||||
draw_label=draw_label,
|
||||
label_text=label_text,
|
||||
label_position="Center", # Only center supported for polygons currently
|
||||
font_scale=font_scale
|
||||
),)
|
||||
return (
|
||||
drawing_engine.draw_polygon(
|
||||
image=image,
|
||||
points_x=points_x,
|
||||
points_y=points_y,
|
||||
color=color_hex,
|
||||
thickness=thickness,
|
||||
fill=fill,
|
||||
is_normalized=is_normalized,
|
||||
batch_mapping=batch_mapping,
|
||||
draw_vertices=draw_vertices,
|
||||
vertex_radius=vertex_radius,
|
||||
draw_label=draw_label,
|
||||
label_text=label_text,
|
||||
label_position="Center", # Only center supported for polygons currently
|
||||
font_scale=font_scale,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class RTCoordinateConverterNode:
|
||||
"""
|
||||
Fast coordinate conversion between normalized (0-1) and pixel space.
|
||||
Optimized for real-time applications with minimal overhead.
|
||||
"""
|
||||
|
||||
CATEGORY = "Realtime Nodes/MediaPipe Vision/Common/Coordinates"
|
||||
FUNCTION = "convert_coordinates"
|
||||
RETURN_TYPES = ("FLOAT", "FLOAT")
|
||||
@@ -197,20 +307,22 @@ class RTCoordinateConverterNode:
|
||||
}
|
||||
}
|
||||
|
||||
def convert_coordinates(self, x: Union[float, List[float]], y: Union[float, List[float]],
|
||||
image_for_dimensions: torch.Tensor, mode: str):
|
||||
def convert_coordinates(
|
||||
self, x: Union[float, List[float]], y: Union[float, List[float]], image_for_dimensions: torch.Tensor, mode: str
|
||||
):
|
||||
"""Convert coordinates using the new coordinate system."""
|
||||
dimensions = CoordinateSystem.get_dimensions_from_tensor(image_for_dimensions)
|
||||
|
||||
|
||||
if mode == "Pixel to Normalized":
|
||||
x_out = CoordinateSystem.normalize(x, dimensions[0], CoordinateSystem.PIXEL)
|
||||
y_out = CoordinateSystem.normalize(y, dimensions[1], CoordinateSystem.PIXEL)
|
||||
else: # "Normalized to Pixel"
|
||||
x_out = CoordinateSystem.denormalize(x, dimensions[0], CoordinateSystem.PIXEL)
|
||||
y_out = CoordinateSystem.denormalize(y, dimensions[1], CoordinateSystem.PIXEL)
|
||||
|
||||
|
||||
return (x_out, y_out)
|
||||
|
||||
|
||||
# Register nodes
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"RTDrawPoints": RTDrawPointsNode,
|
||||
@@ -224,4 +336,4 @@ NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"RTDrawLines": "Draw Lines (Fast)",
|
||||
"RTDrawPolygon": "Draw Polygon (Fast)",
|
||||
"RTCoordinateConverter": "Coordinate Converter (Fast)",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,31 +1,33 @@
|
||||
import logging
|
||||
from typing import Optional, List, Any, Tuple, Union
|
||||
import math
|
||||
import torch
|
||||
import numpy as np
|
||||
from typing import List, Tuple, Union
|
||||
|
||||
import cv2
|
||||
import collections
|
||||
import typing # Add typing for Dict hint
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
#TODO: consider moving base nodes to src
|
||||
from ...src.coordinates.coordinate_delta import BaseCoordinateDelta
|
||||
|
||||
# TODO: consider moving base nodes to src
|
||||
# Import from the new consolidated utilities
|
||||
from ...src.utils.math import scale_value
|
||||
from ...src.coordinates.coordinate_delta import BaseCoordinateDelta
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# --- Position Delta Control Nodes (Int/Float) (Handling Lists) ---
|
||||
class CoordinateDeltaControlNode(BaseCoordinateDelta):
|
||||
"""Base class for position-based delta control nodes (Int/Float).
|
||||
Handles single floats or lists of coordinates.
|
||||
Outputs FLOAT type (actual value is float or list[float]).
|
||||
Handles single floats or lists of coordinates.
|
||||
Outputs FLOAT type (actual value is float or list[float]).
|
||||
"""
|
||||
|
||||
# Output semantic type is FLOAT (or INT for subclass)
|
||||
RETURN_TYPES = ("FLOAT",)
|
||||
RETURN_NAMES = ("value",) # Singular name
|
||||
RETURN_NAMES = ("value",) # Singular name
|
||||
IS_INT_OUTPUT = False
|
||||
DEFAULT_DELTA_MIN = 0.0
|
||||
DEFAULT_DELTA_MAX = 0.1
|
||||
DEFAULT_DELTA_MAX = 0.1
|
||||
DEFAULT_OUTPUT_MIN = 0.0
|
||||
DEFAULT_OUTPUT_MAX = 1.0
|
||||
DEFAULT_CLAMP = True
|
||||
@@ -42,7 +44,15 @@ class CoordinateDeltaControlNode(BaseCoordinateDelta):
|
||||
"x": ("FLOAT", {"default": 0.0, "forceInput": True}),
|
||||
"y": ("FLOAT", {"default": 0.0, "forceInput": True}),
|
||||
"z": ("FLOAT", {"default": 0.0, "forceInput": True}),
|
||||
"window_size": ("INT", { "default": 1, "min": 1, "max": MAX_POSITION_HISTORY - 1, "tooltip": "Number of frames back for delta"}),
|
||||
"window_size": (
|
||||
"INT",
|
||||
{
|
||||
"default": 1,
|
||||
"min": 1,
|
||||
"max": MAX_POSITION_HISTORY - 1,
|
||||
"tooltip": "Number of frames back for delta",
|
||||
},
|
||||
),
|
||||
"delta_min": ("FLOAT", {"default": cls.DEFAULT_DELTA_MIN, "step": 0.01}),
|
||||
"delta_max": ("FLOAT", {"default": cls.DEFAULT_DELTA_MAX, "step": 0.01}),
|
||||
output_min_key: (output_type_str, {"default": cls.DEFAULT_OUTPUT_MIN}),
|
||||
@@ -55,13 +65,13 @@ class CoordinateDeltaControlNode(BaseCoordinateDelta):
|
||||
output_type_str = "INT" if self.IS_INT_OUTPUT else "FLOAT"
|
||||
output_min_key = f"output_min_{output_type_str.lower()}"
|
||||
output_max_key = f"output_max_{output_type_str.lower()}"
|
||||
x_in, y_in, z_in = kwargs.get('x'), kwargs.get('y'), kwargs.get('z')
|
||||
window_size = kwargs.get('window_size', 1)
|
||||
delta_min = kwargs.get('delta_min')
|
||||
delta_max = kwargs.get('delta_max')
|
||||
x_in, y_in, z_in = kwargs.get("x"), kwargs.get("y"), kwargs.get("z")
|
||||
window_size = kwargs.get("window_size", 1)
|
||||
delta_min = kwargs.get("delta_min")
|
||||
delta_max = kwargs.get("delta_max")
|
||||
output_min = kwargs.get(output_min_key)
|
||||
output_max = kwargs.get(output_max_key)
|
||||
clamp = kwargs.get('clamp')
|
||||
clamp = kwargs.get("clamp")
|
||||
fallback_value = int(output_min) if self.IS_INT_OUTPUT else float(output_min)
|
||||
|
||||
if None in [x_in, y_in, z_in, delta_min, delta_max, output_min, output_max, clamp]:
|
||||
@@ -81,11 +91,11 @@ class CoordinateDeltaControlNode(BaseCoordinateDelta):
|
||||
# logger.debug(f"{self.__class__.__name__}: Removing histories for indices {keys_to_remove}")
|
||||
for k in keys_to_remove:
|
||||
del self._histories[k]
|
||||
|
||||
|
||||
if current_batch_size == 0:
|
||||
logger.warning(f"{self.__class__.__name__}: Received empty lists for coordinates.")
|
||||
return ([],)
|
||||
|
||||
logger.warning(f"{self.__class__.__name__}: Received empty lists for coordinates.")
|
||||
return ([],)
|
||||
|
||||
# Process List Input
|
||||
results = []
|
||||
for i in range(current_batch_size):
|
||||
@@ -94,17 +104,17 @@ class CoordinateDeltaControlNode(BaseCoordinateDelta):
|
||||
logger.warning(f"{self.__class__.__name__}: Invalid coordinate type at index {i}. Using fallback value.")
|
||||
results.append(fallback_value)
|
||||
continue
|
||||
|
||||
|
||||
raw_delta = self._get_position_delta(float(xi), float(yi), float(zi), window_size, item_index=i)
|
||||
scaled_value = scale_value(raw_delta, delta_min, delta_max, output_min, output_max, clamp)
|
||||
|
||||
|
||||
if self.IS_INT_OUTPUT:
|
||||
output_val = int(round(scaled_value)) if scaled_value is not None else int(output_min)
|
||||
else:
|
||||
output_val = scaled_value if scaled_value is not None else float(output_min)
|
||||
results.append(output_val)
|
||||
return (results,)
|
||||
|
||||
|
||||
else:
|
||||
# Single Float Input: Remove all histories except for index 0
|
||||
keys_to_remove = [k for k in self._histories if k != 0]
|
||||
@@ -112,43 +122,47 @@ class CoordinateDeltaControlNode(BaseCoordinateDelta):
|
||||
# logger.debug(f"{self.__class__.__name__}: Switching to single input, removing histories for indices {keys_to_remove}")
|
||||
for k in keys_to_remove:
|
||||
del self._histories[k]
|
||||
|
||||
|
||||
# Process Single Float Input
|
||||
if not all(isinstance(c, (float, int)) for c in [x_in, y_in, z_in]):
|
||||
logger.error(f"{self.__class__.__name__}: Invalid coordinate types for single float input.")
|
||||
return (fallback_value,)
|
||||
|
||||
logger.error(f"{self.__class__.__name__}: Invalid coordinate types for single float input.")
|
||||
return (fallback_value,)
|
||||
|
||||
raw_delta = self._get_position_delta(float(x_in), float(y_in), float(z_in), window_size, item_index=0)
|
||||
scaled_value = scale_value(raw_delta, delta_min, delta_max, output_min, output_max, clamp)
|
||||
|
||||
|
||||
if self.IS_INT_OUTPUT:
|
||||
output_val = int(round(scaled_value)) if scaled_value is not None else int(output_min)
|
||||
else:
|
||||
output_val = scaled_value if scaled_value is not None else float(output_min)
|
||||
return (output_val,)
|
||||
|
||||
|
||||
class CoordinateDeltaIntControlNode(CoordinateDeltaControlNode):
|
||||
RETURN_TYPES = ("INT",) # Semantic type is INT
|
||||
RETURN_TYPES = ("INT",) # Semantic type is INT
|
||||
RETURN_NAMES = ("value",)
|
||||
IS_INT_OUTPUT = True
|
||||
DEFAULT_OUTPUT_MIN = 0
|
||||
DEFAULT_OUTPUT_MAX = 100
|
||||
|
||||
|
||||
class CoordinateDeltaFloatControlNode(CoordinateDeltaControlNode):
|
||||
RETURN_TYPES = ("FLOAT",) # Semantic type is FLOAT
|
||||
RETURN_TYPES = ("FLOAT",) # Semantic type is FLOAT
|
||||
RETURN_NAMES = ("value",)
|
||||
IS_INT_OUTPUT = False
|
||||
DEFAULT_OUTPUT_MIN = 0.0
|
||||
DEFAULT_OUTPUT_MAX = 1.0
|
||||
|
||||
|
||||
# --- Position Delta Trigger Node (Handling Lists) ---
|
||||
class CoordinateDeltaTriggerNode(BaseCoordinateDelta):
|
||||
"""Outputs a BOOLEAN trigger if position delta crosses a threshold.
|
||||
Handles single floats or lists of coordinates.
|
||||
Outputs BOOLEAN type (actual value is boolean or list[boolean]).
|
||||
Handles single floats or lists of coordinates.
|
||||
Outputs BOOLEAN type (actual value is boolean or list[boolean]).
|
||||
"""
|
||||
RETURN_TYPES = ("BOOLEAN",) # Semantic type
|
||||
RETURN_NAMES = ("trigger",) # Singular name
|
||||
|
||||
RETURN_TYPES = ("BOOLEAN",) # Semantic type
|
||||
RETURN_NAMES = ("trigger",) # Singular name
|
||||
DEFAULT_THRESHOLD = 0.05
|
||||
CATEGORY = "Realtime Nodes/MediaPipe Vision/Common/Delta"
|
||||
|
||||
@@ -167,15 +181,15 @@ class CoordinateDeltaTriggerNode(BaseCoordinateDelta):
|
||||
}
|
||||
|
||||
def execute(self, **kwargs):
|
||||
x_in, y_in, z_in = kwargs.get('x'), kwargs.get('y'), kwargs.get('z')
|
||||
window_size = kwargs.get('window_size', 1)
|
||||
threshold = kwargs.get('threshold')
|
||||
condition = kwargs.get('condition')
|
||||
x_in, y_in, z_in = kwargs.get("x"), kwargs.get("y"), kwargs.get("z")
|
||||
window_size = kwargs.get("window_size", 1)
|
||||
threshold = kwargs.get("threshold")
|
||||
condition = kwargs.get("condition")
|
||||
fallback_value = False
|
||||
|
||||
if None in [x_in, y_in, z_in, threshold, condition]:
|
||||
logger.error(f"{self.__class__.__name__}: Missing required arguments.")
|
||||
return (fallback_value,)
|
||||
logger.error(f"{self.__class__.__name__}: Missing required arguments.")
|
||||
return (fallback_value,)
|
||||
|
||||
# --- Input Type Handling & History Cleanup ---
|
||||
is_list = isinstance(x_in, list)
|
||||
@@ -190,11 +204,11 @@ class CoordinateDeltaTriggerNode(BaseCoordinateDelta):
|
||||
# logger.debug(f"{self.__class__.__name__}: Removing histories for indices {keys_to_remove}")
|
||||
for k in keys_to_remove:
|
||||
del self._histories[k]
|
||||
|
||||
|
||||
if current_batch_size == 0:
|
||||
logger.warning(f"{self.__class__.__name__}: Received empty lists for coordinates.")
|
||||
return ([],)
|
||||
|
||||
logger.warning(f"{self.__class__.__name__}: Received empty lists for coordinates.")
|
||||
return ([],)
|
||||
|
||||
# Process List Input
|
||||
results = []
|
||||
for i in range(current_batch_size):
|
||||
@@ -203,39 +217,48 @@ class CoordinateDeltaTriggerNode(BaseCoordinateDelta):
|
||||
logger.warning(f"{self.__class__.__name__}: Invalid coordinate type at index {i}. Using fallback value.")
|
||||
results.append(fallback_value)
|
||||
continue
|
||||
|
||||
|
||||
raw_delta = self._get_position_delta(float(xi), float(yi), float(zi), window_size, item_index=i)
|
||||
triggered = fallback_value
|
||||
if raw_delta is not None:
|
||||
if condition == "Above": triggered = raw_delta > threshold
|
||||
elif condition == "Below": triggered = raw_delta < threshold
|
||||
elif condition == "Equals": triggered = abs(raw_delta - threshold) < FLOAT_EQUALITY_TOLERANCE
|
||||
elif condition == "Not Equals": triggered = abs(raw_delta - threshold) >= FLOAT_EQUALITY_TOLERANCE
|
||||
if condition == "Above":
|
||||
triggered = raw_delta > threshold
|
||||
elif condition == "Below":
|
||||
triggered = raw_delta < threshold
|
||||
elif condition == "Equals":
|
||||
triggered = abs(raw_delta - threshold) < FLOAT_EQUALITY_TOLERANCE
|
||||
elif condition == "Not Equals":
|
||||
triggered = abs(raw_delta - threshold) >= FLOAT_EQUALITY_TOLERANCE
|
||||
results.append(triggered)
|
||||
return (results,)
|
||||
|
||||
|
||||
else:
|
||||
# Single Float Input: Remove all histories except for index 0
|
||||
# Single Float Input: Remove all histories except for index 0
|
||||
keys_to_remove = [k for k in self._histories if k != 0]
|
||||
if keys_to_remove:
|
||||
# logger.debug(f"{self.__class__.__name__}: Switching to single input, removing histories for indices {keys_to_remove}")
|
||||
for k in keys_to_remove:
|
||||
del self._histories[k]
|
||||
|
||||
|
||||
# Process Single Float Input
|
||||
if not all(isinstance(c, (float, int)) for c in [x_in, y_in, z_in]):
|
||||
logger.error(f"{self.__class__.__name__}: Invalid coordinate types for single float input.")
|
||||
return (fallback_value,)
|
||||
|
||||
logger.error(f"{self.__class__.__name__}: Invalid coordinate types for single float input.")
|
||||
return (fallback_value,)
|
||||
|
||||
raw_delta = self._get_position_delta(float(x_in), float(y_in), float(z_in), window_size, item_index=0)
|
||||
triggered = fallback_value
|
||||
if raw_delta is not None:
|
||||
if condition == "Above": triggered = raw_delta > threshold
|
||||
elif condition == "Below": triggered = raw_delta < threshold
|
||||
elif condition == "Equals": triggered = abs(raw_delta - threshold) < FLOAT_EQUALITY_TOLERANCE
|
||||
elif condition == "Not Equals": triggered = abs(raw_delta - threshold) >= FLOAT_EQUALITY_TOLERANCE
|
||||
if condition == "Above":
|
||||
triggered = raw_delta > threshold
|
||||
elif condition == "Below":
|
||||
triggered = raw_delta < threshold
|
||||
elif condition == "Equals":
|
||||
triggered = abs(raw_delta - threshold) < FLOAT_EQUALITY_TOLERANCE
|
||||
elif condition == "Not Equals":
|
||||
triggered = abs(raw_delta - threshold) >= FLOAT_EQUALITY_TOLERANCE
|
||||
return (triggered,)
|
||||
|
||||
|
||||
# --- Landmark Proximity Node (Handles Float or List Inputs) ---
|
||||
class CoordinateProximityNode:
|
||||
"""
|
||||
@@ -243,11 +266,12 @@ class CoordinateProximityNode:
|
||||
Handles single coordinate sets or lists thereof.
|
||||
Outputs FLOAT type (actual value is float or list[float]).
|
||||
"""
|
||||
|
||||
CATEGORY = "Realtime Nodes/MediaPipe Vision/Common/Utils"
|
||||
FUNCTION = "calculate_distances"
|
||||
# Output semantic type is FLOAT
|
||||
RETURN_TYPES = ("FLOAT",)
|
||||
RETURN_NAMES = ("distance",) # Singular name
|
||||
RETURN_NAMES = ("distance",) # Singular name
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
@@ -263,60 +287,67 @@ class CoordinateProximityNode:
|
||||
}
|
||||
}
|
||||
|
||||
def calculate_distances(self, x1: Union[float, List[float]], y1: Union[float, List[float]], z1: Union[float, List[float]],
|
||||
x2: Union[float, List[float]], y2: Union[float, List[float]], z2: Union[float, List[float]]) -> Union[float, List[float]]: # Return type hint updated
|
||||
|
||||
def calculate_distances(
|
||||
self,
|
||||
x1: Union[float, List[float]],
|
||||
y1: Union[float, List[float]],
|
||||
z1: Union[float, List[float]],
|
||||
x2: Union[float, List[float]],
|
||||
y2: Union[float, List[float]],
|
||||
z2: Union[float, List[float]],
|
||||
) -> Union[float, List[float]]: # Return type hint updated
|
||||
# --- Input Type Handling ---
|
||||
inputs = [x1, y1, z1, x2, y2, z2]
|
||||
is_list_input = isinstance(x1, list)
|
||||
is_list_input = isinstance(x1, list)
|
||||
fallback_value = 0.0
|
||||
|
||||
if any(isinstance(inp, list) != is_list_input for inp in inputs):
|
||||
raise ValueError("All coordinate inputs (x1..z2) must be of the same type (all floats or all lists).")
|
||||
raise ValueError("All coordinate inputs (x1..z2) must be of the same type (all floats or all lists).")
|
||||
|
||||
if is_list_input:
|
||||
# Process List Input
|
||||
list_len = len(x1)
|
||||
if any(len(inp) != list_len for inp in inputs if isinstance(inp, list)):
|
||||
raise ValueError("If inputs are lists, all coordinate lists (x1..z2) must have the same length.")
|
||||
raise ValueError("If inputs are lists, all coordinate lists (x1..z2) must have the same length.")
|
||||
if list_len == 0:
|
||||
logger.warning("Input coordinate lists are empty. Returning empty distance list.")
|
||||
return ([],) # Return empty list for FLOAT type
|
||||
return ([],) # Return empty list for FLOAT type
|
||||
num_calcs = list_len
|
||||
x1_list, y1_list, z1_list = x1, y1, z1
|
||||
x2_list, y2_list, z2_list = x2, y2, z2
|
||||
|
||||
|
||||
distances = []
|
||||
for i in range(num_calcs):
|
||||
try:
|
||||
_x1, _y1, _z1 = x1_list[i], y1_list[i], z1_list[i]
|
||||
_x2, _y2, _z2 = x2_list[i], y2_list[i], z2_list[i]
|
||||
|
||||
|
||||
if not all(isinstance(c, (float, int)) for c in [_x1, _y1, _z1, _x2, _y2, _z2]):
|
||||
logger.warning(f"Invalid coordinate type found at index {i}. Using fallback distance {fallback_value}.")
|
||||
distances.append(fallback_value)
|
||||
continue
|
||||
|
||||
dist = math.sqrt((_x2 - _x1)**2 + (_y2 - _y1)**2 + (_z2 - _z1)**2)
|
||||
logger.warning(f"Invalid coordinate type found at index {i}. Using fallback distance {fallback_value}.")
|
||||
distances.append(fallback_value)
|
||||
continue
|
||||
|
||||
dist = math.sqrt((_x2 - _x1) ** 2 + (_y2 - _y1) ** 2 + (_z2 - _z1) ** 2)
|
||||
distances.append(dist)
|
||||
except Exception as e:
|
||||
logger.error(f"Error calculating distance at index {i}: {e}", exc_info=True)
|
||||
distances.append(fallback_value)
|
||||
return (distances,) # Return list of results
|
||||
|
||||
distances.append(fallback_value)
|
||||
return (distances,) # Return list of results
|
||||
|
||||
else:
|
||||
# Process Single Float Input
|
||||
_x1, _y1, _z1 = x1, y1, z1
|
||||
_x2, _y2, _z2 = x2, y2, z2
|
||||
if not all(isinstance(c, (float, int)) for c in [_x1, _y1, _z1, _x2, _y2, _z2]):
|
||||
logger.error(f"Invalid coordinate types for single float input.")
|
||||
return (fallback_value,) # Return single fallback
|
||||
logger.error(f"Invalid coordinate types for single float input.")
|
||||
return (fallback_value,) # Return single fallback
|
||||
try:
|
||||
dist = math.sqrt((_x2 - _x1)**2 + (_y2 - _y1)**2 + (_z2 - _z1)**2)
|
||||
return (dist,) # Return single float value
|
||||
dist = math.sqrt((_x2 - _x1) ** 2 + (_y2 - _y1) ** 2 + (_z2 - _z1) ** 2)
|
||||
return (dist,) # Return single float value
|
||||
except Exception as e:
|
||||
logger.error(f"Error calculating distance for single input: {e}", exc_info=True)
|
||||
return (fallback_value,) # Return single fallback
|
||||
return (fallback_value,) # Return single fallback
|
||||
|
||||
|
||||
class MaskFromCoordinate:
|
||||
"""
|
||||
@@ -324,25 +355,50 @@ class MaskFromCoordinate:
|
||||
Handles single (x, y) floats or lists (x_list, y_list) as input.
|
||||
Outputs a batch of masks (N, 1, H, W).
|
||||
"""
|
||||
|
||||
CATEGORY = "Realtime Nodes/MediaPipe Vision/Common/Masking"
|
||||
FUNCTION = "create_masks" # Keep plural name for clarity
|
||||
FUNCTION = "create_masks" # Keep plural name for clarity
|
||||
RETURN_TYPES = ("MASK",)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"x": ("FLOAT", {"default": 0.5, "forceInput": True, "tooltip": "Normalized X coordinate (0.0-1.0) or List thereof"}),
|
||||
"y": ("FLOAT", {"default": 0.5, "forceInput": True, "tooltip": "Normalized Y coordinate (0.0-1.0) or List thereof"}),
|
||||
"x": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"forceInput": True,
|
||||
"tooltip": "Normalized X coordinate (0.0-1.0) or List thereof",
|
||||
},
|
||||
),
|
||||
"y": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"forceInput": True,
|
||||
"tooltip": "Normalized Y coordinate (0.0-1.0) or List thereof",
|
||||
},
|
||||
),
|
||||
"image_for_dimensions": ("IMAGE", {"tooltip": "Reference image to determine the output mask size"}),
|
||||
"radius": ("INT", {"default": 25, "min": 1, "max": 1024, "tooltip": "Radius of the mask shape in pixels"}),
|
||||
"radius": (
|
||||
"INT",
|
||||
{"default": 25, "min": 1, "max": 1024, "tooltip": "Radius of the mask shape in pixels"},
|
||||
),
|
||||
"shape": (["Circle", "Square"], {"default": "Circle"}),
|
||||
"invert": ("BOOLEAN", {"default": False}),
|
||||
}
|
||||
}
|
||||
|
||||
def create_masks(self, x: Union[float, List[float]], y: Union[float, List[float]], image_for_dimensions: torch.Tensor, radius: int, shape: str, invert: bool) -> Tuple[torch.Tensor]:
|
||||
|
||||
def create_masks(
|
||||
self,
|
||||
x: Union[float, List[float]],
|
||||
y: Union[float, List[float]],
|
||||
image_for_dimensions: torch.Tensor,
|
||||
radius: int,
|
||||
shape: str,
|
||||
invert: bool,
|
||||
) -> Tuple[torch.Tensor]:
|
||||
# --- Input Type Handling ---
|
||||
is_x_list = isinstance(x, list)
|
||||
is_y_list = isinstance(y, list)
|
||||
@@ -356,14 +412,14 @@ class MaskFromCoordinate:
|
||||
y_list = y
|
||||
if len(x_list) != len(y_list):
|
||||
raise ValueError(f"x_list (len {len(x_list)}) and y_list (len {len(y_list)}) must have the same length.")
|
||||
if not x_list: # Handle empty list
|
||||
logger.warning("Input coordinate lists are empty. Returning empty mask batch.")
|
||||
# Determine shape for empty tensor
|
||||
if image_for_dimensions.dim() != 4:
|
||||
logger.error("Input image must be BHWC format to determine empty mask size.")
|
||||
return (torch.zeros((0, 1, 64, 64), dtype=torch.float32),)
|
||||
_, height, width, _ = image_for_dimensions.shape
|
||||
return (torch.zeros((0, 1, height, width), dtype=torch.float32, device=image_for_dimensions.device),)
|
||||
if not x_list: # Handle empty list
|
||||
logger.warning("Input coordinate lists are empty. Returning empty mask batch.")
|
||||
# Determine shape for empty tensor
|
||||
if image_for_dimensions.dim() != 4:
|
||||
logger.error("Input image must be BHWC format to determine empty mask size.")
|
||||
return (torch.zeros((0, 1, 64, 64), dtype=torch.float32),)
|
||||
_, height, width, _ = image_for_dimensions.shape
|
||||
return (torch.zeros((0, 1, height, width), dtype=torch.float32, device=image_for_dimensions.device),)
|
||||
num_coords = len(x_list)
|
||||
else:
|
||||
# Input is float - wrap in lists for unified processing
|
||||
@@ -371,7 +427,7 @@ class MaskFromCoordinate:
|
||||
y_list = [y]
|
||||
num_coords = 1
|
||||
|
||||
# --- Get Dimensions ---
|
||||
# --- Get Dimensions ---
|
||||
if image_for_dimensions.dim() != 4:
|
||||
logger.error("Input image must be BHWC format.")
|
||||
# Return empty batch matching input size
|
||||
@@ -379,7 +435,7 @@ class MaskFromCoordinate:
|
||||
_, height, width, _ = image_for_dimensions.shape
|
||||
device = image_for_dimensions.device
|
||||
|
||||
# --- Generate Masks ---
|
||||
# --- Generate Masks ---
|
||||
output_masks = []
|
||||
for i in range(num_coords):
|
||||
xi = x_list[i]
|
||||
@@ -387,10 +443,12 @@ class MaskFromCoordinate:
|
||||
|
||||
# Validate individual coordinates
|
||||
if not isinstance(xi, (float, int)) or not isinstance(yi, (float, int)):
|
||||
logger.warning(f"Invalid coordinate type at index {i} (x={type(xi)}, y={type(yi)}). Skipping mask generation for this index.")
|
||||
mask_tensor = torch.zeros((1, height, width), dtype=torch.float32, device=device)
|
||||
output_masks.append(mask_tensor)
|
||||
continue
|
||||
logger.warning(
|
||||
f"Invalid coordinate type at index {i} (x={type(xi)}, y={type(yi)}). Skipping mask generation for this index."
|
||||
)
|
||||
mask_tensor = torch.zeros((1, height, width), dtype=torch.float32, device=device)
|
||||
output_masks.append(mask_tensor)
|
||||
continue
|
||||
|
||||
# Calculate pixel coordinates
|
||||
px = max(0, min(width - 1, int(xi * width)))
|
||||
@@ -407,7 +465,7 @@ class MaskFromCoordinate:
|
||||
bottom_right = (min(width - 1, px + half_size), min(height - 1, py + half_size))
|
||||
cv2.rectangle(mask_np, top_left, bottom_right, 1, -1)
|
||||
else:
|
||||
cv2.circle(mask_np, center, radius, 1, -1) # Default circle
|
||||
cv2.circle(mask_np, center, radius, 1, -1) # Default circle
|
||||
|
||||
if invert:
|
||||
mask_np = 1 - mask_np
|
||||
|
||||
@@ -1,34 +1,34 @@
|
||||
"""Base node wrapper for MediaPipe Vision detectors."""
|
||||
|
||||
import torch
|
||||
import logging
|
||||
from typing import Any, Dict, List, Tuple, Optional, Type, Callable, Union
|
||||
from typing import Dict
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BaseMediaPipeDetectorNode:
|
||||
"""Base class for all MediaPipe detector nodes in ComfyUI.
|
||||
|
||||
|
||||
This class captures common patterns across detector nodes like:
|
||||
- Detector instance management
|
||||
- Model validation
|
||||
- Common input parameters
|
||||
- Cleanup in __del__
|
||||
|
||||
|
||||
Parameter Naming Standards for Subclasses:
|
||||
-----------------------------------------
|
||||
To maintain consistency across detector types, use these standardized parameter names:
|
||||
|
||||
- min_confidence: Main confidence threshold for detection (replaces various min_detection_confidence,
|
||||
|
||||
- min_confidence: Main confidence threshold for detection (replaces various min_detection_confidence,
|
||||
score_threshold, etc.)
|
||||
- min_tracking_confidence: Confidence threshold for tracking objects across video frames
|
||||
- min_presence_confidence: Confidence threshold for presence of an object in a frame
|
||||
- max_results: Maximum number of detections to return (replaces num_poses, num_hands, etc.)
|
||||
|
||||
When implementing specific detector nodes, map these standardized parameter names to
|
||||
|
||||
When implementing specific detector nodes, map these standardized parameter names to
|
||||
the appropriate underlying MediaPipe API parameters in the detect() method.
|
||||
"""
|
||||
|
||||
|
||||
# To be overridden by subclasses
|
||||
DETECTOR_CLASS = None # Subclasses must set to their specific detector class
|
||||
MODEL_INFO_TYPE = "MEDIAPIPE_MODEL_INFO" # Override with specific model info type
|
||||
@@ -38,12 +38,12 @@ class BaseMediaPipeDetectorNode:
|
||||
RETURN_NAMES = tuple() # Must be set by subclass
|
||||
FUNCTION = "detect" # Default function name, can be overridden
|
||||
DESCRIPTION = "" # Override with specific node description
|
||||
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize with empty detector and model path."""
|
||||
self._detector = None
|
||||
self._model_path = None
|
||||
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
"""Define common input types for detector nodes."""
|
||||
@@ -51,84 +51,90 @@ class BaseMediaPipeDetectorNode:
|
||||
"required": {
|
||||
"image": ("IMAGE", {"tooltip": "Input image for detection"}),
|
||||
"model_info": (cls.MODEL_INFO_TYPE, {"tooltip": f"Model loaded from the appropriate loader node"}),
|
||||
"running_mode": (["image", "video"], {
|
||||
"default": "video",
|
||||
"tooltip": "Processing mode: 'image' for single images, 'video' for sequences/streams"
|
||||
}),
|
||||
"delegate": (["cpu", "gpu"], {
|
||||
"default": "cpu",
|
||||
"tooltip": "Processing device - GPU is faster but CPU is more compatible with all systems"
|
||||
}),
|
||||
"running_mode": (
|
||||
["image", "video"],
|
||||
{
|
||||
"default": "video",
|
||||
"tooltip": "Processing mode: 'image' for single images, 'video' for sequences/streams",
|
||||
},
|
||||
),
|
||||
"delegate": (
|
||||
["cpu", "gpu"],
|
||||
{
|
||||
"default": "cpu",
|
||||
"tooltip": "Processing device - GPU is faster but CPU is more compatible with all systems",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
# Subclasses should extend this with additional parameters
|
||||
|
||||
|
||||
def validate_model_info(self, model_info: Dict) -> str:
|
||||
"""Validate model_info and return model path."""
|
||||
if not isinstance(model_info, dict):
|
||||
raise ValueError(f"Invalid model_info provided. Expected dictionary, got {type(model_info)}")
|
||||
|
||||
task_type = model_info.get('task_type')
|
||||
|
||||
task_type = model_info.get("task_type")
|
||||
if task_type != self.EXPECTED_TASK_TYPE:
|
||||
raise ValueError(f"Invalid model_info provided. Expected task_type '{self.EXPECTED_TASK_TYPE}', got '{task_type}'")
|
||||
|
||||
model_path = model_info.get('model_path')
|
||||
|
||||
model_path = model_info.get("model_path")
|
||||
if not model_path:
|
||||
raise ValueError("Model path not found or invalid in model_info")
|
||||
|
||||
|
||||
return model_path
|
||||
|
||||
|
||||
def initialize_or_update_detector(self, model_path: str):
|
||||
"""Initialize detector or update if model path changed."""
|
||||
if self.DETECTOR_CLASS is None:
|
||||
raise NotImplementedError("Subclasses must set DETECTOR_CLASS to their specific detector class")
|
||||
|
||||
|
||||
if self._detector is None or self._model_path != model_path:
|
||||
# Close existing detector if needed
|
||||
if self._detector and hasattr(self._detector, 'close'):
|
||||
if self._detector and hasattr(self._detector, "close"):
|
||||
try:
|
||||
logger.info(f"Closing existing detector for {self._model_path}")
|
||||
self._detector.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing detector: {e}")
|
||||
|
||||
|
||||
# Create new detector
|
||||
logger.info(f"Creating new {self.DETECTOR_CLASS.__name__} instance for {model_path}")
|
||||
self._detector = self.DETECTOR_CLASS(model_path)
|
||||
self._model_path = model_path
|
||||
elif not hasattr(self._detector, '_detector_instance'):
|
||||
elif not hasattr(self._detector, "_detector_instance"):
|
||||
# Re-initialize if internal instance is missing
|
||||
logger.warning(f"Re-initializing detector wrapper due to missing internal instance")
|
||||
self._detector = self.DETECTOR_CLASS(model_path)
|
||||
self._model_path = model_path
|
||||
|
||||
|
||||
return self._detector
|
||||
|
||||
|
||||
def detect(self, **kwargs):
|
||||
"""
|
||||
Main detection function, should be implemented by subclasses.
|
||||
This is a placeholder that subclasses can use as a template.
|
||||
"""
|
||||
# Basic implementation that subclasses can override
|
||||
|
||||
|
||||
# 1. Extract and validate model_info
|
||||
model_info = kwargs.get('model_info')
|
||||
model_info = kwargs.get("model_info")
|
||||
model_path = self.validate_model_info(model_info)
|
||||
|
||||
|
||||
# 2. Initialize or update detector
|
||||
detector = self.initialize_or_update_detector(model_path)
|
||||
|
||||
|
||||
# 3. Call detector's detect method with appropriate parameters
|
||||
# This part should be implemented by subclasses
|
||||
|
||||
|
||||
raise NotImplementedError("Subclasses must implement or override the detect method")
|
||||
|
||||
|
||||
def __del__(self):
|
||||
"""Clean up resources by closing detector."""
|
||||
if hasattr(self, '_detector') and self._detector and hasattr(self._detector, 'close'):
|
||||
if hasattr(self, "_detector") and self._detector and hasattr(self._detector, "close"):
|
||||
try:
|
||||
logger.info(f"Closing detector instance in __del__ for {self._model_path}")
|
||||
self._detector.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing detector in __del__: {e}")
|
||||
self._detector = None
|
||||
self._detector = None
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import logging
|
||||
from typing import List, Dict, Tuple, Any, Optional, Type
|
||||
from typing import Any, List, Optional
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
from ....src.utils.image import convert_to_cv2, convert_to_tensor
|
||||
|
||||
@@ -12,24 +12,26 @@ DEFAULT_FONT = cv2.FONT_HERSHEY_SIMPLEX
|
||||
DEFAULT_FONT_SCALE = 0.5
|
||||
DEFAULT_FONT_THICKNESS = 1
|
||||
|
||||
|
||||
class BaseLandmarkVisualizationNode:
|
||||
"""Base class for landmark visualization nodes."""
|
||||
|
||||
CATEGORY = "Realtime Nodes/MediaPipe Vision/Visualization"
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "draw_visualizations"
|
||||
|
||||
# --- To be overridden by subclasses ---
|
||||
LANDMARKS_TYPE = "ANY" # The specific ComfyUI landmark type string (e.g., "FACE_LANDMARKS")
|
||||
CONNECTIONS = None # MediaPipe connections list (e.g., mp.solutions.face_mesh.FACEMESH_TESSELATION)
|
||||
|
||||
# --- To be overridden by subclasses ---
|
||||
LANDMARKS_TYPE = "ANY" # The specific ComfyUI landmark type string (e.g., "FACE_LANDMARKS")
|
||||
CONNECTIONS = None # MediaPipe connections list (e.g., mp.solutions.face_mesh.FACEMESH_TESSELATION)
|
||||
DEFAULT_DRAW_POINTS = True
|
||||
DEFAULT_POINT_COLOR = "#FFFFFF"
|
||||
DEFAULT_POINT_RADIUS = 3
|
||||
DEFAULT_DRAW_CONNECTIONS = True
|
||||
DEFAULT_CONNECTION_COLOR = "#00FF00"
|
||||
DEFAULT_CONNECTION_THICKNESS = 2
|
||||
SUPPORTS_VISIBILITY = False # Does this landmark type provide visibility scores?
|
||||
# --- End Override Section ---
|
||||
|
||||
SUPPORTS_VISIBILITY = False # Does this landmark type provide visibility scores?
|
||||
# --- End Override Section ---
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
"""Generate input types based on class properties."""
|
||||
@@ -45,29 +47,35 @@ class BaseLandmarkVisualizationNode:
|
||||
"draw_connections": ("BOOLEAN", {"default": cls.DEFAULT_DRAW_CONNECTIONS}),
|
||||
"connection_color": ("STRING", {"default": cls.DEFAULT_CONNECTION_COLOR}),
|
||||
"connection_thickness": ("INT", {"default": cls.DEFAULT_CONNECTION_THICKNESS, "min": 1, "max": 10}),
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
if cls.SUPPORTS_VISIBILITY:
|
||||
inputs["optional"]["min_visibility"] = ("FLOAT", {
|
||||
"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.05,
|
||||
"tooltip": "Minimum visibility score to draw a point/connection endpoint"
|
||||
})
|
||||
|
||||
inputs["optional"]["min_visibility"] = (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.05,
|
||||
"tooltip": "Minimum visibility score to draw a point/connection endpoint",
|
||||
},
|
||||
)
|
||||
|
||||
return inputs
|
||||
|
||||
|
||||
def draw_visualizations(self, landmarks, image, **kwargs):
|
||||
"""Main execution function: handles batching, image conversion, and calls drawing logic."""
|
||||
|
||||
|
||||
# Extract relevant optional arguments using class defaults
|
||||
draw_points = kwargs.get('draw_points', self.DEFAULT_DRAW_POINTS)
|
||||
point_color = kwargs.get('point_color', self.DEFAULT_POINT_COLOR)
|
||||
point_radius = kwargs.get('point_radius', self.DEFAULT_POINT_RADIUS)
|
||||
draw_connections = kwargs.get('draw_connections', self.DEFAULT_DRAW_CONNECTIONS)
|
||||
connection_color = kwargs.get('connection_color', self.DEFAULT_CONNECTION_COLOR)
|
||||
connection_thickness = kwargs.get('connection_thickness', self.DEFAULT_CONNECTION_THICKNESS)
|
||||
min_visibility = kwargs.get('min_visibility', 0.5) # Default if not supported/provided
|
||||
|
||||
draw_points = kwargs.get("draw_points", self.DEFAULT_DRAW_POINTS)
|
||||
point_color = kwargs.get("point_color", self.DEFAULT_POINT_COLOR)
|
||||
point_radius = kwargs.get("point_radius", self.DEFAULT_POINT_RADIUS)
|
||||
draw_connections = kwargs.get("draw_connections", self.DEFAULT_DRAW_CONNECTIONS)
|
||||
connection_color = kwargs.get("connection_color", self.DEFAULT_CONNECTION_COLOR)
|
||||
connection_thickness = kwargs.get("connection_thickness", self.DEFAULT_CONNECTION_THICKNESS)
|
||||
min_visibility = kwargs.get("min_visibility", 0.5) # Default if not supported/provided
|
||||
|
||||
if not landmarks:
|
||||
logger.warning(f"{self.__class__.__name__}: No landmarks provided.")
|
||||
return (image,)
|
||||
@@ -77,7 +85,9 @@ class BaseLandmarkVisualizationNode:
|
||||
batch_size = len(cv2_images)
|
||||
|
||||
if len(landmarks) != batch_size:
|
||||
logger.error(f"{self.__class__.__name__}: Batch size mismatch. Images: {batch_size}, Landmark sets: {len(landmarks)}. Cannot proceed reliably.")
|
||||
logger.error(
|
||||
f"{self.__class__.__name__}: Batch size mismatch. Images: {batch_size}, Landmark sets: {len(landmarks)}. Cannot proceed reliably."
|
||||
)
|
||||
return (image,)
|
||||
|
||||
# Convert hex colors to BGR tuples
|
||||
@@ -89,55 +99,73 @@ class BaseLandmarkVisualizationNode:
|
||||
landmarks_for_image = landmarks[i] # List of landmark results for this image item
|
||||
|
||||
if not landmarks_for_image:
|
||||
processed_cv2_images.append(cv2_image) # Append unmodified image
|
||||
processed_cv2_images.append(cv2_image) # Append unmodified image
|
||||
continue
|
||||
|
||||
# Iterate through all detections (faces, hands, poses) in this image item
|
||||
for single_detection_result in landmarks_for_image:
|
||||
self.draw_single_detection(cv2_image, width, height, single_detection_result,
|
||||
draw_points, point_color_bgr, point_radius,
|
||||
draw_connections, connection_color_bgr, connection_thickness,
|
||||
min_visibility)
|
||||
|
||||
self.draw_single_detection(
|
||||
cv2_image,
|
||||
width,
|
||||
height,
|
||||
single_detection_result,
|
||||
draw_points,
|
||||
point_color_bgr,
|
||||
point_radius,
|
||||
draw_connections,
|
||||
connection_color_bgr,
|
||||
connection_thickness,
|
||||
min_visibility,
|
||||
)
|
||||
|
||||
processed_cv2_images.append(cv2_image)
|
||||
|
||||
output_image = convert_to_tensor(processed_cv2_images)
|
||||
return (output_image,)
|
||||
|
||||
|
||||
def _hex_to_bgr(self, hex_color):
|
||||
"""Convert hex color string to BGR tuple."""
|
||||
hex_color = hex_color.lstrip('#')
|
||||
r, g, b = tuple(int(hex_color[i:i+2], 16) for i in (0, 2, 4))
|
||||
hex_color = hex_color.lstrip("#")
|
||||
r, g, b = tuple(int(hex_color[i : i + 2], 16) for i in (0, 2, 4))
|
||||
return (b, g, r) # OpenCV uses BGR
|
||||
|
||||
def draw_single_detection(self, cv2_image, width: int, height: int,
|
||||
detection_result: Any,
|
||||
draw_points: bool, point_color, point_radius: int,
|
||||
draw_connections: bool, connection_color, connection_thickness: int,
|
||||
min_visibility: float):
|
||||
|
||||
def draw_single_detection(
|
||||
self,
|
||||
cv2_image,
|
||||
width: int,
|
||||
height: int,
|
||||
detection_result: Any,
|
||||
draw_points: bool,
|
||||
point_color,
|
||||
point_radius: int,
|
||||
draw_connections: bool,
|
||||
connection_color,
|
||||
connection_thickness: int,
|
||||
min_visibility: float,
|
||||
):
|
||||
"""Draws landmarks and connections for a single detected object (face, hand, pose)."""
|
||||
|
||||
|
||||
# Extract the actual list of landmarks (this might vary slightly based on MediaPipe result structure)
|
||||
landmark_list = self.get_landmark_list(detection_result)
|
||||
if not landmark_list:
|
||||
return # Skip if no landmarks found for this detection
|
||||
|
||||
return # Skip if no landmarks found for this detection
|
||||
|
||||
points = {}
|
||||
# Process landmarks: calculate coordinates and optionally draw points
|
||||
for lm in landmark_list:
|
||||
# Check visibility if supported
|
||||
is_visible = True
|
||||
if self.SUPPORTS_VISIBILITY:
|
||||
vis = lm.visibility if hasattr(lm, 'visibility') and lm.visibility is not None else 1.0
|
||||
is_visible = vis >= min_visibility
|
||||
|
||||
vis = lm.visibility if hasattr(lm, "visibility") and lm.visibility is not None else 1.0
|
||||
is_visible = vis >= min_visibility
|
||||
|
||||
if is_visible:
|
||||
cx, cy = int(lm.x * width), int(lm.y * height)
|
||||
points[lm.index] = (cx, cy) # Store denormalized point coordinates
|
||||
|
||||
points[lm.index] = (cx, cy) # Store denormalized point coordinates
|
||||
|
||||
if draw_points:
|
||||
cv2.circle(cv2_image, (cx, cy), point_radius, point_color, -1)
|
||||
|
||||
|
||||
# Draw connections if enabled and applicable
|
||||
if draw_connections and self.CONNECTIONS and points:
|
||||
for connection in self.CONNECTIONS:
|
||||
@@ -149,37 +177,38 @@ class BaseLandmarkVisualizationNode:
|
||||
|
||||
def get_landmark_list(self, detection_result: Any) -> Optional[List[Any]]:
|
||||
"""Abstract method to extract the list of landmark objects from a single detection result.
|
||||
Needs to be implemented by subclasses based on the specific MediaPipe result structure.
|
||||
e.g., for hands/pose: return detection_result.landmarks
|
||||
for face: return detection_result (it's already the list)
|
||||
Needs to be implemented by subclasses based on the specific MediaPipe result structure.
|
||||
e.g., for hands/pose: return detection_result.landmarks
|
||||
for face: return detection_result (it's already the list)
|
||||
"""
|
||||
raise NotImplementedError("Subclasses must implement get_landmark_list")
|
||||
|
||||
|
||||
class BaseDetectionVisualizationNode:
|
||||
"""Base class for visualization nodes that need to draw bounding boxes and labels.
|
||||
|
||||
|
||||
This can be used for object detection, face detection, and similar tasks
|
||||
that require drawing boxes and text labels with a consistent font rendering approach.
|
||||
"""
|
||||
|
||||
CATEGORY = "Realtime Nodes/MediaPipe Vision/Visualization"
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "draw_visualizations"
|
||||
|
||||
|
||||
# Default font settings
|
||||
FONT = DEFAULT_FONT
|
||||
FONT_SCALE = DEFAULT_FONT_SCALE
|
||||
FONT_THICKNESS = DEFAULT_FONT_THICKNESS
|
||||
|
||||
|
||||
@classmethod
|
||||
def get_text_dimensions(cls, text, font_scale=None, thickness=None):
|
||||
"""Gets text dimensions using cv2.
|
||||
|
||||
|
||||
Args:
|
||||
text: The text to measure
|
||||
font_scale: The font scale (defaults to cls.FONT_SCALE)
|
||||
thickness: The font thickness (defaults to cls.FONT_THICKNESS)
|
||||
|
||||
|
||||
Returns:
|
||||
Tuple of (width, height) in pixels
|
||||
"""
|
||||
@@ -187,27 +216,33 @@ class BaseDetectionVisualizationNode:
|
||||
font_scale = cls.FONT_SCALE
|
||||
if thickness is None:
|
||||
thickness = cls.FONT_THICKNESS
|
||||
|
||||
|
||||
try:
|
||||
(text_width, text_height), baseline = cv2.getTextSize(
|
||||
text, cls.FONT, font_scale, thickness
|
||||
)
|
||||
(text_width, text_height), baseline = cv2.getTextSize(text, cls.FONT, font_scale, thickness)
|
||||
return text_width, text_height + baseline
|
||||
except Exception as e:
|
||||
logger.error(f"Error measuring text dimensions: {e}")
|
||||
return 100, 20 # Default fallback
|
||||
|
||||
|
||||
def _hex_to_bgr(self, hex_color):
|
||||
"""Convert hex color string to BGR tuple."""
|
||||
hex_color = hex_color.lstrip('#')
|
||||
r, g, b = tuple(int(hex_color[i:i+2], 16) for i in (0, 2, 4))
|
||||
hex_color = hex_color.lstrip("#")
|
||||
r, g, b = tuple(int(hex_color[i : i + 2], 16) for i in (0, 2, 4))
|
||||
return (b, g, r) # OpenCV uses BGR
|
||||
|
||||
def draw_text_with_background(self, image, text, position,
|
||||
text_color="#FFFFFF", bg_color="#000000",
|
||||
padding=(4, 2), font_scale=None, thickness=None):
|
||||
|
||||
def draw_text_with_background(
|
||||
self,
|
||||
image,
|
||||
text,
|
||||
position,
|
||||
text_color="#FFFFFF",
|
||||
bg_color="#000000",
|
||||
padding=(4, 2),
|
||||
font_scale=None,
|
||||
thickness=None,
|
||||
):
|
||||
"""Draws text with a background rectangle.
|
||||
|
||||
|
||||
Args:
|
||||
image: cv2 image to draw on
|
||||
text: Text to draw
|
||||
@@ -222,46 +257,38 @@ class BaseDetectionVisualizationNode:
|
||||
font_scale = self.FONT_SCALE
|
||||
if thickness is None:
|
||||
thickness = self.FONT_THICKNESS
|
||||
|
||||
|
||||
# Convert hex colors to BGR tuples
|
||||
text_color_bgr = self._hex_to_bgr(text_color)
|
||||
bg_color_bgr = self._hex_to_bgr(bg_color)
|
||||
|
||||
|
||||
x, y = position
|
||||
text_width, text_height = self.get_text_dimensions(text, font_scale, thickness)
|
||||
|
||||
|
||||
# Draw background rectangle
|
||||
x_pad, y_pad = padding
|
||||
cv2.rectangle(
|
||||
image,
|
||||
(x, y),
|
||||
(x + text_width + x_pad*2, y + text_height + y_pad*2),
|
||||
(x + text_width + x_pad * 2, y + text_height + y_pad * 2),
|
||||
bg_color_bgr,
|
||||
-1 # Filled rectangle
|
||||
-1, # Filled rectangle
|
||||
)
|
||||
|
||||
|
||||
# Draw text
|
||||
cv2.putText(
|
||||
image,
|
||||
text,
|
||||
(x + x_pad, y + text_height),
|
||||
self.FONT,
|
||||
font_scale,
|
||||
text_color_bgr,
|
||||
thickness
|
||||
)
|
||||
|
||||
cv2.putText(image, text, (x + x_pad, y + text_height), self.FONT, font_scale, text_color_bgr, thickness)
|
||||
|
||||
return text_width, text_height
|
||||
|
||||
|
||||
def process_batch(self, detections, image, drawing_function, **kwargs):
|
||||
"""Process a batch of images and detections.
|
||||
|
||||
|
||||
Args:
|
||||
detections: List of detection results (one per image)
|
||||
image: Batch of images as tensor
|
||||
drawing_function: Function to call for each image
|
||||
**kwargs: Additional arguments for the drawing function
|
||||
|
||||
|
||||
Returns:
|
||||
Tuple with processed image tensor
|
||||
"""
|
||||
@@ -274,7 +301,9 @@ class BaseDetectionVisualizationNode:
|
||||
batch_size = len(cv2_images)
|
||||
|
||||
if len(detections) != batch_size:
|
||||
logger.error(f"{self.__class__.__name__}: Batch size mismatch. Images: {batch_size}, Detection sets: {len(detections)}. Cannot proceed.")
|
||||
logger.error(
|
||||
f"{self.__class__.__name__}: Batch size mismatch. Images: {batch_size}, Detection sets: {len(detections)}. Cannot proceed."
|
||||
)
|
||||
return (image,)
|
||||
|
||||
for i, cv2_image in enumerate(cv2_images):
|
||||
@@ -290,4 +319,4 @@ class BaseDetectionVisualizationNode:
|
||||
processed_cv2_images.append(cv2_image)
|
||||
|
||||
output_image = convert_to_tensor(processed_cv2_images)
|
||||
return (output_image,)
|
||||
return (output_image,)
|
||||
|
||||
@@ -1,45 +1,49 @@
|
||||
"""Node wrapper for MediaPipe Vision model loader."""
|
||||
|
||||
# Import necessary components from the source directory
|
||||
from ....src.mediapipe_vision.model_loader import MediaPipeModelLoader, get_available_models
|
||||
import logging
|
||||
import traceback # For better error logging
|
||||
import traceback # For better error logging
|
||||
|
||||
# Add import for MEDIAPIPE_MODELS dictionary and the get_model_description function
|
||||
from ....src.mediapipe_vision.model_loader import MEDIAPIPE_MODELS, get_model_description
|
||||
from ....src.mediapipe_vision.model_loader import (
|
||||
MediaPipeModelLoader,
|
||||
get_available_models,
|
||||
get_model_description,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Define a base class for MediaPipe model loading nodes
|
||||
class MediaPipeModelLoaderBaseNode:
|
||||
"""Base ComfyUI node for loading MediaPipe Vision models.
|
||||
|
||||
|
||||
Inheriting classes MUST define the class variable TASK_TYPE.
|
||||
|
||||
This node handles selecting the model variant for a specific task,
|
||||
triggers the download/loading via the src.model_loader,
|
||||
and returns a dictionary containing model information needed by other nodes.
|
||||
"""
|
||||
|
||||
|
||||
# Define common category for all MediaPipe loaders
|
||||
CATEGORY = "Realtime Nodes/MediaPipe Vision/Loaders"
|
||||
# Inheriting classes MUST set this
|
||||
TASK_TYPE = None # e.g., "face_detector", "hand_landmarker"
|
||||
TASK_TYPE = None # e.g., "face_detector", "hand_landmarker"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
"""Define input types for the node."""
|
||||
if cls.TASK_TYPE is None:
|
||||
raise NotImplementedError("Subclasses of MediaPipeModelLoaderBaseNode must define TASK_TYPE")
|
||||
raise NotImplementedError("Subclasses of MediaPipeModelLoaderBaseNode must define TASK_TYPE")
|
||||
|
||||
task_variants = []
|
||||
default_variant = "default" # Sensible default
|
||||
default_variant = "default" # Sensible default
|
||||
|
||||
try:
|
||||
task_variants = get_available_models(cls.TASK_TYPE)
|
||||
if not task_variants:
|
||||
logger.warning(f"No model variants found for task '{cls.TASK_TYPE}'. Defaulting to ['default'].")
|
||||
task_variants = ["default"] # Fallback variants
|
||||
logger.warning(f"No model variants found for task '{cls.TASK_TYPE}'. Defaulting to ['default'].")
|
||||
task_variants = ["default"] # Fallback variants
|
||||
default_variant = task_variants[0] if task_variants else default_variant
|
||||
|
||||
except Exception as e:
|
||||
@@ -50,7 +54,7 @@ class MediaPipeModelLoaderBaseNode:
|
||||
|
||||
# Create tooltip with model descriptions
|
||||
tooltip = f"Select model variant for {cls.TASK_TYPE}."
|
||||
|
||||
|
||||
# Add model descriptions if available
|
||||
tooltip += "\n\nModel Variants:"
|
||||
for variant in task_variants:
|
||||
@@ -60,54 +64,55 @@ class MediaPipeModelLoaderBaseNode:
|
||||
return {
|
||||
"required": {
|
||||
# Dropdown for selecting the model variant for the specific task
|
||||
"model_variant": (task_variants, {
|
||||
"default": default_variant,
|
||||
"tooltip": tooltip
|
||||
}),
|
||||
"model_variant": (task_variants, {"default": default_variant, "tooltip": tooltip}),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# Return a dictionary with model info, not the loader object itself
|
||||
RETURN_TYPES = ("MEDIAPIPE_MODEL_INFO",) # Default, can be overridden by subclasses for specificity
|
||||
RETURN_TYPES = ("MEDIAPIPE_MODEL_INFO",) # Default, can be overridden by subclasses for specificity
|
||||
RETURN_NAMES = ("model_info",)
|
||||
FUNCTION = "load_and_get_info"
|
||||
|
||||
def load_and_get_info(self, model_variant: str): # Removed task_type argument
|
||||
def load_and_get_info(self, model_variant: str): # Removed task_type argument
|
||||
"""
|
||||
Loads the specified MediaPipe model using the src loader
|
||||
and returns a dictionary with model information (path, task, variant).
|
||||
Uses the TASK_TYPE defined by the subclass.
|
||||
"""
|
||||
if self.TASK_TYPE is None:
|
||||
raise NotImplementedError("TASK_TYPE not defined in the subclass.")
|
||||
raise NotImplementedError("TASK_TYPE not defined in the subclass.")
|
||||
|
||||
task_type = self.TASK_TYPE # Use the class variable
|
||||
task_type = self.TASK_TYPE # Use the class variable
|
||||
|
||||
logger.info(f"Attempting to load MediaPipe model: Task='{task_type}', Variant='{model_variant}'")
|
||||
|
||||
|
||||
# Use a temporary loader instance just to trigger the load/download logic
|
||||
# The actual detector nodes will instantiate their own detectors using the path
|
||||
loader = MediaPipeModelLoader()
|
||||
# Pass the class's task_type here
|
||||
success = loader.load_model(task_type, model_variant)
|
||||
|
||||
|
||||
if not success:
|
||||
error_msg = f"Failed to load MediaPipe model for Task='{task_type}', Variant='{model_variant}'"
|
||||
logger.error(error_msg)
|
||||
# It's often better to raise an error to stop the workflow
|
||||
raise RuntimeError(error_msg)
|
||||
|
||||
|
||||
# Get the essential model information
|
||||
model_info = loader.get_model_info() # Should return {'model_path': ..., 'task_type': ..., 'model_variant': ...}
|
||||
|
||||
model_info = loader.get_model_info() # Should return {'model_path': ..., 'task_type': ..., 'model_variant': ...}
|
||||
|
||||
# Ensure the info dict is populated correctly - get_model_info should ideally return all needed keys
|
||||
if 'model_path' not in model_info or not model_info['model_path']:
|
||||
raise RuntimeError(f"Failed to get valid model path from loader for {task_type}/{model_variant}")
|
||||
if 'task_type' not in model_info: model_info['task_type'] = task_type
|
||||
if 'model_variant' not in model_info: model_info['model_variant'] = model_variant
|
||||
|
||||
logger.info(f"Successfully loaded model info: Path='{model_info.get('model_path')}', Task='{model_info.get('task_type')}', Variant='{model_info.get('model_variant')}'")
|
||||
|
||||
if "model_path" not in model_info or not model_info["model_path"]:
|
||||
raise RuntimeError(f"Failed to get valid model path from loader for {task_type}/{model_variant}")
|
||||
if "task_type" not in model_info:
|
||||
model_info["task_type"] = task_type
|
||||
if "model_variant" not in model_info:
|
||||
model_info["model_variant"] = model_variant
|
||||
|
||||
logger.info(
|
||||
f"Successfully loaded model info: Path='{model_info.get('model_path')}', Task='{model_info.get('task_type')}', Variant='{model_info.get('model_variant')}'"
|
||||
)
|
||||
|
||||
# Return the dictionary containing model path and other relevant info
|
||||
# The specific RETURN_TYPES are handled by the subclass definition
|
||||
return (model_info,)
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
|
||||
@@ -1,25 +1,31 @@
|
||||
import torch
|
||||
from .....src.mediapipe_vision.face_detection.detector import FaceDetector
|
||||
from ...common.model_loader import MediaPipeModelLoaderBaseNode
|
||||
from ...common.base_detector_node import BaseMediaPipeDetectorNode
|
||||
import logging
|
||||
|
||||
import torch
|
||||
|
||||
from .....src.mediapipe_vision.face_detection.detector import FaceDetector
|
||||
from ...common.base_detector_node import BaseMediaPipeDetectorNode
|
||||
from ...common.model_loader import MediaPipeModelLoaderBaseNode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_category = "Realtime Nodes/MediaPipe Vision/Face/FaceDetection"
|
||||
|
||||
|
||||
# Inherit from Base Loader
|
||||
class MediaPipeFaceDetectorModelLoaderNode(MediaPipeModelLoaderBaseNode):
|
||||
"""ComfyUI node for loading MediaPipe Face Detector models."""
|
||||
|
||||
TASK_TYPE = "face_detector"
|
||||
RETURN_TYPES = ("FACE_DETECTOR_MODEL_INFO",) # Specific return type
|
||||
RETURN_TYPES = ("FACE_DETECTOR_MODEL_INFO",) # Specific return type
|
||||
RETURN_NAMES = ("model_info",)
|
||||
CATEGORY = _category
|
||||
DESCRIPTION = "Loads a MediaPipe Face Detector model for finding faces in images. Required before using the Face Detector node."
|
||||
# INPUT_TYPES and FUNCTION inherited
|
||||
|
||||
|
||||
# Detection/Landmarking Nodes
|
||||
class MediaPipeFaceDetectorNode(BaseMediaPipeDetectorNode):
|
||||
"""ComfyUI node for MediaPipe Face Detection."""
|
||||
|
||||
|
||||
# Define class variables required by the base class
|
||||
DETECTOR_CLASS = FaceDetector
|
||||
MODEL_INFO_TYPE = "FACE_DETECTOR_MODEL_INFO"
|
||||
@@ -29,40 +35,45 @@ class MediaPipeFaceDetectorNode(BaseMediaPipeDetectorNode):
|
||||
FUNCTION = "detect"
|
||||
CATEGORY = _category
|
||||
DESCRIPTION = "Detects faces in images using MediaPipe's face detection technology. Provides coordinates and confidence scores for all faces found in the image."
|
||||
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
# Start with the base inputs from the parent class
|
||||
inputs = super().INPUT_TYPES()
|
||||
|
||||
|
||||
# Add face detection specific parameters
|
||||
inputs["required"].update({
|
||||
"min_confidence": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.05,
|
||||
"tooltip": "Minimum confidence threshold for face detection - lower values find more faces but may include false positives"}),
|
||||
})
|
||||
|
||||
inputs["required"].update(
|
||||
{
|
||||
"min_confidence": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.05,
|
||||
"tooltip": "Minimum confidence threshold for face detection - lower values find more faces but may include false positives",
|
||||
},
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
return inputs
|
||||
|
||||
def detect(self, image: torch.Tensor, model_info: dict, min_confidence: float,
|
||||
running_mode: str, delegate: str):
|
||||
def detect(self, image: torch.Tensor, model_info: dict, min_confidence: float, running_mode: str, delegate: str):
|
||||
"""Performs face detection with the configured parameters."""
|
||||
|
||||
|
||||
# Validate model_info and get model path
|
||||
model_path = self.validate_model_info(model_info)
|
||||
|
||||
|
||||
# Initialize or update detector
|
||||
detector = self.initialize_or_update_detector(model_path)
|
||||
|
||||
|
||||
# Perform detection with all parameters
|
||||
batch_results = detector.detect(
|
||||
image,
|
||||
min_detection_confidence=min_confidence,
|
||||
running_mode=running_mode,
|
||||
delegate=delegate
|
||||
)
|
||||
|
||||
batch_results = detector.detect(image, min_detection_confidence=min_confidence, running_mode=running_mode, delegate=delegate)
|
||||
|
||||
return (batch_results,)
|
||||
|
||||
|
||||
# Define mappings for ComfyUI
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"MediaPipeFaceDetectorModelLoader": MediaPipeFaceDetectorModelLoaderNode,
|
||||
|
||||
@@ -1,73 +1,140 @@
|
||||
# face_detection_mask.py
|
||||
import torch
|
||||
import numpy as np
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
import scipy.ndimage
|
||||
import torch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_category = "Realtime Nodes/MediaPipe Vision/Face/FaceDetection"
|
||||
|
||||
|
||||
class FaceDetectionToMask:
|
||||
"""Converts face detection bounding boxes to masks that can be used in ComfyUI."""
|
||||
|
||||
CATEGORY = _category
|
||||
DESCRIPTION = "Creates masks from face detections that can be used for selective processing, inpainting, or compositing. Generate masks based on face bounding boxes or facial keypoints."
|
||||
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"face_detections": ("FACE_DETECTIONS", {"tooltip": "Face detection results from the Face Detector node"}),
|
||||
"width": ("INT", {"default": 512, "min": 1, "max": 8192, "step": 1,
|
||||
"tooltip": "Width of the output mask in pixels"}),
|
||||
"height": ("INT", {"default": 512, "min": 1, "max": 8192, "step": 1,
|
||||
"tooltip": "Height of the output mask in pixels"}),
|
||||
"mask_type": (["Bounding Box", "Keypoints", "Both"], {"default": "Bounding Box",
|
||||
"tooltip": "Type of mask to create - Bounding Box (rectangle around face), Keypoints (dots at facial points), or Both"}),
|
||||
"face_selection": (["All Faces", "First Face", "Largest Face", "Highest Confidence Face"],
|
||||
{"default": "All Faces",
|
||||
"tooltip": "Which faces to include in the mask - all detected faces, only the first face, largest face, or most confident detection"}),
|
||||
"feather_amount": ("INT", {"default": 0, "min": 0, "max": 100, "step": 1,
|
||||
"tooltip": "How much to soften the edges of the mask in pixels - higher values create smoother transitions"}),
|
||||
"keypoint_radius": ("INT", {"default": 5, "min": 1, "max": 50, "step": 1,
|
||||
"tooltip": "Size of each keypoint in the mask when using Keypoints or Both mask types"}),
|
||||
"invert": ("BOOLEAN", {"default": False,
|
||||
"tooltip": "When enabled, inverts the mask (white becomes black, black becomes white)"}),
|
||||
"score_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
"tooltip": "Minimum confidence score for including face detections - higher values filter out less confident detections"})
|
||||
"face_detections": (
|
||||
"FACE_DETECTIONS",
|
||||
{"tooltip": "Face detection results from the Face Detector node"},
|
||||
),
|
||||
"width": (
|
||||
"INT",
|
||||
{"default": 512, "min": 1, "max": 8192, "step": 1, "tooltip": "Width of the output mask in pixels"},
|
||||
),
|
||||
"height": (
|
||||
"INT",
|
||||
{
|
||||
"default": 512,
|
||||
"min": 1,
|
||||
"max": 8192,
|
||||
"step": 1,
|
||||
"tooltip": "Height of the output mask in pixels",
|
||||
},
|
||||
),
|
||||
"mask_type": (
|
||||
["Bounding Box", "Keypoints", "Both"],
|
||||
{
|
||||
"default": "Bounding Box",
|
||||
"tooltip": "Type of mask to create - Bounding Box (rectangle around face), Keypoints (dots at facial points), or Both",
|
||||
},
|
||||
),
|
||||
"face_selection": (
|
||||
["All Faces", "First Face", "Largest Face", "Highest Confidence Face"],
|
||||
{
|
||||
"default": "All Faces",
|
||||
"tooltip": "Which faces to include in the mask - all detected faces, only the first face, largest face, or most confident detection",
|
||||
},
|
||||
),
|
||||
"feather_amount": (
|
||||
"INT",
|
||||
{
|
||||
"default": 0,
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"step": 1,
|
||||
"tooltip": "How much to soften the edges of the mask in pixels - higher values create smoother transitions",
|
||||
},
|
||||
),
|
||||
"keypoint_radius": (
|
||||
"INT",
|
||||
{
|
||||
"default": 5,
|
||||
"min": 1,
|
||||
"max": 50,
|
||||
"step": 1,
|
||||
"tooltip": "Size of each keypoint in the mask when using Keypoints or Both mask types",
|
||||
},
|
||||
),
|
||||
"invert": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "When enabled, inverts the mask (white becomes black, black becomes white)",
|
||||
},
|
||||
),
|
||||
"score_threshold": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.0,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Minimum confidence score for including face detections - higher values filter out less confident detections",
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"reference_image": ("IMAGE", {"tooltip": "Optional image to match dimensions - mask will be created with the same size as this image"})
|
||||
}
|
||||
"reference_image": (
|
||||
"IMAGE",
|
||||
{"tooltip": "Optional image to match dimensions - mask will be created with the same size as this image"},
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("MASK",)
|
||||
FUNCTION = "create_mask"
|
||||
|
||||
def create_mask(self, face_detections, width, height, mask_type, face_selection,
|
||||
feather_amount, keypoint_radius, invert, score_threshold, reference_image=None):
|
||||
|
||||
|
||||
def create_mask(
|
||||
self,
|
||||
face_detections,
|
||||
width,
|
||||
height,
|
||||
mask_type,
|
||||
face_selection,
|
||||
feather_amount,
|
||||
keypoint_radius,
|
||||
invert,
|
||||
score_threshold,
|
||||
reference_image=None,
|
||||
):
|
||||
# If reference image provided, use its dimensions
|
||||
if reference_image is not None:
|
||||
height, width = reference_image.shape[1], reference_image.shape[2]
|
||||
|
||||
|
||||
batch_size = len(face_detections)
|
||||
masks = []
|
||||
|
||||
|
||||
for i in range(batch_size):
|
||||
# Create empty mask
|
||||
mask = torch.zeros((height, width), dtype=torch.float32, device="cpu")
|
||||
detections_for_image = face_detections[i]
|
||||
|
||||
|
||||
if not detections_for_image:
|
||||
masks.append(mask)
|
||||
continue
|
||||
|
||||
|
||||
# Filter detections by score threshold
|
||||
filtered_detections = []
|
||||
for detection in detections_for_image:
|
||||
if detection.score is None or detection.score >= score_threshold:
|
||||
filtered_detections.append(detection)
|
||||
|
||||
|
||||
# Select which faces to include based on face_selection parameter
|
||||
selected_detections = []
|
||||
if face_selection == "All Faces":
|
||||
@@ -76,8 +143,7 @@ class FaceDetectionToMask:
|
||||
selected_detections = [filtered_detections[0]]
|
||||
elif face_selection == "Largest Face" and filtered_detections:
|
||||
# Find the detection with the largest area
|
||||
largest_detection = max(filtered_detections,
|
||||
key=lambda d: d.bounding_box.width * d.bounding_box.height)
|
||||
largest_detection = max(filtered_detections, key=lambda d: d.bounding_box.width * d.bounding_box.height)
|
||||
selected_detections = [largest_detection]
|
||||
elif face_selection == "Highest Confidence Face" and filtered_detections:
|
||||
# Find the detection with the highest confidence score
|
||||
@@ -89,67 +155,64 @@ class FaceDetectionToMask:
|
||||
else:
|
||||
# If no detections have scores, fall back to first detection
|
||||
selected_detections = [filtered_detections[0]]
|
||||
|
||||
|
||||
for detection in selected_detections:
|
||||
# Get bounding box
|
||||
bbox = detection.bounding_box
|
||||
x1, y1 = max(0, bbox.origin_x), max(0, bbox.origin_y)
|
||||
x2, y2 = min(width-1, bbox.origin_x + bbox.width), min(height-1, bbox.origin_y + bbox.height)
|
||||
|
||||
x2, y2 = min(width - 1, bbox.origin_x + bbox.width), min(height - 1, bbox.origin_y + bbox.height)
|
||||
|
||||
if x2 <= x1 or y2 <= y1:
|
||||
continue # Skip invalid boxes
|
||||
|
||||
|
||||
# Fill bounding box region if requested
|
||||
if mask_type in ["Bounding Box", "Both"]:
|
||||
mask[y1:y2+1, x1:x2+1] = 1.0
|
||||
|
||||
mask[y1 : y2 + 1, x1 : x2 + 1] = 1.0
|
||||
|
||||
# Add keypoints if requested
|
||||
if mask_type in ["Keypoints", "Both"] and detection.keypoints:
|
||||
for kp in detection.keypoints:
|
||||
# Keypoints are in normalized coordinates, convert to pixel coordinates
|
||||
cx, cy = int(kp.x * width), int(kp.y * height)
|
||||
|
||||
|
||||
# Draw circle for each keypoint
|
||||
for y in range(max(0, cy - keypoint_radius), min(height, cy + keypoint_radius + 1)):
|
||||
for x in range(max(0, cx - keypoint_radius), min(width, cx + keypoint_radius + 1)):
|
||||
# Check if point is within radius
|
||||
if (x - cx) ** 2 + (y - cy) ** 2 <= keypoint_radius ** 2:
|
||||
if (x - cx) ** 2 + (y - cy) ** 2 <= keypoint_radius**2:
|
||||
mask[y, x] = 1.0
|
||||
|
||||
|
||||
# Feather the edges if requested
|
||||
if feather_amount > 0:
|
||||
# Convert mask to numpy for scipy operations
|
||||
mask_np = mask.numpy()
|
||||
|
||||
|
||||
# Calculate distance transform
|
||||
if invert:
|
||||
distance = scipy.ndimage.distance_transform_edt(mask_np == 0)
|
||||
else:
|
||||
distance = scipy.ndimage.distance_transform_edt(mask_np > 0)
|
||||
|
||||
|
||||
# Create feathered edge
|
||||
feather_mask = np.clip(distance / feather_amount, 0, 1)
|
||||
|
||||
|
||||
if invert:
|
||||
feather_mask = 1.0 - feather_mask
|
||||
|
||||
|
||||
# Convert back to tensor
|
||||
mask = torch.from_numpy(feather_mask).float()
|
||||
elif invert:
|
||||
# Simple inversion without feathering
|
||||
mask = 1.0 - mask
|
||||
|
||||
|
||||
masks.append(mask)
|
||||
|
||||
|
||||
# Stack all masks into a batch
|
||||
mask_batch = torch.stack(masks)
|
||||
return (mask_batch,)
|
||||
|
||||
# Node registration
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"FaceDetectionToMask": FaceDetectionToMask
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"FaceDetectionToMask": "Face Detection to Mask (MediaPipe)"
|
||||
}
|
||||
# Node registration
|
||||
NODE_CLASS_MAPPINGS = {"FaceDetectionToMask": FaceDetectionToMask}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {"FaceDetectionToMask": "Face Detection to Mask (MediaPipe)"}
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
# visualize_face_detection.py
|
||||
import torch
|
||||
import numpy as np
|
||||
import logging
|
||||
|
||||
import cv2
|
||||
|
||||
from ...common.base_visualization_nodes import BaseDetectionVisualizationNode
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class VisualizeFaceDetections(BaseDetectionVisualizationNode):
|
||||
"""Draws face detection bounding boxes and keypoints onto images."""
|
||||
|
||||
CATEGORY = "Realtime Nodes/MediaPipe Vision/Face/FaceDetection/Visualization"
|
||||
DESCRIPTION = "Visualizes face detections by drawing boxes and keypoints onto images. Allows customization of colors and sizes to highlight detected faces."
|
||||
|
||||
@@ -16,29 +18,57 @@ class VisualizeFaceDetections(BaseDetectionVisualizationNode):
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"face_detections": ("FACE_DETECTIONS", {"tooltip": "Face detection results from the Face Detector node"}),
|
||||
"face_detections": (
|
||||
"FACE_DETECTIONS",
|
||||
{"tooltip": "Face detection results from the Face Detector node"},
|
||||
),
|
||||
"image": ("IMAGE", {"tooltip": "The image on which to draw face detection visualizations"}),
|
||||
},
|
||||
"optional": {
|
||||
"box_color": ("STRING", {"default": "#00FF00", "tooltip": "Hex color code for the face bounding boxes (default: green)"}),
|
||||
"box_thickness": ("INT", {"default": 2, "min": 1, "max": 10, "tooltip": "Thickness of the face bounding box lines"}),
|
||||
"keypoint_color": ("STRING", {"default": "#FF0000", "tooltip": "Hex color code for facial keypoints (default: red)"}),
|
||||
"keypoint_radius": ("INT", {"default": 2, "min": 1, "max": 10, "tooltip": "Size of the dots representing facial keypoints"}),
|
||||
"label_color": ("STRING", {"default": "#FFFFFF", "tooltip": "Hex color code for the text labels (default: white)"}),
|
||||
"label_bg_color": ("STRING", {"default": "#00FF00", "tooltip": "Hex color code for the label background (default: green)"}),
|
||||
}
|
||||
"box_color": (
|
||||
"STRING",
|
||||
{"default": "#00FF00", "tooltip": "Hex color code for the face bounding boxes (default: green)"},
|
||||
),
|
||||
"box_thickness": (
|
||||
"INT",
|
||||
{"default": 2, "min": 1, "max": 10, "tooltip": "Thickness of the face bounding box lines"},
|
||||
),
|
||||
"keypoint_color": (
|
||||
"STRING",
|
||||
{"default": "#FF0000", "tooltip": "Hex color code for facial keypoints (default: red)"},
|
||||
),
|
||||
"keypoint_radius": (
|
||||
"INT",
|
||||
{"default": 2, "min": 1, "max": 10, "tooltip": "Size of the dots representing facial keypoints"},
|
||||
),
|
||||
"label_color": (
|
||||
"STRING",
|
||||
{"default": "#FFFFFF", "tooltip": "Hex color code for the text labels (default: white)"},
|
||||
),
|
||||
"label_bg_color": (
|
||||
"STRING",
|
||||
{"default": "#00FF00", "tooltip": "Hex color code for the label background (default: green)"},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "draw_visualizations"
|
||||
|
||||
def draw_visualizations(self, face_detections, image,
|
||||
box_color="#00FF00", box_thickness=2,
|
||||
keypoint_color="#FF0000", keypoint_radius=2,
|
||||
label_color="#FFFFFF", label_bg_color="#00FF00"):
|
||||
def draw_visualizations(
|
||||
self,
|
||||
face_detections,
|
||||
image,
|
||||
box_color="#00FF00",
|
||||
box_thickness=2,
|
||||
keypoint_color="#FF0000",
|
||||
keypoint_radius=2,
|
||||
label_color="#FFFFFF",
|
||||
label_bg_color="#00FF00",
|
||||
):
|
||||
"""Main function that processes the batch of images and detections."""
|
||||
return self.process_batch(
|
||||
face_detections,
|
||||
face_detections,
|
||||
image,
|
||||
self.draw_faces,
|
||||
box_color=box_color,
|
||||
@@ -46,26 +76,35 @@ class VisualizeFaceDetections(BaseDetectionVisualizationNode):
|
||||
keypoint_color=keypoint_color,
|
||||
keypoint_radius=keypoint_radius,
|
||||
label_color=label_color,
|
||||
label_bg_color=label_bg_color
|
||||
label_bg_color=label_bg_color,
|
||||
)
|
||||
|
||||
def draw_faces(self, image, width, height, detections,
|
||||
box_color="#00FF00", box_thickness=2,
|
||||
keypoint_color="#FF0000", keypoint_radius=2,
|
||||
label_color="#FFFFFF", label_bg_color="#00FF00"):
|
||||
|
||||
def draw_faces(
|
||||
self,
|
||||
image,
|
||||
width,
|
||||
height,
|
||||
detections,
|
||||
box_color="#00FF00",
|
||||
box_thickness=2,
|
||||
keypoint_color="#FF0000",
|
||||
keypoint_radius=2,
|
||||
label_color="#FFFFFF",
|
||||
label_bg_color="#00FF00",
|
||||
):
|
||||
"""Callback function to draw faces on a single image."""
|
||||
# Convert hex colors to BGR tuples for cv2
|
||||
box_color_bgr = self._hex_to_bgr(box_color)
|
||||
keypoint_color_bgr = self._hex_to_bgr(keypoint_color)
|
||||
|
||||
|
||||
for detection in detections:
|
||||
bbox = detection.bounding_box
|
||||
x1, y1 = bbox.origin_x, bbox.origin_y
|
||||
x2, y2 = bbox.origin_x + bbox.width, bbox.origin_y + bbox.height
|
||||
|
||||
|
||||
# Draw bounding box
|
||||
cv2.rectangle(image, (int(x1), int(y1)), (int(x2), int(y2)), box_color_bgr, box_thickness)
|
||||
|
||||
|
||||
# Draw Keypoints
|
||||
if detection.keypoints:
|
||||
for kp in detection.keypoints:
|
||||
@@ -74,34 +113,25 @@ class VisualizeFaceDetections(BaseDetectionVisualizationNode):
|
||||
|
||||
# Prepare label text (confidence score)
|
||||
label_text = f"Face: {detection.score:.2f}" if detection.score is not None else "Face"
|
||||
|
||||
|
||||
# Draw label with background
|
||||
try:
|
||||
self.draw_text_with_background(
|
||||
image,
|
||||
label_text,
|
||||
(int(x1), int(y1)),
|
||||
text_color=label_color,
|
||||
bg_color=label_bg_color
|
||||
)
|
||||
self.draw_text_with_background(image, label_text, (int(x1), int(y1)), text_color=label_color, bg_color=label_bg_color)
|
||||
except Exception as e:
|
||||
logger.error(f"Error drawing text label: {e}")
|
||||
# Fallback using basic cv2 text
|
||||
cv2.putText(
|
||||
image,
|
||||
label_text,
|
||||
(int(x1) + 2, int(y1) + 15),
|
||||
cv2.FONT_HERSHEY_SIMPLEX,
|
||||
0.5,
|
||||
self._hex_to_bgr(label_color),
|
||||
1
|
||||
image,
|
||||
label_text,
|
||||
(int(x1) + 2, int(y1) + 15),
|
||||
cv2.FONT_HERSHEY_SIMPLEX,
|
||||
0.5,
|
||||
self._hex_to_bgr(label_color),
|
||||
1,
|
||||
)
|
||||
|
||||
# Node registration
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"VisualizeFaceDetections": VisualizeFaceDetections
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"VisualizeFaceDetections": "Visualize Face Detections (MediaPipe)"
|
||||
}
|
||||
# Node registration
|
||||
NODE_CLASS_MAPPINGS = {"VisualizeFaceDetections": VisualizeFaceDetections}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {"VisualizeFaceDetections": "Visualize Face Detections (MediaPipe)"}
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
# node_wrappers/face/blendshape.py
|
||||
import logging
|
||||
from typing import Optional, List, Any, Dict, Tuple
|
||||
|
||||
from .....src.mediapipe_vision.types import BLENDSHAPES_LIST # Updated type alias
|
||||
from .....src.mediapipe_vision.utils.blendshape_utils import get_blendshape_categories # Utility to get all category names
|
||||
from typing import Optional
|
||||
|
||||
# Import base trigger/control logic if we adapt base classes later, or implement directly for now
|
||||
from .....src.mediapipe_vision.common.position import FLOAT_EQUALITY_TOLERANCE # For trigger node comparisons
|
||||
from .....src.mediapipe_vision.common.position import (
|
||||
FLOAT_EQUALITY_TOLERANCE, # For trigger node comparisons
|
||||
)
|
||||
from .....src.mediapipe_vision.types import BLENDSHAPES_LIST # Updated type alias
|
||||
from .....src.mediapipe_vision.utils.blendshape_utils import (
|
||||
get_blendshape_categories, # Utility to get all category names
|
||||
)
|
||||
from .....src.utils.math import scale_value
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -16,7 +19,7 @@ logger = logging.getLogger(__name__)
|
||||
ALL_BLENDSHAPE_CATEGORIES = get_blendshape_categories()
|
||||
if not ALL_BLENDSHAPE_CATEGORIES:
|
||||
logger.warning("Could not retrieve blendshape categories. Blendshape nodes might not function correctly.")
|
||||
ALL_BLENDSHAPE_CATEGORIES = ["_neutral"] # Provide a fallback
|
||||
ALL_BLENDSHAPE_CATEGORIES = ["_neutral"] # Provide a fallback
|
||||
|
||||
_category = "Realtime Nodes/MediaPipe Vision/Face/Blendshape"
|
||||
|
||||
@@ -25,8 +28,8 @@ _category = "Realtime Nodes/MediaPipe Vision/Face/Blendshape"
|
||||
def get_blendshape_score(blendshapes_result: BLENDSHAPES_LIST, category_name: str) -> Optional[float]:
|
||||
"""Extracts the score for a specific blendshape category from the result."""
|
||||
if not blendshapes_result or not blendshapes_result[0] or not blendshapes_result[0][0]:
|
||||
#logger.debug(f"No blendshapes found in result.")
|
||||
return None # No blendshape data
|
||||
# logger.debug(f"No blendshapes found in result.")
|
||||
return None # No blendshape data
|
||||
|
||||
# BLENDSHAPES structure: List[List[Dict[str, float]]]
|
||||
# We typically care about the first face in the first batch item
|
||||
@@ -41,12 +44,15 @@ def get_blendshape_score(blendshapes_result: BLENDSHAPES_LIST, category_name: st
|
||||
elif isinstance(first_face_blendshapes, list):
|
||||
# Try to handle the list format with objects that have category_name attribute
|
||||
try:
|
||||
target_blendshape = next((bs for bs in first_face_blendshapes if hasattr(bs, 'category_name') and bs.category_name == category_name), None)
|
||||
target_blendshape = next(
|
||||
(bs for bs in first_face_blendshapes if hasattr(bs, "category_name") and bs.category_name == category_name),
|
||||
None,
|
||||
)
|
||||
if target_blendshape:
|
||||
return target_blendshape.score
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing blendshape list: {e}")
|
||||
|
||||
|
||||
logger.warning(f"Blendshape category '{category_name}' not found in results. Type: {type(first_face_blendshapes)}")
|
||||
return None
|
||||
|
||||
@@ -54,6 +60,7 @@ def get_blendshape_score(blendshapes_result: BLENDSHAPES_LIST, category_name: st
|
||||
# --- Control Nodes ---
|
||||
class BlendshapeControlFloatNode:
|
||||
"""Outputs a scaled FLOAT based on the score of a selected blendshape."""
|
||||
|
||||
CATEGORY = _category
|
||||
RETURN_TYPES = ("FLOAT",)
|
||||
FUNCTION = "execute"
|
||||
@@ -63,20 +70,57 @@ class BlendshapeControlFloatNode:
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"blendshapes": ("BLENDSHAPES_LIST", {"tooltip": "Facial expression data from the Face Landmarker node"}),
|
||||
"blendshape_name": (ALL_BLENDSHAPE_CATEGORIES, {"default": "mouthSmileLeft",
|
||||
"tooltip": "The specific facial expression to monitor (e.g., smile, frown, raised eyebrows)"}),
|
||||
"blendshapes": (
|
||||
"BLENDSHAPES_LIST",
|
||||
{"tooltip": "Facial expression data from the Face Landmarker node"},
|
||||
),
|
||||
"blendshape_name": (
|
||||
ALL_BLENDSHAPE_CATEGORIES,
|
||||
{
|
||||
"default": "mouthSmileLeft",
|
||||
"tooltip": "The specific facial expression to monitor (e.g., smile, frown, raised eyebrows)",
|
||||
},
|
||||
),
|
||||
# Blendshape scores are typically 0.0 to 1.0, but allow flexibility
|
||||
"score_min": ("FLOAT", {"default": 0.0, "step": 0.01,
|
||||
"tooltip": "The minimum facial expression intensity to consider (usually 0.0)"}),
|
||||
"score_max": ("FLOAT", {"default": 1.0, "step": 0.01,
|
||||
"tooltip": "The maximum facial expression intensity to consider (usually 1.0)"}),
|
||||
"output_min_float": ("FLOAT", {"default": 0.0, "step": 0.01,
|
||||
"tooltip": "The minimum output value when expression is at or below score_min"}),
|
||||
"output_max_float": ("FLOAT", {"default": 1.0, "step": 0.01,
|
||||
"tooltip": "The maximum output value when expression is at or above score_max"}),
|
||||
"clamp": ("BOOLEAN", {"default": True,
|
||||
"tooltip": "When enabled, restricts output values to stay within min and max range"}),
|
||||
"score_min": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "The minimum facial expression intensity to consider (usually 0.0)",
|
||||
},
|
||||
),
|
||||
"score_max": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "The maximum facial expression intensity to consider (usually 1.0)",
|
||||
},
|
||||
),
|
||||
"output_min_float": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "The minimum output value when expression is at or below score_min",
|
||||
},
|
||||
),
|
||||
"output_max_float": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "The maximum output value when expression is at or above score_max",
|
||||
},
|
||||
),
|
||||
"clamp": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": True,
|
||||
"tooltip": "When enabled, restricts output values to stay within min and max range",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,8 +133,10 @@ class BlendshapeControlFloatNode:
|
||||
output_val = scaled_value if scaled_value is not None else float(output_min_float)
|
||||
return (output_val,)
|
||||
|
||||
|
||||
class BlendshapeControlIntNode:
|
||||
"""Outputs a scaled INT based on the score of a selected blendshape."""
|
||||
|
||||
CATEGORY = _category
|
||||
RETURN_TYPES = ("INT",)
|
||||
FUNCTION = "execute"
|
||||
@@ -100,19 +146,54 @@ class BlendshapeControlIntNode:
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"blendshapes": ("BLENDSHAPES_LIST", {"tooltip": "Facial expression data from the Face Landmarker node"}),
|
||||
"blendshape_name": (ALL_BLENDSHAPE_CATEGORIES, {"default": "mouthSmileLeft",
|
||||
"tooltip": "The specific facial expression to monitor (e.g., smile, frown, raised eyebrows)"}),
|
||||
"score_min": ("FLOAT", {"default": 0.0, "step": 0.01,
|
||||
"tooltip": "The minimum facial expression intensity to consider (usually 0.0)"}),
|
||||
"score_max": ("FLOAT", {"default": 1.0, "step": 0.01,
|
||||
"tooltip": "The maximum facial expression intensity to consider (usually 1.0)"}),
|
||||
"output_min_int": ("INT", {"default": 0,
|
||||
"tooltip": "The minimum integer output value when expression is at or below score_min"}),
|
||||
"output_max_int": ("INT", {"default": 100,
|
||||
"tooltip": "The maximum integer output value when expression is at or above score_max"}),
|
||||
"clamp": ("BOOLEAN", {"default": True,
|
||||
"tooltip": "When enabled, restricts output values to stay within min and max range"}),
|
||||
"blendshapes": (
|
||||
"BLENDSHAPES_LIST",
|
||||
{"tooltip": "Facial expression data from the Face Landmarker node"},
|
||||
),
|
||||
"blendshape_name": (
|
||||
ALL_BLENDSHAPE_CATEGORIES,
|
||||
{
|
||||
"default": "mouthSmileLeft",
|
||||
"tooltip": "The specific facial expression to monitor (e.g., smile, frown, raised eyebrows)",
|
||||
},
|
||||
),
|
||||
"score_min": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "The minimum facial expression intensity to consider (usually 0.0)",
|
||||
},
|
||||
),
|
||||
"score_max": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "The maximum facial expression intensity to consider (usually 1.0)",
|
||||
},
|
||||
),
|
||||
"output_min_int": (
|
||||
"INT",
|
||||
{
|
||||
"default": 0,
|
||||
"tooltip": "The minimum integer output value when expression is at or below score_min",
|
||||
},
|
||||
),
|
||||
"output_max_int": (
|
||||
"INT",
|
||||
{
|
||||
"default": 100,
|
||||
"tooltip": "The maximum integer output value when expression is at or above score_max",
|
||||
},
|
||||
),
|
||||
"clamp": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": True,
|
||||
"tooltip": "When enabled, restricts output values to stay within min and max range",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,6 +210,7 @@ class BlendshapeControlIntNode:
|
||||
# --- Trigger Node ---
|
||||
class BlendshapeTriggerNode:
|
||||
"""Outputs a BOOLEAN trigger if a blendshape score crosses a threshold."""
|
||||
|
||||
CATEGORY = _category
|
||||
RETURN_TYPES = ("BOOLEAN",)
|
||||
FUNCTION = "execute"
|
||||
@@ -138,13 +220,31 @@ class BlendshapeTriggerNode:
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"blendshapes": ("BLENDSHAPES_LIST", {"tooltip": "Facial expression data from the Face Landmarker node"}),
|
||||
"blendshape_name": (ALL_BLENDSHAPE_CATEGORIES, {"default": "jawOpen",
|
||||
"tooltip": "The specific facial expression to monitor (e.g., smile, frown, open mouth)"}),
|
||||
"threshold": ("FLOAT", {"default": 0.5, "step": 0.01,
|
||||
"tooltip": "The expression intensity threshold value that triggers the condition"}),
|
||||
"condition": (["Above", "Below", "Equals", "Not Equals"],
|
||||
{"tooltip": "The condition that determines when to trigger - Above (expression > threshold), Below (expression < threshold), etc."}),
|
||||
"blendshapes": (
|
||||
"BLENDSHAPES_LIST",
|
||||
{"tooltip": "Facial expression data from the Face Landmarker node"},
|
||||
),
|
||||
"blendshape_name": (
|
||||
ALL_BLENDSHAPE_CATEGORIES,
|
||||
{
|
||||
"default": "jawOpen",
|
||||
"tooltip": "The specific facial expression to monitor (e.g., smile, frown, open mouth)",
|
||||
},
|
||||
),
|
||||
"threshold": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"step": 0.01,
|
||||
"tooltip": "The expression intensity threshold value that triggers the condition",
|
||||
},
|
||||
),
|
||||
"condition": (
|
||||
["Above", "Below", "Equals", "Not Equals"],
|
||||
{
|
||||
"tooltip": "The condition that determines when to trigger - Above (expression > threshold), Below (expression < threshold), etc."
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -176,4 +276,4 @@ NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"BlendshapeControlFloat": "Blendshape Control (Float)",
|
||||
"BlendshapeControlInt": "Blendshape Control (Int)",
|
||||
"BlendshapeTrigger": "Blendshape Trigger",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,20 +1,26 @@
|
||||
"""Node wrapper for face detection."""
|
||||
|
||||
import logging
|
||||
|
||||
import torch
|
||||
|
||||
from .....src.mediapipe_vision.face_landmark.detector import FaceLandmarkDetector
|
||||
from ...common.base_detector_node import BaseMediaPipeDetectorNode
|
||||
|
||||
# Import Base Classes
|
||||
from ...common.model_loader import MediaPipeModelLoaderBaseNode
|
||||
from ...common.base_detector_node import BaseMediaPipeDetectorNode
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_category = "Realtime Nodes/MediaPipe Vision/Face/FaceLandmark"
|
||||
|
||||
|
||||
# Inherit from Base Loader
|
||||
class MediaPipeFaceLandmarkerModelLoaderNode(MediaPipeModelLoaderBaseNode):
|
||||
"""ComfyUI node for loading MediaPipe Face Landmarker models."""
|
||||
|
||||
TASK_TYPE = "face_landmarker"
|
||||
RETURN_TYPES = ("FACE_LANDMARKER_MODEL_INFO",) # Specific return type
|
||||
RETURN_TYPES = ("FACE_LANDMARKER_MODEL_INFO",) # Specific return type
|
||||
RETURN_NAMES = ("model_info",)
|
||||
CATEGORY = _category
|
||||
DESCRIPTION = "Loads a MediaPipe Face Landmarker model which detects facial landmarks, expressions and face orientation. Required before using the Face Landmarker node."
|
||||
@@ -38,37 +44,90 @@ class MediaPipeFaceLandmarkerNode(BaseMediaPipeDetectorNode):
|
||||
def INPUT_TYPES(cls):
|
||||
# Start with the base inputs from the parent class
|
||||
inputs = super().INPUT_TYPES()
|
||||
|
||||
|
||||
# Add face-specific parameters
|
||||
inputs["required"].update({
|
||||
"max_results": ("INT", {"default": 1, "min": 1, "max": 10, "step": 1,
|
||||
"tooltip": "Maximum number of faces to analyze - higher values detect landmarks on more faces but use more processing power"}),
|
||||
"min_confidence": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
"tooltip": "Minimum confidence threshold for detecting faces - lower finds more faces but may include false positives"}),
|
||||
"min_presence_confidence": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
"tooltip": "(VIDEO mode) Minimum confidence that a face is present - lower values may detect faces that are less clear"}),
|
||||
"min_tracking_confidence": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
"tooltip": "(VIDEO mode) Minimum confidence for tracking a face between frames - lower values maintain tracking longer but may drift"}),
|
||||
"output_blendshapes": ("BOOLEAN", {"default": False,
|
||||
"tooltip": "When enabled, outputs facial expression data (smile, frown, etc.) useful for animation or emotion analysis"}),
|
||||
"output_transform_matrix": ("BOOLEAN", {"default": False,
|
||||
"tooltip": "When enabled, outputs 3D face orientation data (head pose) useful for AR/VR applications"}),
|
||||
})
|
||||
|
||||
inputs["required"].update(
|
||||
{
|
||||
"max_results": (
|
||||
"INT",
|
||||
{
|
||||
"default": 1,
|
||||
"min": 1,
|
||||
"max": 10,
|
||||
"step": 1,
|
||||
"tooltip": "Maximum number of faces to analyze - higher values detect landmarks on more faces but use more processing power",
|
||||
},
|
||||
),
|
||||
"min_confidence": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Minimum confidence threshold for detecting faces - lower finds more faces but may include false positives",
|
||||
},
|
||||
),
|
||||
"min_presence_confidence": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "(VIDEO mode) Minimum confidence that a face is present - lower values may detect faces that are less clear",
|
||||
},
|
||||
),
|
||||
"min_tracking_confidence": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "(VIDEO mode) Minimum confidence for tracking a face between frames - lower values maintain tracking longer but may drift",
|
||||
},
|
||||
),
|
||||
"output_blendshapes": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "When enabled, outputs facial expression data (smile, frown, etc.) useful for animation or emotion analysis",
|
||||
},
|
||||
),
|
||||
"output_transform_matrix": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "When enabled, outputs 3D face orientation data (head pose) useful for AR/VR applications",
|
||||
},
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
return inputs
|
||||
|
||||
def detect(self, image: torch.Tensor, model_info: dict, max_results: int,
|
||||
min_confidence: float, min_presence_confidence: float,
|
||||
min_tracking_confidence: float, output_blendshapes: bool,
|
||||
output_transform_matrix: bool, running_mode: str, delegate: str):
|
||||
def detect(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
model_info: dict,
|
||||
max_results: int,
|
||||
min_confidence: float,
|
||||
min_presence_confidence: float,
|
||||
min_tracking_confidence: float,
|
||||
output_blendshapes: bool,
|
||||
output_transform_matrix: bool,
|
||||
running_mode: str,
|
||||
delegate: str,
|
||||
):
|
||||
"""Performs face landmark detection with the configured parameters."""
|
||||
|
||||
|
||||
# Validate model_info and get model path
|
||||
model_path = self.validate_model_info(model_info)
|
||||
|
||||
|
||||
# Initialize or update detector
|
||||
detector = self.initialize_or_update_detector(model_path)
|
||||
|
||||
|
||||
# Perform detection with all parameters
|
||||
face_landmarks_batch, blendshapes_batch, matrices_batch = detector.detect(
|
||||
image,
|
||||
@@ -79,16 +138,17 @@ class MediaPipeFaceLandmarkerNode(BaseMediaPipeDetectorNode):
|
||||
output_blendshapes=output_blendshapes,
|
||||
output_transform_matrix=output_transform_matrix,
|
||||
running_mode=running_mode,
|
||||
delegate=delegate
|
||||
delegate=delegate,
|
||||
)
|
||||
|
||||
|
||||
# Return the results with optional empty lists for disabled outputs
|
||||
return (
|
||||
face_landmarks_batch,
|
||||
blendshapes_batch if output_blendshapes else [],
|
||||
matrices_batch if output_transform_matrix else []
|
||||
face_landmarks_batch,
|
||||
blendshapes_batch if output_blendshapes else [],
|
||||
matrices_batch if output_transform_matrix else [],
|
||||
)
|
||||
|
||||
|
||||
# Define mappings for ComfyUI
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"MediaPipeFaceLandmarkerModelLoader": MediaPipeFaceLandmarkerModelLoaderNode,
|
||||
@@ -98,4 +158,4 @@ NODE_CLASS_MAPPINGS = {
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"MediaPipeFaceLandmarkerModelLoader": "Load Face Landmarker Model (MediaPipe)",
|
||||
"MediaPipeFaceLandmarker": "Face Landmarker (MediaPipe)",
|
||||
}
|
||||
}
|
||||
|
||||
+9
-10
@@ -1,15 +1,17 @@
|
||||
# visualize_face.py
|
||||
import logging
|
||||
from typing import Any, List, Optional
|
||||
|
||||
import mediapipe as mp
|
||||
from typing import Optional, List, Any
|
||||
|
||||
from ...common.base_visualization_nodes import BaseLandmarkVisualizationNode
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Define standard face connections
|
||||
FACE_CONNECTIONS = mp.solutions.face_mesh.FACEMESH_TESSELATION
|
||||
|
||||
|
||||
class VisualizeFaceLandmarks(BaseLandmarkVisualizationNode):
|
||||
LANDMARKS_TYPE = "FACE_LANDMARKS"
|
||||
CONNECTIONS = FACE_CONNECTIONS
|
||||
@@ -21,15 +23,12 @@ class VisualizeFaceLandmarks(BaseLandmarkVisualizationNode):
|
||||
SUPPORTS_VISIBILITY = False
|
||||
CATEGORY = "Realtime Nodes/MediaPipe Vision/Face/FaceLandmark/Visualization"
|
||||
DESCRIPTION = "Visualizes facial landmarks by drawing a mesh of points and connections on the face. Creates a detailed facial wireframe that shows the precise contours and features of detected faces."
|
||||
|
||||
|
||||
def get_landmark_list(self, detection_result: Any) -> Optional[List[Any]]:
|
||||
return detection_result
|
||||
return detection_result
|
||||
|
||||
|
||||
# Node registration
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"VisualizeFaceLandmarks": VisualizeFaceLandmarks
|
||||
}
|
||||
NODE_CLASS_MAPPINGS = {"VisualizeFaceLandmarks": VisualizeFaceLandmarks}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"VisualizeFaceLandmarks": "Visualize Face Landmarks (MediaPipe)"
|
||||
}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {"VisualizeFaceLandmarks": "Visualize Face Landmarks (MediaPipe)"}
|
||||
|
||||
@@ -1,18 +1,54 @@
|
||||
# node_wrappers/face/masking.py
|
||||
import torch
|
||||
import numpy as np
|
||||
import cv2
|
||||
import logging
|
||||
from typing import List, Dict, Tuple
|
||||
from typing import List
|
||||
|
||||
import torch
|
||||
|
||||
from .....src.mediapipe_vision.types import LandmarkPoint
|
||||
from .....src.utils.image import create_mask_from_points
|
||||
from .....src.utils.image import create_mask_from_points
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_category = "Realtime Nodes/MediaPipe Vision/Face/FaceLandmark/Masking"
|
||||
# --- Face Part Definitions ---
|
||||
FACE_PART_INDICES = { # NOTE: These indices are EXAMPLES and NEED VERIFICATION!
|
||||
"FACE_OVAL": [10, 338, 297, 332, 284, 251, 389, 356, 454, 323, 361, 288, 397, 365, 379, 378, 400, 377, 152, 148, 176, 149, 150, 136, 172, 58, 132, 93, 234, 127, 162, 21, 54, 103, 67, 109],
|
||||
# --- Face Part Definitions ---
|
||||
FACE_PART_INDICES = { # NOTE: These indices are EXAMPLES and NEED VERIFICATION!
|
||||
"FACE_OVAL": [
|
||||
10,
|
||||
338,
|
||||
297,
|
||||
332,
|
||||
284,
|
||||
251,
|
||||
389,
|
||||
356,
|
||||
454,
|
||||
323,
|
||||
361,
|
||||
288,
|
||||
397,
|
||||
365,
|
||||
379,
|
||||
378,
|
||||
400,
|
||||
377,
|
||||
152,
|
||||
148,
|
||||
176,
|
||||
149,
|
||||
150,
|
||||
136,
|
||||
172,
|
||||
58,
|
||||
132,
|
||||
93,
|
||||
234,
|
||||
127,
|
||||
162,
|
||||
21,
|
||||
54,
|
||||
103,
|
||||
67,
|
||||
109,
|
||||
],
|
||||
"LIPS_OUTER": [61, 146, 91, 181, 84, 17, 314, 405, 321, 375, 291, 62, 185, 40, 39, 37, 0, 267, 269, 270, 409, 292],
|
||||
"LIPS_INNER": [78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 76, 191, 81, 82, 13, 312, 311, 310, 415, 306],
|
||||
"LEFT_EYE": [33, 7, 163, 144, 145, 153, 154, 155, 133, 246, 161, 160, 159, 158, 157, 173],
|
||||
@@ -23,9 +59,11 @@ FACE_PART_INDICES = { # NOTE: These indices are EXAMPLES and NEED VERIFICATION!
|
||||
"NOSE_TIP_LOWER": [5, 4, 1, 19, 94, 2],
|
||||
}
|
||||
|
||||
# --- Node Definition ---
|
||||
|
||||
# --- Node Definition ---
|
||||
class MaskFromFaceLandmarksNode:
|
||||
"""Creates a mask for a specific facial part from Face Landmarks."""
|
||||
|
||||
CATEGORY = _category
|
||||
DESCRIPTION = "Creates precise masks for specific facial features like eyes, lips, nose, or the entire face. Perfect for targeted editing, inpainting, or creating effects on specific facial regions."
|
||||
|
||||
@@ -34,10 +72,21 @@ class MaskFromFaceLandmarksNode:
|
||||
part_names = list(FACE_PART_INDICES.keys())
|
||||
return {
|
||||
"required": {
|
||||
"face_landmarks": ("FACE_LANDMARKS", {"tooltip": "Facial landmark data from the Face Landmarker node containing the precise locations of facial features"}),
|
||||
"image_for_dimensions": ("IMAGE", {"tooltip": "Reference image to determine the size of the output mask - the mask will match these dimensions"}),
|
||||
"part_name": (part_names, {"default": "FACE_OVAL",
|
||||
"tooltip": "Choose which facial feature to create a mask for - FACE_OVAL (whole face), LIPS_OUTER (mouth), LEFT_EYE, etc."}),
|
||||
"face_landmarks": (
|
||||
"FACE_LANDMARKS",
|
||||
{"tooltip": "Facial landmark data from the Face Landmarker node containing the precise locations of facial features"},
|
||||
),
|
||||
"image_for_dimensions": (
|
||||
"IMAGE",
|
||||
{"tooltip": "Reference image to determine the size of the output mask - the mask will match these dimensions"},
|
||||
),
|
||||
"part_name": (
|
||||
part_names,
|
||||
{
|
||||
"default": "FACE_OVAL",
|
||||
"tooltip": "Choose which facial feature to create a mask for - FACE_OVAL (whole face), LIPS_OUTER (mouth), LEFT_EYE, etc.",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,7 +96,7 @@ class MaskFromFaceLandmarksNode:
|
||||
def generate_mask(self, face_landmarks: List[List[List[LandmarkPoint]]], image_for_dimensions: torch.Tensor, part_name: str):
|
||||
if part_name not in FACE_PART_INDICES:
|
||||
raise ValueError(f"Unknown face part name: '{part_name}'. Available: {list(FACE_PART_INDICES.keys())}")
|
||||
|
||||
|
||||
# Get dimensions from image
|
||||
if image_for_dimensions.dim() != 4:
|
||||
raise ValueError("image_for_dimensions must be BHWC format.")
|
||||
@@ -58,22 +107,24 @@ class MaskFromFaceLandmarksNode:
|
||||
|
||||
# Validate batch sizes match
|
||||
if len(face_landmarks) != input_batch_size:
|
||||
logger.warning(f"Batch size mismatch between face_landmarks ({len(face_landmarks)}) and image_for_dimensions ({input_batch_size}). Using landmark batch size.")
|
||||
logger.warning(
|
||||
f"Batch size mismatch between face_landmarks ({len(face_landmarks)}) and image_for_dimensions ({input_batch_size}). Using landmark batch size."
|
||||
)
|
||||
# Adjust effective batch size based on landmarks if mismatched, though ideally they match
|
||||
batch_size = len(face_landmarks)
|
||||
else:
|
||||
batch_size = input_batch_size
|
||||
|
||||
batch_size = input_batch_size
|
||||
|
||||
required_indices = FACE_PART_INDICES[part_name]
|
||||
output_masks = []
|
||||
|
||||
for i in range(batch_size):
|
||||
landmarks_for_image = face_landmarks[i] # List[List[LandmarkPoint]] (Faces in image)
|
||||
landmarks_for_image = face_landmarks[i] # List[List[LandmarkPoint]] (Faces in image)
|
||||
combined_points_for_image = []
|
||||
|
||||
# Combine points from all detected faces for the selected part
|
||||
for face_result_landmarks in landmarks_for_image: # face_result_landmarks is List[LandmarkPoint]
|
||||
points_dict = {lm.index: lm for lm in face_result_landmarks} # Iterate directly over the list
|
||||
for face_result_landmarks in landmarks_for_image: # face_result_landmarks is List[LandmarkPoint]
|
||||
points_dict = {lm.index: lm for lm in face_result_landmarks} # Iterate directly over the list
|
||||
part_points = []
|
||||
for idx in required_indices:
|
||||
if idx in points_dict:
|
||||
@@ -83,27 +134,28 @@ class MaskFromFaceLandmarksNode:
|
||||
part_points.append((px, py))
|
||||
# Removed warning here to avoid spam if landmarks are just missing
|
||||
combined_points_for_image.extend(part_points)
|
||||
|
||||
|
||||
# Create mask from all collected points for this image
|
||||
mask_tensor = create_mask_from_points(height, width, combined_points_for_image, device=device)
|
||||
output_masks.append(mask_tensor)
|
||||
|
||||
if not output_masks:
|
||||
logger.warning("No landmarks found to generate any masks. Returning zero mask batch.")
|
||||
output_batch = torch.zeros((input_batch_size, 1, height, width), dtype=torch.float32, device=device)
|
||||
logger.warning("No landmarks found to generate any masks. Returning zero mask batch.")
|
||||
output_batch = torch.zeros((input_batch_size, 1, height, width), dtype=torch.float32, device=device)
|
||||
else:
|
||||
# Stack along batch dimension, ensure channel dim is present
|
||||
output_batch = torch.stack(output_masks, dim=0) # Shape becomes (B, 1, H, W)
|
||||
|
||||
# Stack along batch dimension, ensure channel dim is present
|
||||
output_batch = torch.stack(output_masks, dim=0) # Shape becomes (B, 1, H, W)
|
||||
|
||||
return (output_batch,)
|
||||
|
||||
|
||||
# Remove GetImageDimensionsNode class definition
|
||||
|
||||
# --- Mappings ---
|
||||
# --- Mappings ---
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"MaskFromFaceLandmarks": MaskFromFaceLandmarksNode,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"MaskFromFaceLandmarks": "Mask From Face Landmarks (MediaPipe)",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,17 +1,21 @@
|
||||
# node_wrappers/face/head_pose.py
|
||||
import logging
|
||||
from typing import Dict, Optional
|
||||
|
||||
import numpy as np
|
||||
from scipy.spatial.transform import Rotation as R
|
||||
from typing import Optional, List, Any, Tuple, Dict
|
||||
|
||||
from .....src.mediapipe_vision.common.position import (
|
||||
FLOAT_EQUALITY_TOLERANCE, # For trigger node comparisons
|
||||
)
|
||||
from .....src.mediapipe_vision.types import TRANSFORM_MATRIX_LIST
|
||||
from .....src.utils.math import scale_value
|
||||
from .....src.mediapipe_vision.common.position import FLOAT_EQUALITY_TOLERANCE # For trigger node comparisons
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_category = "Realtime Nodes/MediaPipe Vision/Face/HeadPose"
|
||||
|
||||
|
||||
# --- Helper Functions ---
|
||||
def get_first_transform_matrix(matrix_result: TRANSFORM_MATRIX_LIST) -> Optional[np.ndarray]:
|
||||
"""Extracts the first 4x4 transformation matrix from the result."""
|
||||
@@ -24,6 +28,7 @@ def get_first_transform_matrix(matrix_result: TRANSFORM_MATRIX_LIST) -> Optional
|
||||
# FACE_TRANSFORM_MATRICES structure: List[List[np.ndarray]]
|
||||
return matrix_result[0][0]
|
||||
|
||||
|
||||
def decompose_transform_matrix(matrix: np.ndarray) -> Optional[Dict[str, float]]:
|
||||
"""Decomposes a 4x4 matrix into translation (x, y, z) and Euler angles (pitch, yaw, roll)."""
|
||||
if matrix is None or matrix.shape != (4, 4):
|
||||
@@ -36,16 +41,16 @@ def decompose_transform_matrix(matrix: np.ndarray) -> Optional[Dict[str, float]]
|
||||
# Convert rotation matrix to Euler angles (ZYX convention common for head pose)
|
||||
# Note: Scipy uses intrinsic rotations. ZYX order corresponds to yaw, pitch, roll.
|
||||
rotation = R.from_matrix(rotation_matrix)
|
||||
euler_angles_rad = rotation.as_euler('zyx', degrees=False)
|
||||
euler_angles_rad = rotation.as_euler("zyx", degrees=False)
|
||||
# Convert radians to degrees for easier interpretation
|
||||
euler_angles_deg = np.degrees(euler_angles_rad)
|
||||
|
||||
|
||||
return {
|
||||
"x": translation[0],
|
||||
"y": translation[1],
|
||||
"z": translation[2],
|
||||
"yaw": euler_angles_deg[0], # Rotation around Y (vertical) axis
|
||||
"pitch": euler_angles_deg[1], # Rotation around X (side-to-side) axis
|
||||
"yaw": euler_angles_deg[0], # Rotation around Y (vertical) axis
|
||||
"pitch": euler_angles_deg[1], # Rotation around X (side-to-side) axis
|
||||
"roll": euler_angles_deg[2], # Rotation around Z (front-to-back) axis
|
||||
}
|
||||
except Exception as e:
|
||||
@@ -56,8 +61,10 @@ def decompose_transform_matrix(matrix: np.ndarray) -> Optional[Dict[str, float]]
|
||||
# --- Control Node ---
|
||||
HEAD_POSE_COMPONENTS = ["x", "y", "z", "pitch", "yaw", "roll"]
|
||||
|
||||
|
||||
class HeadPoseControlFloatNode:
|
||||
"""Outputs a scaled FLOAT based on a selected component (translation/rotation) of the head pose."""
|
||||
|
||||
CATEGORY = _category
|
||||
RETURN_TYPES = ("FLOAT",)
|
||||
FUNCTION = "execute"
|
||||
@@ -67,39 +74,80 @@ class HeadPoseControlFloatNode:
|
||||
def INPUT_TYPES(cls):
|
||||
# Example ranges - these might need tuning based on expected matrix values
|
||||
default_ranges = {
|
||||
"x": (-0.5, 0.5), "y": (-0.5, 0.5), "z": (-1.0, 0.5), # Example translation ranges
|
||||
"pitch": (-45.0, 45.0), "yaw": (-60.0, 60.0), "roll": (-45.0, 45.0) # Example angle ranges (degrees)
|
||||
"x": (-0.5, 0.5),
|
||||
"y": (-0.5, 0.5),
|
||||
"z": (-1.0, 0.5), # Example translation ranges
|
||||
"pitch": (-45.0, 45.0),
|
||||
"yaw": (-60.0, 60.0),
|
||||
"roll": (-45.0, 45.0), # Example angle ranges (degrees)
|
||||
}
|
||||
default_component = "yaw"
|
||||
default_min, default_max = default_ranges[default_component]
|
||||
|
||||
|
||||
return {
|
||||
"required": {
|
||||
"face_transform_matrices": ("TRANSFORM_MATRIX_LIST", {"tooltip": "3D head orientation data from the Face Landmarker node"}),
|
||||
"component": (HEAD_POSE_COMPONENTS, {"default": default_component,
|
||||
"tooltip": "Which aspect of head movement to track - yaw (left/right), pitch (up/down), roll (tilt), or position (x/y/z)"}),
|
||||
"value_min": ("FLOAT", {"default": default_min, "step": 0.1,
|
||||
"tooltip": "The minimum value of the selected movement component to consider"}),
|
||||
"value_max": ("FLOAT", {"default": default_max, "step": 0.1,
|
||||
"tooltip": "The maximum value of the selected movement component to consider"}),
|
||||
"output_min_float": ("FLOAT", {"default": 0.0, "step": 0.01,
|
||||
"tooltip": "The minimum output value when head position is at or below value_min"}),
|
||||
"output_max_float": ("FLOAT", {"default": 1.0, "step": 0.01,
|
||||
"tooltip": "The maximum output value when head position is at or above value_max"}),
|
||||
"clamp": ("BOOLEAN", {"default": True,
|
||||
"tooltip": "When enabled, restricts output values to stay within min and max range"}),
|
||||
"face_transform_matrices": (
|
||||
"TRANSFORM_MATRIX_LIST",
|
||||
{"tooltip": "3D head orientation data from the Face Landmarker node"},
|
||||
),
|
||||
"component": (
|
||||
HEAD_POSE_COMPONENTS,
|
||||
{
|
||||
"default": default_component,
|
||||
"tooltip": "Which aspect of head movement to track - yaw (left/right), pitch (up/down), roll (tilt), or position (x/y/z)",
|
||||
},
|
||||
),
|
||||
"value_min": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": default_min,
|
||||
"step": 0.1,
|
||||
"tooltip": "The minimum value of the selected movement component to consider",
|
||||
},
|
||||
),
|
||||
"value_max": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": default_max,
|
||||
"step": 0.1,
|
||||
"tooltip": "The maximum value of the selected movement component to consider",
|
||||
},
|
||||
),
|
||||
"output_min_float": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "The minimum output value when head position is at or below value_min",
|
||||
},
|
||||
),
|
||||
"output_max_float": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "The maximum output value when head position is at or above value_max",
|
||||
},
|
||||
),
|
||||
"clamp": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": True,
|
||||
"tooltip": "When enabled, restricts output values to stay within min and max range",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
def execute(self, face_transform_matrices, component, value_min, value_max, output_min_float, output_max_float, clamp):
|
||||
matrix = get_first_transform_matrix(face_transform_matrices)
|
||||
decomposed = decompose_transform_matrix(matrix)
|
||||
|
||||
|
||||
raw_value = None
|
||||
if decomposed and component in decomposed:
|
||||
raw_value = decomposed[component]
|
||||
else:
|
||||
logger.debug(f"Could not get head pose component '{component}'")
|
||||
logger.debug(f"Could not get head pose component '{component}'")
|
||||
|
||||
scaled_value = scale_value(raw_value, value_min, value_max, output_min_float, output_max_float, clamp)
|
||||
|
||||
@@ -107,8 +155,10 @@ class HeadPoseControlFloatNode:
|
||||
output_val = scaled_value if scaled_value is not None else float(output_min_float)
|
||||
return (output_val,)
|
||||
|
||||
|
||||
class HeadPoseControlIntNode:
|
||||
"""Outputs a scaled INT based on a selected component (translation/rotation) of the head pose."""
|
||||
|
||||
CATEGORY = _category
|
||||
RETURN_TYPES = ("INT",)
|
||||
FUNCTION = "execute"
|
||||
@@ -118,40 +168,79 @@ class HeadPoseControlIntNode:
|
||||
def INPUT_TYPES(cls):
|
||||
# Share the same default range logic as Float node
|
||||
default_ranges = {
|
||||
"x": (-0.5, 0.5), "y": (-0.5, 0.5), "z": (-1.0, 0.5),
|
||||
"pitch": (-45.0, 45.0), "yaw": (-60.0, 60.0), "roll": (-45.0, 45.0)
|
||||
"x": (-0.5, 0.5),
|
||||
"y": (-0.5, 0.5),
|
||||
"z": (-1.0, 0.5),
|
||||
"pitch": (-45.0, 45.0),
|
||||
"yaw": (-60.0, 60.0),
|
||||
"roll": (-45.0, 45.0),
|
||||
}
|
||||
default_component = "yaw"
|
||||
default_min, default_max = default_ranges[default_component]
|
||||
|
||||
|
||||
return {
|
||||
"required": {
|
||||
"face_transform_matrices": ("TRANSFORM_MATRIX_LIST", {"tooltip": "3D head orientation data from the Face Landmarker node"}),
|
||||
"component": (HEAD_POSE_COMPONENTS, {"default": default_component,
|
||||
"tooltip": "Which aspect of head movement to track - yaw (left/right), pitch (up/down), roll (tilt), or position (x/y/z)"}),
|
||||
"value_min": ("FLOAT", {"default": default_min, "step": 0.1,
|
||||
"tooltip": "The minimum value of the selected movement component to consider"}),
|
||||
"value_max": ("FLOAT", {"default": default_max, "step": 0.1,
|
||||
"tooltip": "The maximum value of the selected movement component to consider"}),
|
||||
"output_min_int": ("INT", {"default": 0,
|
||||
"tooltip": "The minimum integer output value when head position is at or below value_min"}),
|
||||
"output_max_int": ("INT", {"default": 100,
|
||||
"tooltip": "The maximum integer output value when head position is at or above value_max"}),
|
||||
"clamp": ("BOOLEAN", {"default": True,
|
||||
"tooltip": "When enabled, restricts output values to stay within min and max range"}),
|
||||
"face_transform_matrices": (
|
||||
"TRANSFORM_MATRIX_LIST",
|
||||
{"tooltip": "3D head orientation data from the Face Landmarker node"},
|
||||
),
|
||||
"component": (
|
||||
HEAD_POSE_COMPONENTS,
|
||||
{
|
||||
"default": default_component,
|
||||
"tooltip": "Which aspect of head movement to track - yaw (left/right), pitch (up/down), roll (tilt), or position (x/y/z)",
|
||||
},
|
||||
),
|
||||
"value_min": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": default_min,
|
||||
"step": 0.1,
|
||||
"tooltip": "The minimum value of the selected movement component to consider",
|
||||
},
|
||||
),
|
||||
"value_max": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": default_max,
|
||||
"step": 0.1,
|
||||
"tooltip": "The maximum value of the selected movement component to consider",
|
||||
},
|
||||
),
|
||||
"output_min_int": (
|
||||
"INT",
|
||||
{
|
||||
"default": 0,
|
||||
"tooltip": "The minimum integer output value when head position is at or below value_min",
|
||||
},
|
||||
),
|
||||
"output_max_int": (
|
||||
"INT",
|
||||
{
|
||||
"default": 100,
|
||||
"tooltip": "The maximum integer output value when head position is at or above value_max",
|
||||
},
|
||||
),
|
||||
"clamp": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": True,
|
||||
"tooltip": "When enabled, restricts output values to stay within min and max range",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
def execute(self, face_transform_matrices, component, value_min, value_max, output_min_int, output_max_int, clamp):
|
||||
matrix = get_first_transform_matrix(face_transform_matrices)
|
||||
decomposed = decompose_transform_matrix(matrix)
|
||||
|
||||
|
||||
raw_value = None
|
||||
if decomposed and component in decomposed:
|
||||
raw_value = decomposed[component]
|
||||
else:
|
||||
logger.debug(f"Could not get head pose component '{component}'")
|
||||
|
||||
|
||||
scaled_value = scale_value(raw_value, value_min, value_max, output_min_int, output_max_int, clamp)
|
||||
|
||||
# Default to min output value on error
|
||||
@@ -162,6 +251,7 @@ class HeadPoseControlIntNode:
|
||||
# --- Trigger Node ---
|
||||
class HeadPoseTriggerNode:
|
||||
"""Outputs a BOOLEAN trigger if a head pose component crosses a threshold."""
|
||||
|
||||
CATEGORY = _category
|
||||
RETURN_TYPES = ("BOOLEAN",)
|
||||
FUNCTION = "execute"
|
||||
@@ -171,25 +261,43 @@ class HeadPoseTriggerNode:
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"face_transform_matrices": ("TRANSFORM_MATRIX_LIST", {"tooltip": "3D head orientation data from the Face Landmarker node"}),
|
||||
"component": (HEAD_POSE_COMPONENTS, {"default": "yaw",
|
||||
"tooltip": "Which aspect of head movement to track - yaw (left/right), pitch (up/down), roll (tilt), or position (x/y/z)"}),
|
||||
"threshold": ("FLOAT", {"default": 30.0, "step": 0.5,
|
||||
"tooltip": "The angle or position value that triggers the condition (example: 30 degrees of head turn)"}),
|
||||
"condition": (["Above", "Below", "Equals", "Not Equals"],
|
||||
{"tooltip": "The condition that determines when to trigger - Above (movement > threshold), Below (movement < threshold), etc."}),
|
||||
"face_transform_matrices": (
|
||||
"TRANSFORM_MATRIX_LIST",
|
||||
{"tooltip": "3D head orientation data from the Face Landmarker node"},
|
||||
),
|
||||
"component": (
|
||||
HEAD_POSE_COMPONENTS,
|
||||
{
|
||||
"default": "yaw",
|
||||
"tooltip": "Which aspect of head movement to track - yaw (left/right), pitch (up/down), roll (tilt), or position (x/y/z)",
|
||||
},
|
||||
),
|
||||
"threshold": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 30.0,
|
||||
"step": 0.5,
|
||||
"tooltip": "The angle or position value that triggers the condition (example: 30 degrees of head turn)",
|
||||
},
|
||||
),
|
||||
"condition": (
|
||||
["Above", "Below", "Equals", "Not Equals"],
|
||||
{
|
||||
"tooltip": "The condition that determines when to trigger - Above (movement > threshold), Below (movement < threshold), etc."
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
def execute(self, face_transform_matrices, component, threshold, condition):
|
||||
matrix = get_first_transform_matrix(face_transform_matrices)
|
||||
decomposed = decompose_transform_matrix(matrix)
|
||||
|
||||
|
||||
raw_value = None
|
||||
if decomposed and component in decomposed:
|
||||
raw_value = decomposed[component]
|
||||
else:
|
||||
logger.debug(f"Could not get head pose component '{component}' for trigger")
|
||||
logger.debug(f"Could not get head pose component '{component}' for trigger")
|
||||
|
||||
triggered = False
|
||||
if raw_value is not None:
|
||||
@@ -216,4 +324,4 @@ NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"HeadPoseControlFloat": "Head Pose Control (Float)",
|
||||
"HeadPoseControlInt": "Head Pose Control (Int)",
|
||||
"HeadPoseTrigger": "Head Pose Trigger",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
import logging
|
||||
|
||||
from .....src.mediapipe_vision.common.position import LandmarkPositionBase
|
||||
from .....src.mediapipe_vision.landmark_definitions import FACE_LANDMARK_TOOLTIP
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_category = "Realtime Nodes/MediaPipe Vision/Face/FaceLandmark/Position"
|
||||
|
||||
|
||||
class FaceLandmarkPositionNode(LandmarkPositionBase):
|
||||
"""Extracts position lists (x, y, z, vis, pres) for a specific landmark index
|
||||
from FACE_LANDMARKS across the batch."""
|
||||
from FACE_LANDMARKS across the batch."""
|
||||
|
||||
CATEGORY = _category
|
||||
# Define the specific input type string this node expects
|
||||
LANDMARKS_TYPE = "FACE_LANDMARKS"
|
||||
@@ -19,12 +22,16 @@ class FaceLandmarkPositionNode(LandmarkPositionBase):
|
||||
# Define the specific input type for this node
|
||||
return {
|
||||
"required": {
|
||||
"landmarks": (cls.LANDMARKS_TYPE, {"forceInput": True}), # Use the class variable
|
||||
"landmarks": (cls.LANDMARKS_TYPE, {"forceInput": True}), # Use the class variable
|
||||
"landmark_index": ("INT", {"default": 0, "min": 0, "tooltip": FACE_LANDMARK_TOOLTIP}),
|
||||
"result_index": ("INT", {"default": 0, "min": 0, "tooltip": "Index of the face detection to use (0=first detected face)"}),
|
||||
"result_index": (
|
||||
"INT",
|
||||
{"default": 0, "min": 0, "tooltip": "Index of the face detection to use (0=first detected face)"},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# --- Node Mappings ---
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"FaceLandmarkPosition": FaceLandmarkPositionNode,
|
||||
@@ -32,4 +39,4 @@ NODE_CLASS_MAPPINGS = {
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"FaceLandmarkPosition": "Face Landmark Position Extractor (Batch)",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
# node_wrappers/face/texture_warp.py
|
||||
import logging
|
||||
import torch
|
||||
import numpy as np
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_category = "Realtime Nodes/MediaPipe Vision/Face/Experimental"
|
||||
|
||||
|
||||
class FaceTextureWarpNode:
|
||||
"""(Experimental) Attempts to warp a texture based on the face transform matrix.
|
||||
NOTE: This is a conceptual placeholder. Actual warping requires significant
|
||||
integration with sampling or image processing libraries.
|
||||
NOTE: This is a conceptual placeholder. Actual warping requires significant
|
||||
integration with sampling or image processing libraries.
|
||||
"""
|
||||
|
||||
CATEGORY = _category
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "execute"
|
||||
@@ -21,35 +21,66 @@ class FaceTextureWarpNode:
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"target_image": ("IMAGE", {"tooltip": "The base image containing a face onto which the texture will be applied"}),
|
||||
"texture_image": ("IMAGE", {"tooltip": "The texture image to be warped onto the face (could be another face, pattern, or design)"}),
|
||||
"face_transform_matrices": ("TRANSFORM_MATRIX_LIST", {"tooltip": "3D head orientation data from the Face Landmarker node to determine warping angles"}),
|
||||
"target_image": (
|
||||
"IMAGE",
|
||||
{"tooltip": "The base image containing a face onto which the texture will be applied"},
|
||||
),
|
||||
"texture_image": (
|
||||
"IMAGE",
|
||||
{"tooltip": "The texture image to be warped onto the face (could be another face, pattern, or design)"},
|
||||
),
|
||||
"face_transform_matrices": (
|
||||
"TRANSFORM_MATRIX_LIST",
|
||||
{"tooltip": "3D head orientation data from the Face Landmarker node to determine warping angles"},
|
||||
),
|
||||
# Potentially add mask input for blending?
|
||||
},
|
||||
"optional": {
|
||||
"offset_x": ("FLOAT", {"default": 0.0, "step": 0.01, "tooltip": "Horizontal positioning adjustment for the texture overlay"}),
|
||||
"offset_y": ("FLOAT", {"default": 0.0, "step": 0.01, "tooltip": "Vertical positioning adjustment for the texture overlay"}),
|
||||
"scale": ("FLOAT", {"default": 1.0, "step": 0.01, "tooltip": "Size adjustment for the texture overlay (1.0 = original size)"}),
|
||||
}
|
||||
"offset_x": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Horizontal positioning adjustment for the texture overlay",
|
||||
},
|
||||
),
|
||||
"offset_y": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Vertical positioning adjustment for the texture overlay",
|
||||
},
|
||||
),
|
||||
"scale": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Size adjustment for the texture overlay (1.0 = original size)",
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
def execute(self, target_image, texture_image, face_transform_matrices, offset_x=0.0, offset_y=0.0, scale=1.0):
|
||||
logger.warning(f"{self.__class__.__name__} is experimental and does not perform actual warping yet.")
|
||||
#TODO: implement this
|
||||
# --- Conceptual Logic ---
|
||||
# TODO: implement this
|
||||
# --- Conceptual Logic ---
|
||||
# 1. Get the primary face transform matrix
|
||||
# 2. Potentially combine with offset/scale parameters
|
||||
# 3. Define or load a canonical 3D face model mesh.
|
||||
# 4. Project the canonical mesh onto the 2D target_image plane using the matrix.
|
||||
# 5. Determine the UV coordinates on the texture_image corresponding to the projected vertices.
|
||||
# 6. Perform perspective warping of the texture_image onto the target_image using the
|
||||
# calculated transformation (e.g., using OpenCV's getPerspectiveTransform and warpPerspective,
|
||||
# 6. Perform perspective warping of the texture_image onto the target_image using the
|
||||
# calculated transformation (e.g., using OpenCV's getPerspectiveTransform and warpPerspective,
|
||||
# or more advanced 3D rendering techniques).
|
||||
# 7. Blend the warped texture onto the target image (potentially using a face mask).
|
||||
|
||||
|
||||
# For now, just return the original target image
|
||||
return (target_image,)
|
||||
|
||||
|
||||
# --- Mappings ---
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"FaceTextureWarp": FaceTextureWarpNode,
|
||||
@@ -57,4 +88,4 @@ NODE_CLASS_MAPPINGS = {
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"FaceTextureWarp": "Face Texture Warp (Experimental)",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,21 +1,28 @@
|
||||
import torch
|
||||
import logging
|
||||
|
||||
# Import Base Loader and Detector
|
||||
from ...common.model_loader import MediaPipeModelLoaderBaseNode
|
||||
import torch
|
||||
|
||||
from .....src.mediapipe_vision.face_stylization.detector import FaceStylizer
|
||||
|
||||
# Import Base Loader and Detector
|
||||
from ...common.model_loader import MediaPipeModelLoaderBaseNode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_category = "Realtime Nodes/MediaPipe Vision/Face/FaceStylization"
|
||||
# --- Model Loader ---
|
||||
|
||||
|
||||
# --- Model Loader ---
|
||||
class MediaPipeFaceStylizerModelLoaderNode(MediaPipeModelLoaderBaseNode):
|
||||
"""ComfyUI node for loading MediaPipe Face Stylizer models."""
|
||||
TASK_TYPE = "face_stylizer" # Need to add this task to MEDIAPIPE_MODELS
|
||||
|
||||
TASK_TYPE = "face_stylizer" # Need to add this task to MEDIAPIPE_MODELS
|
||||
RETURN_TYPES = ("FACE_STYLIZER_MODEL_INFO",)
|
||||
RETURN_NAMES = ("model_info",)
|
||||
CATEGORY = _category
|
||||
DESCRIPTION = "Loads a MediaPipe Face Stylizer model that can transform face images into artistic styles. Required before using the Face Stylizer node."
|
||||
# --- Stylizer Node ---
|
||||
|
||||
|
||||
# --- Stylizer Node ---
|
||||
class MediaPipeFaceStylizerNode:
|
||||
"""ComfyUI node for MediaPipe Face Stylization."""
|
||||
|
||||
@@ -28,11 +35,14 @@ class MediaPipeFaceStylizerNode:
|
||||
return {
|
||||
"required": {
|
||||
"image": ("IMAGE", {"tooltip": "Input image containing faces to be artistically stylized"}),
|
||||
"model_info": ("FACE_STYLIZER_MODEL_INFO", {"tooltip": "Face stylizer model loaded from the 'Load Face Stylizer Model' node"}),
|
||||
"model_info": (
|
||||
"FACE_STYLIZER_MODEL_INFO",
|
||||
{"tooltip": "Face stylizer model loaded from the 'Load Face Stylizer Model' node"},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",) # Output is a stylized image
|
||||
RETURN_TYPES = ("IMAGE",) # Output is a stylized image
|
||||
RETURN_NAMES = ("stylized_image",)
|
||||
FUNCTION = "stylize_face"
|
||||
CATEGORY = _category
|
||||
@@ -40,36 +50,41 @@ class MediaPipeFaceStylizerNode:
|
||||
|
||||
def stylize_face(self, image: torch.Tensor, model_info: dict):
|
||||
"""Applies face stylization."""
|
||||
|
||||
task_type = model_info.get('task_type')
|
||||
|
||||
task_type = model_info.get("task_type")
|
||||
expected_task_type = MediaPipeFaceStylizerModelLoaderNode.TASK_TYPE
|
||||
if not isinstance(model_info, dict) or task_type != expected_task_type:
|
||||
raise ValueError(f"Invalid model_info. Expected task_type '{expected_task_type}' but got '{task_type}'.")
|
||||
model_path = model_info.get('model_path')
|
||||
raise ValueError(f"Invalid model_info. Expected task_type '{expected_task_type}' but got '{task_type}'.")
|
||||
model_path = model_info.get("model_path")
|
||||
if not model_path:
|
||||
raise ValueError("Model path not found or invalid in model_info.")
|
||||
|
||||
raise ValueError("Model path not found or invalid in model_info.")
|
||||
|
||||
# Manage detector instance
|
||||
if self._detector is None or self._model_path != model_path:
|
||||
if self._detector and hasattr(self._detector, 'close'):
|
||||
try: self._detector.close()
|
||||
except Exception as e: logger.warning(f"Error closing FaceStylizer: {e}")
|
||||
if self._detector and hasattr(self._detector, "close"):
|
||||
try:
|
||||
self._detector.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing FaceStylizer: {e}")
|
||||
logger.info(f"Creating new FaceStylizer instance for {model_path}")
|
||||
self._detector = FaceStylizer(model_path)
|
||||
self._model_path = model_path
|
||||
|
||||
|
||||
# Call the detector's stylize method
|
||||
stylized_image_batch = self._detector.stylize(image)
|
||||
|
||||
|
||||
return (stylized_image_batch,)
|
||||
|
||||
def __del__(self):
|
||||
if hasattr(self, '_detector') and self._detector and hasattr(self._detector, 'close'):
|
||||
try: self._detector.close()
|
||||
except Exception as e: logger.warning(f"Error closing FaceStylizer in __del__: {e}")
|
||||
self._detector = None
|
||||
if hasattr(self, "_detector") and self._detector and hasattr(self._detector, "close"):
|
||||
try:
|
||||
self._detector.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing FaceStylizer in __del__: {e}")
|
||||
self._detector = None
|
||||
|
||||
# --- Mappings ---
|
||||
|
||||
# --- Mappings ---
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"MediaPipeFaceStylizerModelLoader": MediaPipeFaceStylizerModelLoaderNode,
|
||||
"MediaPipeFaceStylizer": MediaPipeFaceStylizerNode,
|
||||
@@ -78,4 +93,4 @@ NODE_CLASS_MAPPINGS = {
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"MediaPipeFaceStylizerModelLoader": "Load Face Stylizer Model (MediaPipe)",
|
||||
"MediaPipeFaceStylizer": "Face Stylizer (MediaPipe)",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
|
||||
@@ -1,20 +1,21 @@
|
||||
# visualize_gestures.py
|
||||
import torch
|
||||
import numpy as np
|
||||
import logging
|
||||
|
||||
import cv2
|
||||
|
||||
from ...common.base_visualization_nodes import BaseDetectionVisualizationNode
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_category = "Realtime Nodes/MediaPipe Vision/Hand/GestureVisualization"
|
||||
|
||||
|
||||
class VisualizeGestureRecognitions(BaseDetectionVisualizationNode):
|
||||
"""Draws recognized gesture labels onto images near the detected hand."""
|
||||
|
||||
CATEGORY = _category
|
||||
DESCRIPTION = "Visualizes recognized hand gestures by drawing text labels next to hands in the image. Displays gesture names like 'Thumbs Up', 'Victory', 'Open Palm' etc. near each detected hand."
|
||||
|
||||
|
||||
# Override font scale for gesture labels (larger than default)
|
||||
FONT_SCALE = 0.7 # Larger than default for gesture labels
|
||||
|
||||
@@ -22,39 +23,61 @@ class VisualizeGestureRecognitions(BaseDetectionVisualizationNode):
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"gesture_recognitions": ("GESTURE_RECOGNITIONS", {"tooltip": "Gesture recognition results from the Gesture Recognizer node"}),
|
||||
"hand_landmarks": ("HAND_LANDMARKS", {"tooltip": "Hand landmark data from the Hand Landmarker node - needed to position the text labels"}),
|
||||
"gesture_recognitions": (
|
||||
"GESTURE_RECOGNITIONS",
|
||||
{"tooltip": "Gesture recognition results from the Gesture Recognizer node"},
|
||||
),
|
||||
"hand_landmarks": (
|
||||
"HAND_LANDMARKS",
|
||||
{"tooltip": "Hand landmark data from the Hand Landmarker node - needed to position the text labels"},
|
||||
),
|
||||
"image": ("IMAGE", {"tooltip": "The image on which to draw gesture labels"}),
|
||||
},
|
||||
"optional": {
|
||||
"label_color": ("STRING", {"default": "#FFFFFF", "tooltip": "Hex color code for the gesture label text (default: white)"}),
|
||||
"label_bg_color": ("STRING", {"default": "#0000FF", "tooltip": "Hex color code for the label background (default: blue)"}),
|
||||
"y_offset": ("INT", {"default": -30, "step": 5, "tooltip": "Vertical position adjustment for text labels relative to the wrist - negative moves up, positive moves down"}),
|
||||
}
|
||||
"label_color": (
|
||||
"STRING",
|
||||
{"default": "#FFFFFF", "tooltip": "Hex color code for the gesture label text (default: white)"},
|
||||
),
|
||||
"label_bg_color": (
|
||||
"STRING",
|
||||
{"default": "#0000FF", "tooltip": "Hex color code for the label background (default: blue)"},
|
||||
),
|
||||
"y_offset": (
|
||||
"INT",
|
||||
{
|
||||
"default": -30,
|
||||
"step": 5,
|
||||
"tooltip": "Vertical position adjustment for text labels relative to the wrist - negative moves up, positive moves down",
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "draw_visualizations"
|
||||
|
||||
def draw_visualizations(self, gesture_recognitions, hand_landmarks, image,
|
||||
label_color="#FFFFFF", label_bg_color="#0000FF", y_offset=-30):
|
||||
def draw_visualizations(
|
||||
self, gesture_recognitions, hand_landmarks, image, label_color="#FFFFFF", label_bg_color="#0000FF", y_offset=-30
|
||||
):
|
||||
"""Main validation and batch processing function."""
|
||||
if not gesture_recognitions:
|
||||
logger.warning("VisualizeGestureRecognitions: No gestures provided.")
|
||||
return (image,)
|
||||
if not hand_landmarks:
|
||||
logger.warning("VisualizeGestureRecognitions: Hand landmarks required for positioning text.")
|
||||
return (image,)
|
||||
|
||||
logger.warning("VisualizeGestureRecognitions: Hand landmarks required for positioning text.")
|
||||
return (image,)
|
||||
|
||||
# Basic batch size check
|
||||
batch_size = len(image)
|
||||
if len(gesture_recognitions) != batch_size or len(hand_landmarks) != batch_size:
|
||||
logger.error(f"VisualizeGestureRecognitions: Batch size mismatch. Images: {batch_size}, Gestures: {len(gesture_recognitions)}, Landmarks: {len(hand_landmarks)}. Cannot proceed.")
|
||||
logger.error(
|
||||
f"VisualizeGestureRecognitions: Batch size mismatch. Images: {batch_size}, Gestures: {len(gesture_recognitions)}, Landmarks: {len(hand_landmarks)}. Cannot proceed."
|
||||
)
|
||||
return (image,)
|
||||
|
||||
|
||||
# Pair gestures with landmarks for convenience
|
||||
paired_data = list(zip(gesture_recognitions, hand_landmarks))
|
||||
|
||||
|
||||
# Use the base class process_batch method with our custom drawing function
|
||||
return self.process_batch(
|
||||
paired_data, # We're passing paired data instead of just detections
|
||||
@@ -62,51 +85,50 @@ class VisualizeGestureRecognitions(BaseDetectionVisualizationNode):
|
||||
self.draw_gestures,
|
||||
label_color=label_color,
|
||||
label_bg_color=label_bg_color,
|
||||
y_offset=y_offset
|
||||
y_offset=y_offset,
|
||||
)
|
||||
|
||||
def draw_gestures(self, image, width, height, paired_data,
|
||||
label_color="#FFFFFF", label_bg_color="#0000FF", y_offset=-30):
|
||||
|
||||
def draw_gestures(self, image, width, height, paired_data, label_color="#FFFFFF", label_bg_color="#0000FF", y_offset=-30):
|
||||
"""Draw gestures for a single image."""
|
||||
gestures_for_image, landmarks_for_image = paired_data
|
||||
|
||||
|
||||
# Check if number of detected hands matches for gestures and landmarks
|
||||
if len(gestures_for_image) != len(landmarks_for_image):
|
||||
logger.warning(f"VisualizeGestureRecognitions: Mismatch between detected gestures and landmarks. Skipping labels.")
|
||||
return
|
||||
|
||||
logger.warning(f"VisualizeGestureRecognitions: Mismatch between detected gestures and landmarks. Skipping labels.")
|
||||
return
|
||||
|
||||
if not gestures_for_image:
|
||||
return
|
||||
|
||||
# Iterate through each detected hand/gesture set for this image
|
||||
for hand_idx, gesture_result in enumerate(gestures_for_image):
|
||||
landmark_result = landmarks_for_image[hand_idx]
|
||||
|
||||
|
||||
# Get top gesture category
|
||||
top_gesture = "N/A"
|
||||
if gesture_result.gestures:
|
||||
top_gesture = gesture_result.gestures[0].display_name or gesture_result.gestures[0].category_name or "N/A"
|
||||
|
||||
|
||||
# Use wrist landmark (index 0) for positioning text
|
||||
if not landmark_result.landmarks or len(landmark_result.landmarks) == 0:
|
||||
logger.warning(f"VisualizeGestureRecognitions: No landmarks found for this hand to position text.")
|
||||
continue
|
||||
|
||||
wrist_lm = landmark_result.landmarks[0] # Assuming index 0 is wrist
|
||||
|
||||
wrist_lm = landmark_result.landmarks[0] # Assuming index 0 is wrist
|
||||
cx = int(wrist_lm.x * width)
|
||||
cy = int(wrist_lm.y * height) + y_offset # Apply offset
|
||||
cy = int(wrist_lm.y * height) + y_offset # Apply offset
|
||||
|
||||
label_text = f"{top_gesture}"
|
||||
|
||||
|
||||
# Draw label with background
|
||||
try:
|
||||
# Get text size
|
||||
text_width, text_height = self.get_text_dimensions(label_text, self.FONT_SCALE)
|
||||
|
||||
|
||||
# Center the text around the wrist point
|
||||
text_x = cx - (text_width // 2)
|
||||
text_y = cy - (text_height // 2)
|
||||
|
||||
|
||||
# Draw the text with background
|
||||
self.draw_text_with_background(
|
||||
image,
|
||||
@@ -114,27 +136,24 @@ class VisualizeGestureRecognitions(BaseDetectionVisualizationNode):
|
||||
(text_x, text_y),
|
||||
text_color=label_color,
|
||||
bg_color=label_bg_color,
|
||||
font_scale=self.FONT_SCALE
|
||||
font_scale=self.FONT_SCALE,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error drawing text label for gesture: {e}")
|
||||
# Fallback: Draw text directly
|
||||
cv2.putText(
|
||||
image,
|
||||
label_text,
|
||||
(cx, cy),
|
||||
cv2.FONT_HERSHEY_SIMPLEX,
|
||||
self.FONT_SCALE,
|
||||
self._hex_to_bgr(label_color),
|
||||
1,
|
||||
cv2.LINE_AA
|
||||
image,
|
||||
label_text,
|
||||
(cx, cy),
|
||||
cv2.FONT_HERSHEY_SIMPLEX,
|
||||
self.FONT_SCALE,
|
||||
self._hex_to_bgr(label_color),
|
||||
1,
|
||||
cv2.LINE_AA,
|
||||
)
|
||||
|
||||
# Node registration
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"VisualizeGestureRecognitions": VisualizeGestureRecognitions
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"VisualizeGestureRecognitions": "Visualize Gestures (MediaPipe)"
|
||||
}
|
||||
# Node registration
|
||||
NODE_CLASS_MAPPINGS = {"VisualizeGestureRecognitions": VisualizeGestureRecognitions}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {"VisualizeGestureRecognitions": "Visualize Gestures (MediaPipe)"}
|
||||
|
||||
@@ -1,22 +1,29 @@
|
||||
import torch
|
||||
import logging
|
||||
|
||||
import torch
|
||||
|
||||
from .....src.mediapipe_vision.gesture_recognition.detector import GestureRecognizer
|
||||
from ...common.base_detector_node import BaseMediaPipeDetectorNode
|
||||
|
||||
# Import Base Loader and Detector
|
||||
from ...common.model_loader import MediaPipeModelLoaderBaseNode
|
||||
from ...common.base_detector_node import BaseMediaPipeDetectorNode
|
||||
from .....src.mediapipe_vision.gesture_recognition.detector import GestureRecognizer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_category = "Realtime Nodes/MediaPipe Vision/Hand/GestureRecognition"
|
||||
# --- Model Loader ---
|
||||
|
||||
|
||||
# --- Model Loader ---
|
||||
class MediaPipeGestureRecognizerModelLoaderNode(MediaPipeModelLoaderBaseNode):
|
||||
"""ComfyUI node for loading MediaPipe Gesture Recognizer models."""
|
||||
TASK_TYPE = "gesture_recognizer" # Need to add this task to MEDIAPIPE_MODELS in src/model_loader.py
|
||||
|
||||
TASK_TYPE = "gesture_recognizer" # Need to add this task to MEDIAPIPE_MODELS in src/model_loader.py
|
||||
RETURN_TYPES = ("GESTURE_RECOGNIZER_MODEL_INFO",)
|
||||
RETURN_NAMES = ("model_info",)
|
||||
CATEGORY = _category
|
||||
DESCRIPTION = "Loads a MediaPipe Gesture Recognizer model for identifying hand gestures in images, such as thumbs up, open palm, pointing, etc. Required before using the Gesture Recognizer node."
|
||||
# --- Recognizer Node ---
|
||||
|
||||
|
||||
# --- Recognizer Node ---
|
||||
class MediaPipeGestureRecognizerNode(BaseMediaPipeDetectorNode):
|
||||
"""ComfyUI node for MediaPipe Gesture Recognition."""
|
||||
|
||||
@@ -34,32 +41,74 @@ class MediaPipeGestureRecognizerNode(BaseMediaPipeDetectorNode):
|
||||
def INPUT_TYPES(cls):
|
||||
# Start with the base inputs from the parent class
|
||||
inputs = super().INPUT_TYPES()
|
||||
|
||||
|
||||
# Add gesture-specific parameters
|
||||
inputs["required"].update({
|
||||
"max_results": ("INT", {"default": 2, "min": 1, "max": 4, "step": 1,
|
||||
"tooltip": "Maximum number of hands to analyze in the image - higher values can detect more hands but use more processing power"}),
|
||||
"min_confidence": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
"tooltip": "Minimum confidence threshold for hand detection - lower values detect more hands but may include false positives"}),
|
||||
"min_tracking_confidence": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
"tooltip": "(VIDEO mode) Minimum confidence threshold for tracking hands between frames - lower values maintain tracking longer but may be less accurate"}),
|
||||
"min_presence_confidence": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
"tooltip": "(VIDEO mode) Minimum confidence that a hand is present - lower values may detect hands that are less clear"}),
|
||||
})
|
||||
|
||||
inputs["required"].update(
|
||||
{
|
||||
"max_results": (
|
||||
"INT",
|
||||
{
|
||||
"default": 2,
|
||||
"min": 1,
|
||||
"max": 4,
|
||||
"step": 1,
|
||||
"tooltip": "Maximum number of hands to analyze in the image - higher values can detect more hands but use more processing power",
|
||||
},
|
||||
),
|
||||
"min_confidence": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Minimum confidence threshold for hand detection - lower values detect more hands but may include false positives",
|
||||
},
|
||||
),
|
||||
"min_tracking_confidence": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "(VIDEO mode) Minimum confidence threshold for tracking hands between frames - lower values maintain tracking longer but may be less accurate",
|
||||
},
|
||||
),
|
||||
"min_presence_confidence": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "(VIDEO mode) Minimum confidence that a hand is present - lower values may detect hands that are less clear",
|
||||
},
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
return inputs
|
||||
|
||||
def recognize_gestures(self, image: torch.Tensor, model_info: dict, max_results: int,
|
||||
min_confidence: float, min_tracking_confidence: float, min_presence_confidence: float,
|
||||
running_mode: str, delegate: str):
|
||||
def recognize_gestures(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
model_info: dict,
|
||||
max_results: int,
|
||||
min_confidence: float,
|
||||
min_tracking_confidence: float,
|
||||
min_presence_confidence: float,
|
||||
running_mode: str,
|
||||
delegate: str,
|
||||
):
|
||||
"""Performs gesture recognition on the input image."""
|
||||
|
||||
|
||||
# Validate model_info and get model path
|
||||
model_path = self.validate_model_info(model_info)
|
||||
|
||||
|
||||
# Initialize or update detector
|
||||
detector = self.initialize_or_update_detector(model_path)
|
||||
|
||||
|
||||
# Call the detector's recognize method with standardized parameters
|
||||
gesture_results_batch = detector.recognize(
|
||||
image,
|
||||
@@ -68,12 +117,13 @@ class MediaPipeGestureRecognizerNode(BaseMediaPipeDetectorNode):
|
||||
min_tracking_confidence=min_tracking_confidence,
|
||||
min_presence_confidence=min_presence_confidence,
|
||||
running_mode=running_mode,
|
||||
delegate=delegate
|
||||
delegate=delegate,
|
||||
)
|
||||
|
||||
|
||||
return (gesture_results_batch,)
|
||||
|
||||
# --- Mappings ---
|
||||
|
||||
# --- Mappings ---
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"MediaPipeGestureRecognizerModelLoader": MediaPipeGestureRecognizerModelLoaderNode,
|
||||
"MediaPipeGestureRecognizer": MediaPipeGestureRecognizerNode,
|
||||
@@ -82,4 +132,4 @@ NODE_CLASS_MAPPINGS = {
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"MediaPipeGestureRecognizerModelLoader": "Load Gesture Recognizer Model (MediaPipe)",
|
||||
"MediaPipeGestureRecognizer": "Gesture Recognizer (MediaPipe)",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,20 +1,26 @@
|
||||
import torch
|
||||
from .....src.mediapipe_vision.hand_landmark.detector import HandLandmarkDetector
|
||||
from ...common.model_loader import MediaPipeModelLoaderBaseNode
|
||||
from ...common.base_detector_node import BaseMediaPipeDetectorNode
|
||||
import logging
|
||||
|
||||
import torch
|
||||
|
||||
from .....src.mediapipe_vision.hand_landmark.detector import HandLandmarkDetector
|
||||
from ...common.base_detector_node import BaseMediaPipeDetectorNode
|
||||
from ...common.model_loader import MediaPipeModelLoaderBaseNode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_category = "Realtime Nodes/MediaPipe Vision/Hand/HandLandmark"
|
||||
|
||||
|
||||
class MediaPipeHandLandmarkerModelLoaderNode(MediaPipeModelLoaderBaseNode):
|
||||
"""ComfyUI node for loading MediaPipe Hand Landmarker models."""
|
||||
|
||||
TASK_TYPE = "hand_landmarker"
|
||||
RETURN_TYPES = ("HAND_LANDMARKER_MODEL_INFO",)
|
||||
RETURN_NAMES = ("model_info",)
|
||||
CATEGORY = _category
|
||||
DESCRIPTION = "Loads a MediaPipe Hand Landmarker model for detecting hand positions and gestures in images. Required before using the Hand Landmarker node."
|
||||
|
||||
|
||||
|
||||
class MediaPipeHandLandmarkerNode(BaseMediaPipeDetectorNode):
|
||||
"""ComfyUI node for MediaPipe Hand Landmark Detection."""
|
||||
|
||||
@@ -32,32 +38,74 @@ class MediaPipeHandLandmarkerNode(BaseMediaPipeDetectorNode):
|
||||
def INPUT_TYPES(cls):
|
||||
# Start with the base inputs from the parent class
|
||||
inputs = super().INPUT_TYPES()
|
||||
|
||||
|
||||
# Add hand-specific parameters
|
||||
inputs["required"].update({
|
||||
"max_results": ("INT", {"default": 2, "min": 1, "max": 4, "step": 1,
|
||||
"tooltip": "Maximum number of hands to detect and track in the image - higher values can detect more hands but use more processing power"}),
|
||||
"min_confidence": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
"tooltip": "Minimum confidence threshold for hand detection - lower values detect more hands but may include false positives"}),
|
||||
"min_presence_confidence": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
"tooltip": "(VIDEO mode) Minimum confidence that a hand is present - lower values may detect hands that are less clear"}),
|
||||
"min_tracking_confidence": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
"tooltip": "(VIDEO mode) Minimum confidence for tracking a hand between frames - lower values maintain tracking longer but may drift"}),
|
||||
})
|
||||
|
||||
inputs["required"].update(
|
||||
{
|
||||
"max_results": (
|
||||
"INT",
|
||||
{
|
||||
"default": 2,
|
||||
"min": 1,
|
||||
"max": 4,
|
||||
"step": 1,
|
||||
"tooltip": "Maximum number of hands to detect and track in the image - higher values can detect more hands but use more processing power",
|
||||
},
|
||||
),
|
||||
"min_confidence": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Minimum confidence threshold for hand detection - lower values detect more hands but may include false positives",
|
||||
},
|
||||
),
|
||||
"min_presence_confidence": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "(VIDEO mode) Minimum confidence that a hand is present - lower values may detect hands that are less clear",
|
||||
},
|
||||
),
|
||||
"min_tracking_confidence": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "(VIDEO mode) Minimum confidence for tracking a hand between frames - lower values maintain tracking longer but may drift",
|
||||
},
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
return inputs
|
||||
|
||||
def detect(self, image: torch.Tensor, model_info: dict, max_results: int,
|
||||
min_confidence: float, min_presence_confidence: float,
|
||||
min_tracking_confidence: float, running_mode: str, delegate: str):
|
||||
def detect(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
model_info: dict,
|
||||
max_results: int,
|
||||
min_confidence: float,
|
||||
min_presence_confidence: float,
|
||||
min_tracking_confidence: float,
|
||||
running_mode: str,
|
||||
delegate: str,
|
||||
):
|
||||
"""Performs hand landmark detection with the configured parameters."""
|
||||
|
||||
|
||||
# Validate model_info and get model path
|
||||
model_path = self.validate_model_info(model_info)
|
||||
|
||||
|
||||
# Initialize or update detector
|
||||
detector = self.initialize_or_update_detector(model_path)
|
||||
|
||||
|
||||
# Perform detection with all parameters
|
||||
hand_landmarks_batch = detector.detect(
|
||||
image,
|
||||
@@ -66,18 +114,19 @@ class MediaPipeHandLandmarkerNode(BaseMediaPipeDetectorNode):
|
||||
min_presence_confidence=min_presence_confidence,
|
||||
min_tracking_confidence=min_tracking_confidence,
|
||||
running_mode=running_mode,
|
||||
delegate=delegate
|
||||
delegate=delegate,
|
||||
)
|
||||
|
||||
|
||||
# Extract handedness information
|
||||
handedness_batch = []
|
||||
if hand_landmarks_batch:
|
||||
for image_hands in hand_landmarks_batch:
|
||||
handedness_per_image = [hand.handedness for hand in image_hands if hand.handedness is not None]
|
||||
handedness_batch.append(handedness_per_image)
|
||||
|
||||
|
||||
return (hand_landmarks_batch, handedness_batch)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"MediaPipeHandLandmarkerModelLoader": MediaPipeHandLandmarkerModelLoaderNode,
|
||||
"MediaPipeHandLandmarker": MediaPipeHandLandmarkerNode,
|
||||
@@ -86,4 +135,4 @@ NODE_CLASS_MAPPINGS = {
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"MediaPipeHandLandmarkerModelLoader": "Load Hand Landmarker Model (MediaPipe)",
|
||||
"MediaPipeHandLandmarker": "Hand Landmarker (MediaPipe)",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
# node_wrappers/hand/masking.py
|
||||
import torch
|
||||
import numpy as np
|
||||
import logging
|
||||
from typing import List
|
||||
from typing import List
|
||||
|
||||
import torch
|
||||
|
||||
from .....src.mediapipe_vision.types import HandLandmarksResult
|
||||
|
||||
# Updated import path
|
||||
from .....src.utils.image import create_mask_from_points
|
||||
|
||||
@@ -13,20 +14,22 @@ _category = "Realtime Nodes/MediaPipe Vision/Hand/HandLandmark/Masking"
|
||||
# --- Hand Part Definitions (Based on standard MediaPipe Hand Indices) ---
|
||||
# These indices need verification
|
||||
HAND_PART_INDICES = {
|
||||
"PALM": [0, 1, 5, 9, 13, 17], # Example
|
||||
"THUMB": [1, 2, 3, 4], # Example
|
||||
"INDEX_FINGER": [5, 6, 7, 8], # Example
|
||||
"MIDDLE_FINGER": [9, 10, 11, 12], # Example
|
||||
"RING_FINGER": [13, 14, 15, 16], # Example
|
||||
"PINKY_FINGER": [17, 18, 19, 20], # Example
|
||||
"WRIST": [0], # Single point, convex hull won't work
|
||||
"ALL_FINGERS": list(range(1, 21)), # Example excluding wrist
|
||||
"FULL_HAND": list(range(21)), # All points
|
||||
"PALM": [0, 1, 5, 9, 13, 17], # Example
|
||||
"THUMB": [1, 2, 3, 4], # Example
|
||||
"INDEX_FINGER": [5, 6, 7, 8], # Example
|
||||
"MIDDLE_FINGER": [9, 10, 11, 12], # Example
|
||||
"RING_FINGER": [13, 14, 15, 16], # Example
|
||||
"PINKY_FINGER": [17, 18, 19, 20], # Example
|
||||
"WRIST": [0], # Single point, convex hull won't work
|
||||
"ALL_FINGERS": list(range(1, 21)), # Example excluding wrist
|
||||
"FULL_HAND": list(range(21)), # All points
|
||||
}
|
||||
|
||||
# --- Node Definition ---
|
||||
|
||||
# --- Node Definition ---
|
||||
class MaskFromHandLandmarksNode:
|
||||
"""Creates a mask for a specific hand part from Hand Landmarks."""
|
||||
|
||||
CATEGORY = _category
|
||||
DESCRIPTION = "Creates precise masks for specific hand parts like individual fingers, palm, or the entire hand. Perfect for targeted editing or creating effects on hands and fingers."
|
||||
|
||||
@@ -35,10 +38,23 @@ class MaskFromHandLandmarksNode:
|
||||
part_names = list(HAND_PART_INDICES.keys())
|
||||
return {
|
||||
"required": {
|
||||
"hand_landmarks": ("HAND_LANDMARKS", {"tooltip": "Hand landmark data from the Hand Landmarker node containing the precise locations of hand and finger joints"}),
|
||||
"image_for_dimensions": ("IMAGE", {"tooltip": "Reference image to determine the size of the output mask - the mask will match these dimensions"}),
|
||||
"part_name": (part_names, {"default": "FULL_HAND",
|
||||
"tooltip": "Choose which part of the hand to create a mask for - FULL_HAND (entire hand), THUMB, INDEX_FINGER, PALM, etc."}),
|
||||
"hand_landmarks": (
|
||||
"HAND_LANDMARKS",
|
||||
{
|
||||
"tooltip": "Hand landmark data from the Hand Landmarker node containing the precise locations of hand and finger joints"
|
||||
},
|
||||
),
|
||||
"image_for_dimensions": (
|
||||
"IMAGE",
|
||||
{"tooltip": "Reference image to determine the size of the output mask - the mask will match these dimensions"},
|
||||
),
|
||||
"part_name": (
|
||||
part_names,
|
||||
{
|
||||
"default": "FULL_HAND",
|
||||
"tooltip": "Choose which part of the hand to create a mask for - FULL_HAND (entire hand), THUMB, INDEX_FINGER, PALM, etc.",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,7 +64,7 @@ class MaskFromHandLandmarksNode:
|
||||
def generate_mask(self, hand_landmarks: List[List[HandLandmarksResult]], image_for_dimensions: torch.Tensor, part_name: str):
|
||||
if part_name not in HAND_PART_INDICES:
|
||||
raise ValueError(f"Unknown hand part name: '{part_name}'. Available: {list(HAND_PART_INDICES.keys())}")
|
||||
|
||||
|
||||
# Get dimensions from image
|
||||
if image_for_dimensions.dim() != 4:
|
||||
raise ValueError("image_for_dimensions must be BHWC format.")
|
||||
@@ -59,16 +75,18 @@ class MaskFromHandLandmarksNode:
|
||||
|
||||
# Validate batch sizes match
|
||||
if len(hand_landmarks) != input_batch_size:
|
||||
logger.warning(f"Batch size mismatch between hand_landmarks ({len(hand_landmarks)}) and image_for_dimensions ({input_batch_size}). Using landmark batch size.")
|
||||
batch_size = len(hand_landmarks)
|
||||
logger.warning(
|
||||
f"Batch size mismatch between hand_landmarks ({len(hand_landmarks)}) and image_for_dimensions ({input_batch_size}). Using landmark batch size."
|
||||
)
|
||||
batch_size = len(hand_landmarks)
|
||||
else:
|
||||
batch_size = input_batch_size
|
||||
batch_size = input_batch_size
|
||||
|
||||
required_indices = HAND_PART_INDICES[part_name]
|
||||
output_masks = []
|
||||
|
||||
for i in range(batch_size):
|
||||
landmarks_for_image = hand_landmarks[i] # List[HandLandmarksResult]
|
||||
landmarks_for_image = hand_landmarks[i] # List[HandLandmarksResult]
|
||||
combined_points_for_image = []
|
||||
|
||||
# Combine points from all detected hands for the selected part
|
||||
@@ -83,25 +101,25 @@ class MaskFromHandLandmarksNode:
|
||||
part_points.append((px, py))
|
||||
# No warning for missing index
|
||||
combined_points_for_image.extend(part_points)
|
||||
|
||||
|
||||
# Create mask from all collected points for this image
|
||||
mask_tensor = create_mask_from_points(height, width, combined_points_for_image, device=device)
|
||||
output_masks.append(mask_tensor)
|
||||
|
||||
if not output_masks:
|
||||
logger.warning("No landmarks found to generate any masks. Returning zero mask batch.")
|
||||
output_batch = torch.zeros((input_batch_size, 1, height, width), dtype=torch.float32, device=device)
|
||||
logger.warning("No landmarks found to generate any masks. Returning zero mask batch.")
|
||||
output_batch = torch.zeros((input_batch_size, 1, height, width), dtype=torch.float32, device=device)
|
||||
else:
|
||||
output_batch = torch.stack(output_masks, dim=0)
|
||||
|
||||
output_batch = torch.stack(output_masks, dim=0)
|
||||
|
||||
return (output_batch,)
|
||||
|
||||
# --- Mappings ---
|
||||
|
||||
# --- Mappings ---
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"MaskFromHandLandmarks": MaskFromHandLandmarksNode,
|
||||
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"MaskFromHandLandmarks": "Mask From Hand Landmarks (MediaPipe)",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,19 +1,20 @@
|
||||
# visualize_hand.py
|
||||
import logging
|
||||
from typing import Any, List, Optional
|
||||
|
||||
import mediapipe as mp
|
||||
from typing import Optional, List, Any
|
||||
|
||||
from .....src.mediapipe_vision.types import HandLandmarksResult # Import specific result type if needed for extraction
|
||||
from ...common.base_visualization_nodes import BaseLandmarkVisualizationNode
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
_category = "Realtime Nodes/MediaPipe Vision/Hand/HandLandmark/Visualization"
|
||||
# Define standard hand connections
|
||||
HAND_CONNECTIONS = mp.solutions.hands.HAND_CONNECTIONS
|
||||
|
||||
|
||||
class VisualizeHandLandmarks(BaseLandmarkVisualizationNode):
|
||||
"""Draws Hand landmarks and connections onto images."""
|
||||
|
||||
|
||||
# Override base class properties
|
||||
LANDMARKS_TYPE = "HAND_LANDMARKS"
|
||||
CONNECTIONS = HAND_CONNECTIONS
|
||||
@@ -25,16 +26,12 @@ class VisualizeHandLandmarks(BaseLandmarkVisualizationNode):
|
||||
SUPPORTS_VISIBILITY = False
|
||||
CATEGORY = _category
|
||||
DESCRIPTION = "Visualizes hand landmarks by drawing points and connections on detected hands. Creates a wireframe overlay showing finger joints, palm, and hand structure."
|
||||
|
||||
|
||||
def get_landmark_list(self, detection_result: Any) -> Optional[List[Any]]:
|
||||
return detection_result.landmarks
|
||||
|
||||
|
||||
# Node registration
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"VisualizeHandLandmarks": VisualizeHandLandmarks
|
||||
}
|
||||
NODE_CLASS_MAPPINGS = {"VisualizeHandLandmarks": VisualizeHandLandmarks}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"VisualizeHandLandmarks": "Visualize Hand Landmarks (MediaPipe)"
|
||||
}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {"VisualizeHandLandmarks": "Visualize Hand Landmarks (MediaPipe)"}
|
||||
|
||||
@@ -1,20 +1,21 @@
|
||||
# node_wrappers/hand/position.py
|
||||
import logging
|
||||
from typing import List, Tuple, Any, Optional
|
||||
|
||||
# Import the base class
|
||||
from .....src.mediapipe_vision.common.position import LandmarkPositionBase
|
||||
|
||||
# Import the specific landmark type for clarity and potential future use
|
||||
from .....src.mediapipe_vision.types import HAND_LANDMARKS
|
||||
# Import the landmark definitions
|
||||
from .....src.mediapipe_vision.landmark_definitions import HAND_LANDMARK_TOOLTIP
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_category = "Realtime Nodes/MediaPipe Vision/Hand/HandLandmark/Position"
|
||||
|
||||
|
||||
class HandLandmarkPositionNode(LandmarkPositionBase):
|
||||
"""Extracts position lists (x, y, z, vis, pres) for a specific landmark index
|
||||
from HAND_LANDMARKS across the batch."""
|
||||
from HAND_LANDMARKS across the batch."""
|
||||
|
||||
CATEGORY = _category
|
||||
# Define the specific input type string this node expects
|
||||
LANDMARKS_TYPE = "HAND_LANDMARKS"
|
||||
@@ -27,11 +28,18 @@ class HandLandmarkPositionNode(LandmarkPositionBase):
|
||||
"required": {
|
||||
"landmarks": (cls.LANDMARKS_TYPE, {"forceInput": True}),
|
||||
"landmark_index": ("INT", {"default": 8, "min": 0, "max": 20, "tooltip": HAND_LANDMARK_TOOLTIP}),
|
||||
"result_index": ("INT", {"default": 0, "min": 0, "tooltip": "Index of the hand detection to use (0=first detected hand)"}),
|
||||
"use_world_coordinates": ("BOOLEAN", {"default": False, "tooltip": "Use world coordinates if available"}),
|
||||
"result_index": (
|
||||
"INT",
|
||||
{"default": 0, "min": 0, "tooltip": "Index of the hand detection to use (0=first detected hand)"},
|
||||
),
|
||||
"use_world_coordinates": (
|
||||
"BOOLEAN",
|
||||
{"default": False, "tooltip": "Use world coordinates if available"},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# --- Node Mappings ---
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"HandLandmarkPosition": HandLandmarkPositionNode,
|
||||
@@ -39,4 +47,4 @@ NODE_CLASS_MAPPINGS = {
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"HandLandmarkPosition": "Hand Landmark Position Extractor (Batch)",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,5 +6,4 @@ to this is released, this module will be updated to use the new api.
|
||||
"""
|
||||
|
||||
# Import modules to ensure they're registered
|
||||
from . import holistic_landmark
|
||||
from . import holistic_landmark_visualization
|
||||
from . import holistic_landmark, holistic_landmark_visualization
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
import logging
|
||||
|
||||
import torch
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
import numpy as np
|
||||
|
||||
from ....src.mediapipe_vision.holistic_landmark.detector import HolisticLandmarkDetector
|
||||
from ..common.base_detector_node import BaseMediaPipeDetectorNode
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_category = "Realtime Nodes/MediaPipe Vision/Holistic"
|
||||
|
||||
|
||||
class MediaPipeHolisticLandmarkerNode(BaseMediaPipeDetectorNode):
|
||||
"""ComfyUI node for MediaPipe Holistic Landmark Detection.
|
||||
|
||||
|
||||
This node uses the legacy MediaPipe Holistic API that combines face, pose and hand
|
||||
tracking in a single detector.
|
||||
|
||||
|
||||
Note: This uses the legacy MediaPipe Python Solution API while we wait for an official
|
||||
Holistic Task implementation, and thus it stands out among other implementations in this project.
|
||||
Holistic Task implementation, and thus it stands out among other implementations in this project.
|
||||
It does not require a model file to be loaded separately.
|
||||
"""
|
||||
|
||||
@@ -24,7 +24,10 @@ class MediaPipeHolisticLandmarkerNode(BaseMediaPipeDetectorNode):
|
||||
DETECTOR_CLASS = HolisticLandmarkDetector
|
||||
MODEL_INFO_TYPE = None # Not used - legacy solution doesn't need model file
|
||||
EXPECTED_TASK_TYPE = None # Not used - legacy solution doesn't need model info
|
||||
RETURN_TYPES = ("HOLISTIC_LANDMARKS", "MASK_LIST",)
|
||||
RETURN_TYPES = (
|
||||
"HOLISTIC_LANDMARKS",
|
||||
"MASK_LIST",
|
||||
)
|
||||
RETURN_NAMES = ("landmarks", "segmentation_masks")
|
||||
FUNCTION = "detect_holistic"
|
||||
CATEGORY = _category
|
||||
@@ -41,36 +44,78 @@ class MediaPipeHolisticLandmarkerNode(BaseMediaPipeDetectorNode):
|
||||
return {
|
||||
"required": {
|
||||
"image": ("IMAGE", {"tooltip": "Input image for detection"}),
|
||||
"min_confidence": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
"tooltip": "Minimum confidence threshold for detection - lower values detect more poses but may include false positives"}),
|
||||
"min_tracking_confidence": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
"tooltip": "Minimum confidence threshold for tracking - lower values maintain tracking longer but may be less accurate"}),
|
||||
"model_complexity": ("INT", {"default": 1, "min": 0, "max": 2, "step": 1,
|
||||
"tooltip": "Model complexity (0=Lite, 1=Full, 2=Heavy) - higher values are more accurate but slower"}),
|
||||
"enable_segmentation": ("BOOLEAN", {"default": False,
|
||||
"tooltip": "When enabled, generates masks that separate people from the background"}),
|
||||
"refine_face_landmarks": ("BOOLEAN", {"default": False,
|
||||
"tooltip": "When enabled, provides more detailed face landmarks around eyes and lips"}),
|
||||
"static_image_mode": ("BOOLEAN", {"default": False,
|
||||
"tooltip": "When enabled, treats each frame as independent - more accurate but slower and doesn't benefit from tracking"}),
|
||||
"min_confidence": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Minimum confidence threshold for detection - lower values detect more poses but may include false positives",
|
||||
},
|
||||
),
|
||||
"min_tracking_confidence": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Minimum confidence threshold for tracking - lower values maintain tracking longer but may be less accurate",
|
||||
},
|
||||
),
|
||||
"model_complexity": (
|
||||
"INT",
|
||||
{
|
||||
"default": 1,
|
||||
"min": 0,
|
||||
"max": 2,
|
||||
"step": 1,
|
||||
"tooltip": "Model complexity (0=Lite, 1=Full, 2=Heavy) - higher values are more accurate but slower",
|
||||
},
|
||||
),
|
||||
"enable_segmentation": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "When enabled, generates masks that separate people from the background",
|
||||
},
|
||||
),
|
||||
"refine_face_landmarks": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "When enabled, provides more detailed face landmarks around eyes and lips",
|
||||
},
|
||||
),
|
||||
"static_image_mode": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "When enabled, treats each frame as independent - more accurate but slower and doesn't benefit from tracking",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
def detect_holistic(self, image: torch.Tensor,
|
||||
min_confidence: float,
|
||||
min_tracking_confidence: float,
|
||||
model_complexity: int,
|
||||
enable_segmentation: bool,
|
||||
refine_face_landmarks: bool,
|
||||
static_image_mode: bool):
|
||||
def detect_holistic(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
min_confidence: float,
|
||||
min_tracking_confidence: float,
|
||||
model_complexity: int,
|
||||
enable_segmentation: bool,
|
||||
refine_face_landmarks: bool,
|
||||
static_image_mode: bool,
|
||||
):
|
||||
"""Performs holistic landmark detection with the configured parameters.
|
||||
|
||||
|
||||
Unlike other detectors in this codebase, this doesn't need a model_info parameter
|
||||
as it uses the built-in legacy Holistic API.
|
||||
"""
|
||||
# Initialize detector (without model path)
|
||||
detector = self.initialize_or_update_detector(None)
|
||||
|
||||
|
||||
# Perform detection with all parameters
|
||||
holistic_landmarks_batch, segmentation_masks_batch = detector.detect(
|
||||
image,
|
||||
@@ -79,17 +124,17 @@ class MediaPipeHolisticLandmarkerNode(BaseMediaPipeDetectorNode):
|
||||
model_complexity=model_complexity,
|
||||
enable_segmentation=enable_segmentation,
|
||||
refine_face_landmarks=refine_face_landmarks,
|
||||
static_image_mode=static_image_mode
|
||||
static_image_mode=static_image_mode,
|
||||
)
|
||||
|
||||
|
||||
return (holistic_landmarks_batch, segmentation_masks_batch or [])
|
||||
|
||||
|
||||
def initialize_or_update_detector(self, model_path=None):
|
||||
"""Override to handle the case where model_path is not needed."""
|
||||
if self._detector is None:
|
||||
self._detector = self.DETECTOR_CLASS()
|
||||
return self._detector
|
||||
|
||||
|
||||
def validate_model_info(self, model_info=None):
|
||||
"""Override to handle the case where model_info is not needed."""
|
||||
# No validation needed for legacy API
|
||||
@@ -103,4 +148,4 @@ NODE_CLASS_MAPPINGS = {
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"MediaPipeHolisticLandmarker": "**CAUTION** Holistic Landmarker (MediaPipe Legacy)",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import logging
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
import numpy as np
|
||||
import cv2
|
||||
from typing import List, Dict, Tuple, Any, Optional
|
||||
import torch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -13,19 +14,20 @@ POSE_CONNECTIONS = mp.solutions.pose.POSE_CONNECTIONS
|
||||
HAND_CONNECTIONS = mp.solutions.hands.HAND_CONNECTIONS
|
||||
FACE_CONNECTIONS = mp.solutions.face_mesh.FACEMESH_TESSELATION
|
||||
|
||||
|
||||
class MediaPipeHolisticVisualizerNode:
|
||||
"""ComfyUI node for visualizing holistic landmarks using direct drawing.
|
||||
|
||||
|
||||
Since holistic landmarks combine multiple landmark types (face, pose, hands),
|
||||
this doesn't use the BaseLandmarkVisualizationNode but implements a similar approach.
|
||||
"""
|
||||
|
||||
|
||||
CATEGORY = _category
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
RETURN_NAMES = ("image",)
|
||||
FUNCTION = "visualize_holistic"
|
||||
DESCRIPTION = "Visualizes holistic landmarks by drawing face mesh, pose skeleton, and hand landmarks on the input image."
|
||||
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
@@ -39,39 +41,61 @@ class MediaPipeHolisticVisualizerNode:
|
||||
"draw_face": ("BOOLEAN", {"default": True, "tooltip": "Toggle drawing face landmarks"}),
|
||||
"draw_pose": ("BOOLEAN", {"default": True, "tooltip": "Toggle drawing pose landmarks"}),
|
||||
"draw_hands": ("BOOLEAN", {"default": True, "tooltip": "Toggle drawing hand landmarks"}),
|
||||
"face_point_radius": ("INT", {"default": 1, "min": 1, "max": 5, "tooltip": "Size of face landmark points"}),
|
||||
"pose_point_radius": ("INT", {"default": 2, "min": 1, "max": 10, "tooltip": "Size of pose landmark points"}),
|
||||
"hand_point_radius": ("INT", {"default": 2, "min": 1, "max": 10, "tooltip": "Size of hand landmark points"}),
|
||||
"connection_thickness": ("INT", {"default": 1, "min": 1, "max": 5, "tooltip": "Thickness of connection lines"}),
|
||||
"min_visibility": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.05,
|
||||
"tooltip": "Minimum visibility score to draw a pose landmark"})
|
||||
"face_point_radius": (
|
||||
"INT",
|
||||
{"default": 1, "min": 1, "max": 5, "tooltip": "Size of face landmark points"},
|
||||
),
|
||||
"pose_point_radius": (
|
||||
"INT",
|
||||
{"default": 2, "min": 1, "max": 10, "tooltip": "Size of pose landmark points"},
|
||||
),
|
||||
"hand_point_radius": (
|
||||
"INT",
|
||||
{"default": 2, "min": 1, "max": 10, "tooltip": "Size of hand landmark points"},
|
||||
),
|
||||
"connection_thickness": (
|
||||
"INT",
|
||||
{"default": 1, "min": 1, "max": 5, "tooltip": "Thickness of connection lines"},
|
||||
),
|
||||
"min_visibility": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.05,
|
||||
"tooltip": "Minimum visibility score to draw a pose landmark",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _hex_to_rgb(self, hex_color):
|
||||
"""Convert hex color string to RGB tuple for OpenCV (BGR format)."""
|
||||
hex_color = hex_color.lstrip('#')
|
||||
hex_color = hex_color.lstrip("#")
|
||||
# Convert to BGR for OpenCV
|
||||
r, g, b = tuple(int(hex_color[i:i+2], 16) for i in (0, 2, 4))
|
||||
r, g, b = tuple(int(hex_color[i : i + 2], 16) for i in (0, 2, 4))
|
||||
return (b, g, r) # Return in BGR format for OpenCV
|
||||
|
||||
def visualize_holistic(self,
|
||||
image,
|
||||
landmarks,
|
||||
face_color,
|
||||
pose_color,
|
||||
left_hand_color,
|
||||
right_hand_color,
|
||||
draw_face,
|
||||
draw_pose,
|
||||
draw_hands,
|
||||
face_point_radius,
|
||||
pose_point_radius,
|
||||
hand_point_radius,
|
||||
connection_thickness,
|
||||
min_visibility):
|
||||
|
||||
def visualize_holistic(
|
||||
self,
|
||||
image,
|
||||
landmarks,
|
||||
face_color,
|
||||
pose_color,
|
||||
left_hand_color,
|
||||
right_hand_color,
|
||||
draw_face,
|
||||
draw_pose,
|
||||
draw_hands,
|
||||
face_point_radius,
|
||||
pose_point_radius,
|
||||
hand_point_radius,
|
||||
connection_thickness,
|
||||
min_visibility,
|
||||
):
|
||||
"""Visualize holistic landmarks using OpenCV for faster performance."""
|
||||
|
||||
|
||||
if not landmarks:
|
||||
logger.warning("No holistic landmarks provided.")
|
||||
return (image,)
|
||||
@@ -89,107 +113,111 @@ class MediaPipeHolisticVisualizerNode:
|
||||
pose_color_bgr = self._hex_to_rgb(pose_color)
|
||||
left_hand_color_bgr = self._hex_to_rgb(left_hand_color)
|
||||
right_hand_color_bgr = self._hex_to_rgb(right_hand_color)
|
||||
|
||||
|
||||
for i in range(batch_size):
|
||||
# Clone the tensor to avoid modifying the original
|
||||
img_tensor = image[i].clone()
|
||||
|
||||
img_tensor = image[i].clone()
|
||||
|
||||
# Convert tensor to numpy array scaled to 0-255 range for OpenCV
|
||||
np_image = (img_tensor.cpu().numpy() * 255).astype(np.uint8)
|
||||
|
||||
|
||||
# Convert to BGR for OpenCV processing
|
||||
cv_image = cv2.cvtColor(np_image, cv2.COLOR_RGB2BGR)
|
||||
|
||||
|
||||
if i < len(landmarks):
|
||||
holistic_result = landmarks[i] # One holistic result per image
|
||||
|
||||
|
||||
# Draw face landmarks if requested
|
||||
if draw_face and holistic_result.face_landmarks:
|
||||
self._draw_landmarks_cv2(
|
||||
cv_image,
|
||||
holistic_result.face_landmarks,
|
||||
holistic_result.face_landmarks,
|
||||
FACE_CONNECTIONS,
|
||||
face_color_bgr, face_point_radius, connection_thickness,
|
||||
min_visibility=None # Face doesn't use visibility
|
||||
face_color_bgr,
|
||||
face_point_radius,
|
||||
connection_thickness,
|
||||
min_visibility=None, # Face doesn't use visibility
|
||||
)
|
||||
|
||||
|
||||
# Draw pose landmarks if requested
|
||||
if draw_pose and holistic_result.pose_landmarks:
|
||||
self._draw_landmarks_cv2(
|
||||
cv_image,
|
||||
holistic_result.pose_landmarks,
|
||||
holistic_result.pose_landmarks,
|
||||
POSE_CONNECTIONS,
|
||||
pose_color_bgr, pose_point_radius, connection_thickness,
|
||||
min_visibility=min_visibility
|
||||
pose_color_bgr,
|
||||
pose_point_radius,
|
||||
connection_thickness,
|
||||
min_visibility=min_visibility,
|
||||
)
|
||||
|
||||
|
||||
# Draw left hand landmarks if requested
|
||||
if draw_hands and holistic_result.left_hand_landmarks:
|
||||
self._draw_landmarks_cv2(
|
||||
cv_image,
|
||||
holistic_result.left_hand_landmarks,
|
||||
holistic_result.left_hand_landmarks,
|
||||
HAND_CONNECTIONS,
|
||||
left_hand_color_bgr, hand_point_radius, connection_thickness,
|
||||
min_visibility=None # Hands don't use visibility
|
||||
left_hand_color_bgr,
|
||||
hand_point_radius,
|
||||
connection_thickness,
|
||||
min_visibility=None, # Hands don't use visibility
|
||||
)
|
||||
|
||||
|
||||
# Draw right hand landmarks if requested
|
||||
if draw_hands and holistic_result.right_hand_landmarks:
|
||||
self._draw_landmarks_cv2(
|
||||
cv_image,
|
||||
holistic_result.right_hand_landmarks,
|
||||
holistic_result.right_hand_landmarks,
|
||||
HAND_CONNECTIONS,
|
||||
right_hand_color_bgr, hand_point_radius, connection_thickness,
|
||||
min_visibility=None # Hands don't use visibility
|
||||
right_hand_color_bgr,
|
||||
hand_point_radius,
|
||||
connection_thickness,
|
||||
min_visibility=None, # Hands don't use visibility
|
||||
)
|
||||
|
||||
|
||||
# Convert back to RGB and tensor format
|
||||
cv_image_rgb = cv2.cvtColor(cv_image, cv2.COLOR_BGR2RGB)
|
||||
result_tensor = torch.from_numpy(cv_image_rgb.astype(np.float32) / 255.0)
|
||||
result_images.append(result_tensor)
|
||||
|
||||
|
||||
return (torch.stack(result_images),)
|
||||
|
||||
def _draw_landmarks_cv2(self,
|
||||
image: np.ndarray,
|
||||
landmark_list: List,
|
||||
connections: List,
|
||||
color: Tuple[int, int, int],
|
||||
point_radius: int,
|
||||
connection_thickness: int,
|
||||
min_visibility: Optional[float] = None):
|
||||
|
||||
def _draw_landmarks_cv2(
|
||||
self,
|
||||
image: np.ndarray,
|
||||
landmark_list: List,
|
||||
connections: List,
|
||||
color: Tuple[int, int, int],
|
||||
point_radius: int,
|
||||
connection_thickness: int,
|
||||
min_visibility: Optional[float] = None,
|
||||
):
|
||||
"""Helper method to draw landmarks with OpenCV for faster performance."""
|
||||
|
||||
|
||||
height, width = image.shape[:2]
|
||||
points = {}
|
||||
|
||||
|
||||
# Process landmarks and draw points
|
||||
for lm in landmark_list:
|
||||
# Check visibility if required
|
||||
is_visible = True
|
||||
if min_visibility is not None and hasattr(lm, 'visibility') and lm.visibility is not None:
|
||||
if min_visibility is not None and hasattr(lm, "visibility") and lm.visibility is not None:
|
||||
is_visible = lm.visibility >= min_visibility
|
||||
|
||||
|
||||
if is_visible:
|
||||
# Convert normalized coordinates to pixel coordinates
|
||||
cx, cy = int(lm.x * width), int(lm.y * height)
|
||||
points[lm.index] = (cx, cy)
|
||||
|
||||
|
||||
# Draw point as a filled circle
|
||||
cv2.circle(image, (cx, cy), point_radius, color, -1)
|
||||
|
||||
|
||||
# Draw connections
|
||||
if connections and points:
|
||||
for connection in connections:
|
||||
start_idx, end_idx = connection
|
||||
if start_idx in points and end_idx in points:
|
||||
cv2.line(
|
||||
image,
|
||||
points[start_idx],
|
||||
points[end_idx],
|
||||
color,
|
||||
connection_thickness
|
||||
)
|
||||
cv2.line(image, points[start_idx], points[end_idx], color, connection_thickness)
|
||||
|
||||
|
||||
# Define mappings for ComfyUI
|
||||
@@ -199,4 +227,4 @@ NODE_CLASS_MAPPINGS = {
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"MediaPipeHolisticVisualizer": "Holistic Visualizer (MediaPipe)",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
|
||||
@@ -1,23 +1,28 @@
|
||||
import torch
|
||||
import logging
|
||||
|
||||
import torch
|
||||
|
||||
from ....src.mediapipe_vision.image_embedding.detector import ImageEmbedder
|
||||
from ..common.base_detector_node import BaseMediaPipeDetectorNode
|
||||
|
||||
# Import Base Loader and Detector
|
||||
from ..common.model_loader import MediaPipeModelLoaderBaseNode
|
||||
from ..common.base_detector_node import BaseMediaPipeDetectorNode
|
||||
from ....src.mediapipe_vision.image_embedding.detector import ImageEmbedder
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_category = "Realtime Nodes/MediaPipe Vision/ImageEmbedding"
|
||||
|
||||
# --- Model Loader ---
|
||||
|
||||
# --- Model Loader ---
|
||||
class MediaPipeImageEmbedderModelLoaderNode(MediaPipeModelLoaderBaseNode):
|
||||
"""ComfyUI node for loading MediaPipe Image Embedder models."""
|
||||
TASK_TYPE = "image_embedder" # Need to add this task to MEDIAPIPE_MODELS
|
||||
|
||||
TASK_TYPE = "image_embedder" # Need to add this task to MEDIAPIPE_MODELS
|
||||
RETURN_TYPES = ("IMAGE_EMBEDDER_MODEL_INFO",)
|
||||
RETURN_NAMES = ("model_info",)
|
||||
CATEGORY = _category
|
||||
CATEGORY = _category
|
||||
|
||||
# --- Embedder Node ---
|
||||
|
||||
# --- Embedder Node ---
|
||||
class MediaPipeImageEmbedderNode(BaseMediaPipeDetectorNode):
|
||||
"""ComfyUI node for MediaPipe Image Embedding."""
|
||||
|
||||
@@ -34,39 +39,44 @@ class MediaPipeImageEmbedderNode(BaseMediaPipeDetectorNode):
|
||||
def INPUT_TYPES(cls):
|
||||
# Start with the base inputs from the parent class
|
||||
inputs = super().INPUT_TYPES()
|
||||
|
||||
|
||||
# Add embedder-specific parameters
|
||||
inputs["required"].update({
|
||||
"l2_normalize": ("BOOLEAN", {"default": True,
|
||||
"tooltip": "Apply L2 normalization to the embedding"}),
|
||||
"quantize": ("BOOLEAN", {"default": False,
|
||||
"tooltip": "Output quantized (uint8) embedding instead of float"}),
|
||||
})
|
||||
|
||||
inputs["required"].update(
|
||||
{
|
||||
"l2_normalize": ("BOOLEAN", {"default": True, "tooltip": "Apply L2 normalization to the embedding"}),
|
||||
"quantize": (
|
||||
"BOOLEAN",
|
||||
{"default": False, "tooltip": "Output quantized (uint8) embedding instead of float"},
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
return inputs
|
||||
|
||||
def detect(self, image: torch.Tensor, model_info: dict, l2_normalize: bool,
|
||||
quantize: bool, running_mode: str, delegate: str):
|
||||
def detect(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
model_info: dict,
|
||||
l2_normalize: bool,
|
||||
quantize: bool,
|
||||
running_mode: str,
|
||||
delegate: str,
|
||||
):
|
||||
"""Generates image embeddings with the configured parameters."""
|
||||
|
||||
|
||||
# Validate model_info and get model path
|
||||
model_path = self.validate_model_info(model_info)
|
||||
|
||||
|
||||
# Initialize or update detector
|
||||
detector = self.initialize_or_update_detector(model_path)
|
||||
|
||||
|
||||
# Generate embeddings with all parameters
|
||||
embeddings_batch = detector.embed(
|
||||
image,
|
||||
l2_normalize=l2_normalize,
|
||||
quantize=quantize,
|
||||
running_mode=running_mode,
|
||||
delegate=delegate
|
||||
)
|
||||
|
||||
embeddings_batch = detector.embed(image, l2_normalize=l2_normalize, quantize=quantize, running_mode=running_mode, delegate=delegate)
|
||||
|
||||
return (embeddings_batch,)
|
||||
|
||||
# --- Mappings ---
|
||||
|
||||
# --- Mappings ---
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"MediaPipeImageEmbedderModelLoader": MediaPipeImageEmbedderModelLoaderNode,
|
||||
"MediaPipeImageEmbedder": MediaPipeImageEmbedderNode,
|
||||
@@ -75,4 +85,4 @@ NODE_CLASS_MAPPINGS = {
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"MediaPipeImageEmbedderModelLoader": "Load Image Embedder Model (MediaPipe)",
|
||||
"MediaPipeImageEmbedder": "Image Embedder (MediaPipe)",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,26 +1,39 @@
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import logging
|
||||
from typing import List, Optional
|
||||
|
||||
from ....src.mediapipe_vision.types import ImageEmbedderResult # Assuming this is the relevant type
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from ....src.mediapipe_vision.types import (
|
||||
ImageEmbedderResult, # Assuming this is the relevant type
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ReshapeMediaPipeEmbeddingNode:
|
||||
"""Reshapes MediaPipe embedding for experimental use as ComfyUI Conditioning.
|
||||
WARNING: Semantically different from standard CLIP conditioning.
|
||||
Pads/truncates the embedding vector and replicates it across 77 tokens.
|
||||
WARNING: Semantically different from standard CLIP conditioning.
|
||||
Pads/truncates the embedding vector and replicates it across 77 tokens.
|
||||
"""
|
||||
|
||||
CATEGORY = "Realtime Nodes/MediaPipe Vision/ImageEmbedding/_Experimental"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"image_embeddings": ("IMAGE_EMBEDDINGS",), # List[List[ImageEmbedderResult]]
|
||||
"target_dimension": ("INT", {"default": 768, "min": 64, "max": 4096, "step": 8,
|
||||
"tooltip": "Target dimension D for the reshaped (1, 77, D) tensor (e.g., 768 for SD1.5, 1024/1280 for SDXL)"}),
|
||||
"image_embeddings": ("IMAGE_EMBEDDINGS",), # List[List[ImageEmbedderResult]]
|
||||
"target_dimension": (
|
||||
"INT",
|
||||
{
|
||||
"default": 768,
|
||||
"min": 64,
|
||||
"max": 4096,
|
||||
"step": 8,
|
||||
"tooltip": "Target dimension D for the reshaped (1, 77, D) tensor (e.g., 768 for SD1.5, 1024/1280 for SDXL)",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,11 +43,11 @@ class ReshapeMediaPipeEmbeddingNode:
|
||||
def reshape_embedding(self, image_embeddings: List[List[ImageEmbedderResult]], target_dimension: int):
|
||||
if not image_embeddings or not image_embeddings[0]:
|
||||
raise ValueError("Input image_embeddings are empty.")
|
||||
|
||||
|
||||
# --- Process First Embedding in Batch/Head ---
|
||||
# Assuming we use the first embedding result from the first image in the batch
|
||||
first_embedding_result = image_embeddings[0][0]
|
||||
|
||||
|
||||
if first_embedding_result.float_embedding:
|
||||
embedding_vector = first_embedding_result.float_embedding
|
||||
elif first_embedding_result.quantized_embedding:
|
||||
@@ -43,12 +56,12 @@ class ReshapeMediaPipeEmbeddingNode:
|
||||
raise NotImplementedError("Reshaping quantized embeddings is not directly supported. Use float embeddings.")
|
||||
else:
|
||||
raise ValueError("No valid embedding found in the input.")
|
||||
|
||||
|
||||
# Convert to tensor
|
||||
embed_tensor = torch.tensor(embedding_vector, dtype=torch.float32)
|
||||
orig_dim = embed_tensor.shape[0]
|
||||
|
||||
# --- Pad or Truncate ---
|
||||
# --- Pad or Truncate ---
|
||||
if orig_dim < target_dimension:
|
||||
# Pad with zeros
|
||||
padding = torch.zeros(target_dimension - orig_dim, dtype=torch.float32)
|
||||
@@ -58,35 +71,37 @@ class ReshapeMediaPipeEmbeddingNode:
|
||||
final_vector = embed_tensor[:target_dimension]
|
||||
else:
|
||||
final_vector = embed_tensor
|
||||
|
||||
# --- Reshape and Replicate for Conditioning ---
|
||||
|
||||
# --- Reshape and Replicate for Conditioning ---
|
||||
# Reshape to (1, 1, D)
|
||||
cond_vec = final_vector.unsqueeze(0).unsqueeze(0)
|
||||
# Replicate to (1, 77, D)
|
||||
cond_batch = cond_vec.repeat(1, 77, 1)
|
||||
|
||||
# --- Create Pooled Output ---
|
||||
# --- Create Pooled Output ---
|
||||
# Use the padded/truncated vector as the pooled output (1, D)
|
||||
pooled_output = final_vector.unsqueeze(0)
|
||||
|
||||
# --- Package as CONDITIONING ---
|
||||
# --- Package as CONDITIONING ---
|
||||
# Standard ComfyUI conditioning format: List[List[Union[torch.Tensor, Dict[str, torch.Tensor]]]]
|
||||
conditioning = [[cond_batch, {"pooled_output": pooled_output}]]
|
||||
|
||||
logger.warning("Reshaped MediaPipe embedding to CONDITIONING format. Use experimentally.")
|
||||
return (conditioning,)
|
||||
|
||||
# --- Comparison Node ---
|
||||
|
||||
# --- Comparison Node ---
|
||||
class CompareMediaPipeEmbeddingsNode:
|
||||
"""Calculates the cosine similarity between two MediaPipe image embeddings."""
|
||||
|
||||
CATEGORY = "Realtime Nodes/MediaPipe Vision/ImageEmbedding"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"image_embeddings_1": ("IMAGE_EMBEDDINGS",),
|
||||
"image_embeddings_2": ("IMAGE_EMBEDDINGS",),
|
||||
"image_embeddings_1": ("IMAGE_EMBEDDINGS",),
|
||||
"image_embeddings_2": ("IMAGE_EMBEDDINGS",),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,9 +109,7 @@ class CompareMediaPipeEmbeddingsNode:
|
||||
RETURN_NAMES = ("cosine_similarity",)
|
||||
FUNCTION = "compare_embeddings"
|
||||
|
||||
def compare_embeddings(self, image_embeddings_1: List[List[ImageEmbedderResult]],
|
||||
image_embeddings_2: List[List[ImageEmbedderResult]]):
|
||||
|
||||
def compare_embeddings(self, image_embeddings_1: List[List[ImageEmbedderResult]], image_embeddings_2: List[List[ImageEmbedderResult]]):
|
||||
def _extract_vector(embeddings: List[List[ImageEmbedderResult]]) -> Optional[torch.Tensor]:
|
||||
"""Helper to extract the first float embedding vector."""
|
||||
if not embeddings or not embeddings[0]:
|
||||
@@ -113,8 +126,8 @@ class CompareMediaPipeEmbeddingsNode:
|
||||
|
||||
if vec1 is None or vec2 is None:
|
||||
logger.error("Could not extract valid float embedding vectors for comparison.")
|
||||
return (0.0,) # Return 0 similarity on error
|
||||
|
||||
return (0.0,) # Return 0 similarity on error
|
||||
|
||||
# Ensure vectors have the same dimension (pad/truncate shorter one)
|
||||
dim1, dim2 = vec1.shape[0], vec2.shape[0]
|
||||
if dim1 != dim2:
|
||||
@@ -124,21 +137,21 @@ class CompareMediaPipeEmbeddingsNode:
|
||||
vec1 = F.pad(vec1, (0, max_dim - dim1))
|
||||
elif dim2 < max_dim:
|
||||
vec2 = F.pad(vec2, (0, max_dim - dim2))
|
||||
|
||||
|
||||
# Calculate Cosine Similarity
|
||||
# Normalize vectors before dot product for cosine similarity
|
||||
similarity = F.cosine_similarity(vec1.unsqueeze(0), vec2.unsqueeze(0), dim=1)
|
||||
|
||||
|
||||
return (similarity.item(),)
|
||||
|
||||
|
||||
# --- Mappings ---
|
||||
# --- Mappings ---
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"ReshapeMediaPipeEmbedding": ReshapeMediaPipeEmbeddingNode,
|
||||
"CompareMediaPipeEmbeddings": CompareMediaPipeEmbeddingsNode, # Add new node
|
||||
"CompareMediaPipeEmbeddings": CompareMediaPipeEmbeddingsNode, # Add new node
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"ReshapeMediaPipeEmbedding": "Reshape MediaPipe Embedding (Experimental)",
|
||||
"CompareMediaPipeEmbeddings": "Compare MediaPipe Embeddings", # Add new node name
|
||||
}
|
||||
"CompareMediaPipeEmbeddings": "Compare MediaPipe Embeddings", # Add new node name
|
||||
}
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
|
||||
@@ -1,16 +1,18 @@
|
||||
"""Node wrapper for image segmentation"""
|
||||
|
||||
import torch
|
||||
import numpy as np
|
||||
import cv2 # For visualization
|
||||
import logging
|
||||
|
||||
import cv2 # For visualization
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
# Imports from this project
|
||||
from ....src.mediapipe_vision.image_segmentation.segmenter import ImageSegmenter
|
||||
# Import Base Classes
|
||||
from ..common.model_loader import MediaPipeModelLoaderBaseNode
|
||||
from ..common.base_detector_node import BaseMediaPipeDetectorNode
|
||||
|
||||
# Import Base Classes
|
||||
from ..common.model_loader import MediaPipeModelLoaderBaseNode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_category = "Realtime Nodes/MediaPipe Vision/ImageSegmentation"
|
||||
@@ -26,34 +28,41 @@ MULTICLASS_NAMES = {
|
||||
|
||||
# Uses MULTICLASS_NAMES indices
|
||||
CLASS_COLORS_BGR = {
|
||||
MULTICLASS_NAMES["Background"]: (0, 0, 0), # Black
|
||||
MULTICLASS_NAMES["Hair"]: (255, 0, 0), # Blue
|
||||
MULTICLASS_NAMES["Body-skin"]: (0, 255, 255), # Yellow
|
||||
MULTICLASS_NAMES["Face-skin"]: (255, 200, 100), # Light Blue
|
||||
MULTICLASS_NAMES["Clothes"]: (0, 255, 0), # Green
|
||||
MULTICLASS_NAMES["Accessories/Other"]: (255, 0, 255), # Magenta
|
||||
MULTICLASS_NAMES["Background"]: (0, 0, 0), # Black
|
||||
MULTICLASS_NAMES["Hair"]: (255, 0, 0), # Blue
|
||||
MULTICLASS_NAMES["Body-skin"]: (0, 255, 255), # Yellow
|
||||
MULTICLASS_NAMES["Face-skin"]: (255, 200, 100), # Light Blue
|
||||
MULTICLASS_NAMES["Clothes"]: (0, 255, 0), # Green
|
||||
MULTICLASS_NAMES["Accessories/Other"]: (255, 0, 255), # Magenta
|
||||
}
|
||||
# --- End Constants ---
|
||||
# --- End Constants ---
|
||||
|
||||
|
||||
# Inherit from the base loader
|
||||
class MediaPipeImageSegmenterModelLoaderNode(MediaPipeModelLoaderBaseNode):
|
||||
"""ComfyUI node for loading MediaPipe Image Segmenter models."""
|
||||
|
||||
TASK_TYPE = "image_segmenter"
|
||||
RETURN_TYPES = ("IMAGE_SEGMENTER_MODEL_INFO",)
|
||||
RETURN_NAMES = ("model_info",)
|
||||
# INPUT_TYPES and FUNCTION inherited
|
||||
CATEGORY = _category
|
||||
CATEGORY = _category
|
||||
|
||||
|
||||
class MediaPipeImageSegmenterNode(BaseMediaPipeDetectorNode):
|
||||
"""ComfyUI node for MediaPipe Image Segmentation, adapted from Stream-Pack.
|
||||
Manages the ImageSegmenter instance for potential reuse based on config.
|
||||
Manages the ImageSegmenter instance for potential reuse based on config.
|
||||
"""
|
||||
|
||||
|
||||
# Define class variables required by the base class
|
||||
DETECTOR_CLASS = ImageSegmenter
|
||||
MODEL_INFO_TYPE = "IMAGE_SEGMENTER_MODEL_INFO"
|
||||
EXPECTED_TASK_TYPE = "image_segmenter"
|
||||
RETURN_TYPES = ("MASK", "IMAGE", "MP_INT_MASK",)
|
||||
RETURN_TYPES = (
|
||||
"MASK",
|
||||
"IMAGE",
|
||||
"MP_INT_MASK",
|
||||
)
|
||||
RETURN_NAMES = ("mask", "visualization", "multiclass_segments")
|
||||
FUNCTION = "detect"
|
||||
CATEGORY = _category
|
||||
@@ -62,37 +71,54 @@ class MediaPipeImageSegmenterNode(BaseMediaPipeDetectorNode):
|
||||
def INPUT_TYPES(cls):
|
||||
# Start with the base inputs from the parent class
|
||||
inputs = super().INPUT_TYPES()
|
||||
|
||||
|
||||
# Add segmentation-specific parameters
|
||||
inputs["required"].update({
|
||||
"output_confidence_masks": ("BOOLEAN", {"default": False,
|
||||
"tooltip": "Output confidence mask (0-1) instead of category mask. Disables multiclass/visualization."}),
|
||||
"threshold": ("FLOAT", {
|
||||
"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
"tooltip": "Confidence threshold for confidence mask output mode. Pixels below threshold become 0."
|
||||
}),
|
||||
"generate_visualization": ("BOOLEAN", {"default": False,
|
||||
"tooltip": "Generate a colored visualization image (only works in category mask mode)."
|
||||
}),
|
||||
})
|
||||
|
||||
inputs["required"].update(
|
||||
{
|
||||
"output_confidence_masks": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "Output confidence mask (0-1) instead of category mask. Disables multiclass/visualization.",
|
||||
},
|
||||
),
|
||||
"threshold": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Confidence threshold for confidence mask output mode. Pixels below threshold become 0.",
|
||||
},
|
||||
),
|
||||
"generate_visualization": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "Generate a colored visualization image (only works in category mask mode).",
|
||||
},
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
# Rename the delegate parameter for consistency with original implementation
|
||||
inputs["required"]["delegate_mode"] = inputs["required"].pop("delegate")
|
||||
|
||||
|
||||
return inputs
|
||||
|
||||
# --- Helper methods from Stream-Pack ---
|
||||
# --- Helper methods from Stream-Pack ---
|
||||
def is_multiclass_model(self, model_info: dict):
|
||||
# Improved check based on variant or path
|
||||
model_variant = model_info.get('model_variant', '').lower()
|
||||
model_path = model_info.get('model_path', '').lower()
|
||||
model_variant = model_info.get("model_variant", "").lower()
|
||||
model_path = model_info.get("model_path", "").lower()
|
||||
# Add known multiclass identifiers
|
||||
return 'multiclass' in model_variant or 'multiclass' in model_path
|
||||
return "multiclass" in model_variant or "multiclass" in model_path
|
||||
|
||||
def create_visualization(self, category_mask_np, is_multiclass=False):
|
||||
"""Generates a colored visualization from a category mask."""
|
||||
if category_mask_np is None:
|
||||
return None
|
||||
return None
|
||||
if not is_multiclass:
|
||||
# Simple binary visualization (white on black)
|
||||
vis_image = np.where(category_mask_np > 0, 255, 0).astype(np.uint8)
|
||||
@@ -102,38 +128,47 @@ class MediaPipeImageSegmenterNode(BaseMediaPipeDetectorNode):
|
||||
height, width = category_mask_np.shape
|
||||
vis_image = np.zeros((height, width, 3), dtype=np.uint8)
|
||||
for class_name, class_id in MULTICLASS_NAMES.items():
|
||||
color_bgr = CLASS_COLORS_BGR.get(class_id, (128, 128, 128)) # Default grey
|
||||
vis_image[category_mask_np == class_id] = color_bgr
|
||||
color_bgr = CLASS_COLORS_BGR.get(class_id, (128, 128, 128)) # Default grey
|
||||
vis_image[category_mask_np == class_id] = color_bgr
|
||||
return vis_image
|
||||
# --- End Helper methods ---
|
||||
|
||||
def detect(self, image: torch.Tensor, model_info: dict, output_confidence_masks: bool,
|
||||
threshold: float, generate_visualization: bool, running_mode: str, delegate_mode: str):
|
||||
# --- End Helper methods ---
|
||||
|
||||
def detect(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
model_info: dict,
|
||||
output_confidence_masks: bool,
|
||||
threshold: float,
|
||||
generate_visualization: bool,
|
||||
running_mode: str,
|
||||
delegate_mode: str,
|
||||
):
|
||||
"""Performs image segmentation with the configured parameters."""
|
||||
|
||||
|
||||
# Validate model_info and get model path
|
||||
model_path = self.validate_model_info(model_info)
|
||||
|
||||
|
||||
# Initialize or update detector
|
||||
detector = self.initialize_or_update_detector(model_path)
|
||||
|
||||
|
||||
# Determine which output type the detector needs
|
||||
request_confidence = output_confidence_masks
|
||||
# Need category mask if confidence isn't requested OR if visualization is needed
|
||||
request_category = not output_confidence_masks or generate_visualization
|
||||
|
||||
request_category = not output_confidence_masks or generate_visualization
|
||||
|
||||
if not request_confidence and not request_category:
|
||||
# Should not happen with the logic above, but as a safeguard
|
||||
raise ValueError("Internal logic error: No mask type determined for segmenter.")
|
||||
# Should not happen with the logic above, but as a safeguard
|
||||
raise ValueError("Internal logic error: No mask type determined for segmenter.")
|
||||
|
||||
# Call segmenter's segment method
|
||||
# Returns lists (one element per image in batch)
|
||||
batch_results_confidence, batch_results_category = detector.segment(
|
||||
image,
|
||||
output_confidence_masks=request_confidence,
|
||||
output_category_mask=request_category,
|
||||
output_category_mask=request_category,
|
||||
running_mode=running_mode,
|
||||
delegate_mode=delegate_mode
|
||||
delegate_mode=delegate_mode,
|
||||
)
|
||||
|
||||
# Initialize output lists
|
||||
@@ -142,17 +177,17 @@ class MediaPipeImageSegmenterNode(BaseMediaPipeDetectorNode):
|
||||
all_primary_masks = []
|
||||
all_vis_images = []
|
||||
all_category_masks = []
|
||||
default_vis_tensor = torch.zeros((h, w, 3), dtype=torch.float32, device=image.device) # HWC
|
||||
default_vis_tensor = torch.zeros((h, w, 3), dtype=torch.float32, device=image.device) # HWC
|
||||
|
||||
# --- Process results based on node inputs ---
|
||||
# --- Process results based on node inputs ---
|
||||
is_multiclass = self.is_multiclass_model(model_info)
|
||||
|
||||
|
||||
# Loop through batch results
|
||||
for i in range(batch_size):
|
||||
# Get results for the current image
|
||||
confidence_masks_hw_list = batch_results_confidence[i] if batch_results_confidence else None
|
||||
category_mask_hw = batch_results_category[i] if batch_results_category else None
|
||||
|
||||
|
||||
# Initialize outputs for this image
|
||||
current_primary_mask_hw = torch.zeros((h, w), dtype=torch.float32, device=image.device)
|
||||
current_vis_image_hwc = default_vis_tensor
|
||||
@@ -162,53 +197,54 @@ class MediaPipeImageSegmenterNode(BaseMediaPipeDetectorNode):
|
||||
# User wants confidence mask as primary output
|
||||
if confidence_masks_hw_list:
|
||||
# Take the first confidence mask (often foreground vs background)
|
||||
conf_mask_hw = confidence_masks_hw_list[0]
|
||||
conf_mask_hw = confidence_masks_hw_list[0]
|
||||
# Apply threshold
|
||||
conf_mask_hw_thresh = torch.where(conf_mask_hw >= threshold, conf_mask_hw, torch.zeros_like(conf_mask_hw))
|
||||
current_primary_mask_hw = conf_mask_hw_thresh # HW
|
||||
current_primary_mask_hw = conf_mask_hw_thresh # HW
|
||||
# Visualization and multiclass output are disabled/invalid in confidence mode
|
||||
generate_visualization_for_this = False
|
||||
else:
|
||||
generate_visualization_for_this = False
|
||||
else:
|
||||
# User wants category mask (default or for visualization)
|
||||
if category_mask_hw is not None:
|
||||
# Determine primary MASK output (HW Float)
|
||||
if is_multiclass:
|
||||
# Treat non-background as foreground for the primary mask
|
||||
primary_mask_hw = (category_mask_hw > 0).float()
|
||||
else:
|
||||
# Binary models: Category 1 is foreground
|
||||
primary_mask_hw = (category_mask_hw > 0).float()
|
||||
current_primary_mask_hw = primary_mask_hw # HW
|
||||
|
||||
# Prepare raw category mask output (HW Long Tensor)
|
||||
current_category_mask_hw = category_mask_hw # Already Long
|
||||
|
||||
# Generate visualization if requested (and category mask exists)
|
||||
generate_visualization_for_this = generate_visualization
|
||||
if generate_visualization_for_this:
|
||||
# Convert to NumPy for OpenCV, then back to tensor
|
||||
category_np = category_mask_hw.cpu().numpy().astype(np.uint8)
|
||||
vis_np = self.create_visualization(category_np, is_multiclass)
|
||||
if vis_np is not None:
|
||||
# Convert back to tensor (HWC)
|
||||
vis_tensor = torch.from_numpy(vis_np).to(dtype=torch.float32, device=image.device) / 255.0
|
||||
current_vis_image_hwc = vis_tensor # HWC
|
||||
|
||||
# Determine primary MASK output (HW Float)
|
||||
if is_multiclass:
|
||||
# Treat non-background as foreground for the primary mask
|
||||
primary_mask_hw = (category_mask_hw > 0).float()
|
||||
else:
|
||||
# Binary models: Category 1 is foreground
|
||||
primary_mask_hw = (category_mask_hw > 0).float()
|
||||
current_primary_mask_hw = primary_mask_hw # HW
|
||||
|
||||
# Prepare raw category mask output (HW Long Tensor)
|
||||
current_category_mask_hw = category_mask_hw # Already Long
|
||||
|
||||
# Generate visualization if requested (and category mask exists)
|
||||
generate_visualization_for_this = generate_visualization
|
||||
if generate_visualization_for_this:
|
||||
# Convert to NumPy for OpenCV, then back to tensor
|
||||
category_np = category_mask_hw.cpu().numpy().astype(np.uint8)
|
||||
vis_np = self.create_visualization(category_np, is_multiclass)
|
||||
if vis_np is not None:
|
||||
# Convert back to tensor (HWC)
|
||||
vis_tensor = torch.from_numpy(vis_np).to(dtype=torch.float32, device=image.device) / 255.0
|
||||
current_vis_image_hwc = vis_tensor # HWC
|
||||
|
||||
# Add results for this image to the batch results
|
||||
all_primary_masks.append(current_primary_mask_hw) # HW
|
||||
all_vis_images.append(current_vis_image_hwc) # HWC
|
||||
all_category_masks.append(current_category_mask_hw) # HW
|
||||
|
||||
all_primary_masks.append(current_primary_mask_hw) # HW
|
||||
all_vis_images.append(current_vis_image_hwc) # HWC
|
||||
all_category_masks.append(current_category_mask_hw) # HW
|
||||
|
||||
# Stack all results to return as tensors
|
||||
primary_mask_tensor = torch.stack(all_primary_masks, dim=0) # BHW
|
||||
vis_image_tensor = torch.stack(all_vis_images, dim=0) # BHWC
|
||||
category_mask_tensor = torch.stack(all_category_masks, dim=0) # BHW
|
||||
|
||||
primary_mask_tensor = torch.stack(all_primary_masks, dim=0) # BHW
|
||||
vis_image_tensor = torch.stack(all_vis_images, dim=0) # BHWC
|
||||
category_mask_tensor = torch.stack(all_category_masks, dim=0) # BHW
|
||||
|
||||
# Ensure multiclass_segments is in the right format (BHW) for downstream nodes
|
||||
# Not adding an unsqueeze here, as we're handling both formats in the select_segment node
|
||||
|
||||
|
||||
return (primary_mask_tensor, vis_image_tensor, category_mask_tensor)
|
||||
|
||||
|
||||
# Define mappings for ComfyUI
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"MediaPipeImageSegmenterModelLoader": MediaPipeImageSegmenterModelLoaderNode,
|
||||
@@ -218,4 +254,4 @@ NODE_CLASS_MAPPINGS = {
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"MediaPipeImageSegmenterModelLoader": "Load Image Segmenter Model (MediaPipe)",
|
||||
"MediaPipeImageSegmenter": "Image Segmenter (MediaPipe)",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import torch
|
||||
import numpy as np
|
||||
import logging
|
||||
|
||||
import torch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_category = "Realtime Nodes/MediaPipe Vision/ImageSegmentation/Masking"
|
||||
@@ -15,23 +14,29 @@ MULTICLASS_NAMES = {
|
||||
"Accessories/Other": 5,
|
||||
}
|
||||
|
||||
|
||||
class SelectMediaPipeSegmentNode:
|
||||
"""
|
||||
Selects a specific segmentation mask from the raw multiclass output
|
||||
of the MediaPipeImageSegmenterNode.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
available_segments = list(MULTICLASS_NAMES.keys())
|
||||
return {
|
||||
"required": {
|
||||
"multiclass_segments": ("MP_INT_MASK", {
|
||||
"tooltip": "Raw multiclass segmentation data (INT_MASK) from MediaPipe Image Segmenter node."
|
||||
}),
|
||||
"segment_name": (available_segments, {
|
||||
"default": "Clothes" if "Clothes" in available_segments else available_segments[0],
|
||||
"tooltip": "Select the specific segment to extract as a mask."
|
||||
}),
|
||||
"multiclass_segments": (
|
||||
"MP_INT_MASK",
|
||||
{"tooltip": "Raw multiclass segmentation data (INT_MASK) from MediaPipe Image Segmenter node."},
|
||||
),
|
||||
"segment_name": (
|
||||
available_segments,
|
||||
{
|
||||
"default": "Clothes" if "Clothes" in available_segments else available_segments[0],
|
||||
"tooltip": "Select the specific segment to extract as a mask.",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,14 +49,14 @@ class SelectMediaPipeSegmentNode:
|
||||
def select_segment(self, multiclass_segments: torch.Tensor, segment_name: str):
|
||||
if segment_name not in MULTICLASS_NAMES:
|
||||
raise ValueError(f"Invalid segment_name '{segment_name}'. Available: {list(MULTICLASS_NAMES.keys())}")
|
||||
|
||||
|
||||
if multiclass_segments is None or multiclass_segments.numel() == 0:
|
||||
logger.warning("Input multiclass_segments is empty or None. Returning zero mask.")
|
||||
# Need a shape to return a zero mask. How to get expected shape?
|
||||
# For now, maybe raise error or return None (caller must handle)
|
||||
# Returning None is safer if shape is unknown.
|
||||
return (None,) # Or raise error
|
||||
|
||||
logger.warning("Input multiclass_segments is empty or None. Returning zero mask.")
|
||||
# Need a shape to return a zero mask. How to get expected shape?
|
||||
# For now, maybe raise error or return None (caller must handle)
|
||||
# Returning None is safer if shape is unknown.
|
||||
return (None,) # Or raise error
|
||||
|
||||
# Check tensor format and reshape if needed
|
||||
# Input formats can be either BHW or B1HW (both are common in ComfyUI)
|
||||
if multiclass_segments.dim() == 3: # BHW format
|
||||
@@ -62,22 +67,24 @@ class SelectMediaPipeSegmentNode:
|
||||
elif multiclass_segments.dim() == 4 and multiclass_segments.shape[1] != 1:
|
||||
# If it's BCHW but not B1HW, raise error
|
||||
raise ValueError(f"Expected multiclass_segments to be B1HW or BHW tensor. Got shape {multiclass_segments.shape}")
|
||||
|
||||
|
||||
# Ensure input is a LongTensor
|
||||
if multiclass_segments.dtype != torch.int64:
|
||||
multiclass_segments = multiclass_segments.to(torch.int64)
|
||||
|
||||
target_id = MULTICLASS_NAMES[segment_name]
|
||||
selected_mask = (multiclass_segments == target_id).float()
|
||||
|
||||
|
||||
# If result is B1HW, squeeze back to BHW for ComfyUI mask format compatibility
|
||||
if selected_mask.dim() == 4:
|
||||
selected_mask = selected_mask.squeeze(1)
|
||||
|
||||
return (selected_mask,)
|
||||
# --- End Select Segment Node ---
|
||||
|
||||
# --- Mappings ---
|
||||
return (selected_mask,)
|
||||
|
||||
|
||||
# --- End Select Segment Node ---
|
||||
|
||||
# --- Mappings ---
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"SelectMediaPipeSegment": SelectMediaPipeSegmentNode,
|
||||
}
|
||||
|
||||
+78
-42
@@ -1,23 +1,32 @@
|
||||
import torch
|
||||
import logging
|
||||
import json
|
||||
import logging
|
||||
|
||||
import torch
|
||||
|
||||
from ....src.mediapipe_vision.interactive_segmentation.detector import (
|
||||
InteractiveSegmenterProcessor,
|
||||
)
|
||||
from ....src.mediapipe_vision.types import PointOfInterest
|
||||
|
||||
# Import Base Loader and Detector
|
||||
from ..common.model_loader import MediaPipeModelLoaderBaseNode
|
||||
from ....src.mediapipe_vision.interactive_segmentation.detector import InteractiveSegmenterProcessor
|
||||
from ....src.mediapipe_vision. types import PointOfInterest
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_category = "Realtime Nodes/MediaPipe Vision/InteractiveSegmentation"
|
||||
# --- Model Loader ---
|
||||
|
||||
|
||||
# --- Model Loader ---
|
||||
class MediaPipeInteractiveSegmenterModelLoaderNode(MediaPipeModelLoaderBaseNode):
|
||||
"""ComfyUI node for loading MediaPipe Interactive Segmenter models."""
|
||||
|
||||
# Usually uses a specific model like selfie_segmenter or a custom one
|
||||
TASK_TYPE = "interactive_segmenter" # Need to define this task and models in model_loader.py
|
||||
TASK_TYPE = "interactive_segmenter" # Need to define this task and models in model_loader.py
|
||||
RETURN_TYPES = ("INTERACTIVE_SEGMENTER_MODEL_INFO",)
|
||||
RETURN_NAMES = ("model_info",)
|
||||
CATEGORY = _category
|
||||
# --- Segmenter Node ---
|
||||
|
||||
|
||||
# --- Segmenter Node ---
|
||||
class MediaPipeInteractiveSegmenterNode:
|
||||
"""ComfyUI node for MediaPipe Interactive Image Segmentation."""
|
||||
|
||||
@@ -32,32 +41,54 @@ class MediaPipeInteractiveSegmenterNode:
|
||||
"image": ("IMAGE",),
|
||||
"model_info": ("INTERACTIVE_SEGMENTER_MODEL_INFO",),
|
||||
# Input points as a JSON string list of lists/tuples: e.g., "[[0.5, 0.5], [0.2, 0.3]]"
|
||||
"points_json": ("STRING", {"default": "[[0.5, 0.5]]", "multiline": True,
|
||||
"tooltip": "Points of interest (normalized coords) as JSON: [[x1,y1], [x2,y2], ...]"}),
|
||||
"output_category_mask": ("BOOLEAN", {"default": True,
|
||||
"tooltip": "Output a category mask (0=background, 1=foreground). Priority over confidence."}),
|
||||
"output_confidence_mask": ("BOOLEAN", {"default": False,
|
||||
"tooltip": "Output a confidence mask (0.0-1.0). Used if category mask is False."}),
|
||||
"points_json": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "[[0.5, 0.5]]",
|
||||
"multiline": True,
|
||||
"tooltip": "Points of interest (normalized coords) as JSON: [[x1,y1], [x2,y2], ...]",
|
||||
},
|
||||
),
|
||||
"output_category_mask": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": True,
|
||||
"tooltip": "Output a category mask (0=background, 1=foreground). Priority over confidence.",
|
||||
},
|
||||
),
|
||||
"output_confidence_mask": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "Output a confidence mask (0.0-1.0). Used if category mask is False.",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("MASK",) # Output is a standard ComfyUI mask
|
||||
RETURN_TYPES = ("MASK",) # Output is a standard ComfyUI mask
|
||||
RETURN_NAMES = ("mask",)
|
||||
FUNCTION = "segment_interactive"
|
||||
CATEGORY = _category
|
||||
|
||||
def segment_interactive(self, image: torch.Tensor, model_info: dict, points_json: str,
|
||||
output_category_mask: bool, output_confidence_mask: bool):
|
||||
def segment_interactive(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
model_info: dict,
|
||||
points_json: str,
|
||||
output_category_mask: bool,
|
||||
output_confidence_mask: bool,
|
||||
):
|
||||
"""Performs interactive segmentation."""
|
||||
|
||||
task_type = model_info.get('task_type')
|
||||
|
||||
task_type = model_info.get("task_type")
|
||||
expected_task_type = MediaPipeInteractiveSegmenterModelLoaderNode.TASK_TYPE
|
||||
if not isinstance(model_info, dict) or task_type != expected_task_type:
|
||||
raise ValueError(f"Invalid model_info. Expected task_type '{expected_task_type}' but got '{task_type}'.")
|
||||
model_path = model_info.get('model_path')
|
||||
raise ValueError(f"Invalid model_info. Expected task_type '{expected_task_type}' but got '{task_type}'.")
|
||||
model_path = model_info.get("model_path")
|
||||
if not model_path:
|
||||
raise ValueError("Model path not found or invalid in model_info.")
|
||||
|
||||
raise ValueError("Model path not found or invalid in model_info.")
|
||||
|
||||
# Parse points from JSON
|
||||
try:
|
||||
points_list = json.loads(points_json)
|
||||
@@ -69,42 +100,47 @@ class MediaPipeInteractiveSegmenterNode:
|
||||
|
||||
# Manage detector instance
|
||||
if self._processor is None or self._model_path != model_path:
|
||||
if self._processor and hasattr(self._processor, 'close'):
|
||||
try: self._processor.close()
|
||||
except Exception as e: logger.warning(f"Error closing InteractiveSegmenterProcessor: {e}")
|
||||
if self._processor and hasattr(self._processor, "close"):
|
||||
try:
|
||||
self._processor.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing InteractiveSegmenterProcessor: {e}")
|
||||
logger.info(f"Creating new InteractiveSegmenterProcessor instance for {model_path}")
|
||||
self._processor = InteractiveSegmenterProcessor(model_path)
|
||||
self._model_path = model_path
|
||||
|
||||
|
||||
# Call the detector's segment method
|
||||
mask_batch = self._processor.segment(
|
||||
image,
|
||||
points_of_interest=points_of_interest,
|
||||
output_category_mask=output_category_mask,
|
||||
output_confidence_mask=output_confidence_mask
|
||||
output_confidence_mask=output_confidence_mask,
|
||||
)
|
||||
|
||||
|
||||
# Convert HW mask list to B1HW tensor for ComfyUI MASK type
|
||||
# Assuming detector returns one mask per batch item
|
||||
comfy_masks = []
|
||||
for mask_hw in mask_batch:
|
||||
if mask_hw is not None:
|
||||
comfy_masks.append(mask_hw.unsqueeze(0)) # Add channel dim
|
||||
else:
|
||||
# Handle None case - maybe return zero mask? Needs image shape
|
||||
h, w = image.shape[1], image.shape[2]
|
||||
comfy_masks.append(torch.zeros((1, h, w), dtype=torch.float32, device=image.device))
|
||||
|
||||
output_tensor = torch.stack(comfy_masks, dim=0) # Stack along batch dim
|
||||
if mask_hw is not None:
|
||||
comfy_masks.append(mask_hw.unsqueeze(0)) # Add channel dim
|
||||
else:
|
||||
# Handle None case - maybe return zero mask? Needs image shape
|
||||
h, w = image.shape[1], image.shape[2]
|
||||
comfy_masks.append(torch.zeros((1, h, w), dtype=torch.float32, device=image.device))
|
||||
|
||||
output_tensor = torch.stack(comfy_masks, dim=0) # Stack along batch dim
|
||||
return (output_tensor,)
|
||||
|
||||
def __del__(self):
|
||||
if hasattr(self, '_processor') and self._processor and hasattr(self._processor, 'close'):
|
||||
try: self._processor.close()
|
||||
except Exception as e: logger.warning(f"Error closing InteractiveSegmenterProcessor in __del__: {e}")
|
||||
self._processor = None
|
||||
if hasattr(self, "_processor") and self._processor and hasattr(self._processor, "close"):
|
||||
try:
|
||||
self._processor.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing InteractiveSegmenterProcessor in __del__: {e}")
|
||||
self._processor = None
|
||||
|
||||
# --- Mappings ---
|
||||
|
||||
# --- Mappings ---
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"MediaPipeInteractiveSegmenterModelLoader": MediaPipeInteractiveSegmenterModelLoaderNode,
|
||||
"MediaPipeInteractiveSegmenter": MediaPipeInteractiveSegmenterNode,
|
||||
@@ -113,4 +149,4 @@ NODE_CLASS_MAPPINGS = {
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"MediaPipeInteractiveSegmenterModelLoader": "Load Interactive Segmenter Model (MediaPipe)",
|
||||
"MediaPipeInteractiveSegmenter": "Interactive Segmenter (MediaPipe)",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
|
||||
@@ -1,19 +1,24 @@
|
||||
import torch
|
||||
from ....src.mediapipe_vision.object_detection.detector import ObjectDetector
|
||||
from ..common.model_loader import MediaPipeModelLoaderBaseNode
|
||||
from ..common.base_detector_node import BaseMediaPipeDetectorNode
|
||||
import logging
|
||||
|
||||
import torch
|
||||
|
||||
from ....src.mediapipe_vision.object_detection.detector import ObjectDetector
|
||||
from ..common.base_detector_node import BaseMediaPipeDetectorNode
|
||||
from ..common.model_loader import MediaPipeModelLoaderBaseNode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_category = "Realtime Nodes/MediaPipe Vision/ObjectDetection"
|
||||
|
||||
|
||||
class MediaPipeObjectDetectorModelLoaderNode(MediaPipeModelLoaderBaseNode):
|
||||
"""ComfyUI node for loading MediaPipe Object Detector models."""
|
||||
|
||||
TASK_TYPE = "object_detector"
|
||||
RETURN_TYPES = ("OBJECT_DETECTOR_MODEL_INFO",)
|
||||
RETURN_NAMES = ("model_info",)
|
||||
CATEGORY = _category
|
||||
|
||||
|
||||
class MediaPipeObjectDetectorNode(BaseMediaPipeDetectorNode):
|
||||
"""ComfyUI node for MediaPipe Object Detection."""
|
||||
|
||||
@@ -30,38 +35,54 @@ class MediaPipeObjectDetectorNode(BaseMediaPipeDetectorNode):
|
||||
def INPUT_TYPES(cls):
|
||||
# Start with the base inputs from the parent class
|
||||
inputs = super().INPUT_TYPES()
|
||||
|
||||
|
||||
# Add object detection specific parameters
|
||||
inputs["required"].update({
|
||||
"min_confidence": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.05,
|
||||
"tooltip": "Minimum confidence score for detected objects"}),
|
||||
"max_results": ("INT", {"default": 5, "min": 1, "max": 50, "step": 1,
|
||||
"tooltip": "Maximum number of objects to detect"}),
|
||||
})
|
||||
|
||||
inputs["required"].update(
|
||||
{
|
||||
"min_confidence": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.05,
|
||||
"tooltip": "Minimum confidence score for detected objects",
|
||||
},
|
||||
),
|
||||
"max_results": (
|
||||
"INT",
|
||||
{"default": 5, "min": 1, "max": 50, "step": 1, "tooltip": "Maximum number of objects to detect"},
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
return inputs
|
||||
|
||||
def detect(self, image: torch.Tensor, model_info: dict, min_confidence: float,
|
||||
max_results: int, running_mode: str, delegate: str):
|
||||
def detect(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
model_info: dict,
|
||||
min_confidence: float,
|
||||
max_results: int,
|
||||
running_mode: str,
|
||||
delegate: str,
|
||||
):
|
||||
"""Performs object detection with the configured parameters."""
|
||||
|
||||
|
||||
# Validate model_info and get model path
|
||||
model_path = self.validate_model_info(model_info)
|
||||
|
||||
|
||||
# Initialize or update detector
|
||||
detector = self.initialize_or_update_detector(model_path)
|
||||
|
||||
|
||||
# Perform detection with all parameters
|
||||
batch_results = detector.detect(
|
||||
image,
|
||||
score_threshold=min_confidence,
|
||||
max_results=max_results,
|
||||
running_mode=running_mode,
|
||||
delegate=delegate
|
||||
image, score_threshold=min_confidence, max_results=max_results, running_mode=running_mode, delegate=delegate
|
||||
)
|
||||
|
||||
|
||||
return (batch_results,)
|
||||
|
||||
|
||||
# Define mappings for ComfyUI
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"MediaPipeObjectDetectorModelLoader": MediaPipeObjectDetectorModelLoaderNode,
|
||||
@@ -71,4 +92,4 @@ NODE_CLASS_MAPPINGS = {
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"MediaPipeObjectDetectorModelLoader": "Load Object Detector Model (MediaPipe)",
|
||||
"MediaPipeObjectDetector": "Object Detector (MediaPipe)",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,69 +1,83 @@
|
||||
# visualize_object_detection.py
|
||||
import torch
|
||||
import numpy as np
|
||||
import logging
|
||||
|
||||
import cv2
|
||||
|
||||
from ....src.utils.image import convert_to_cv2, convert_to_tensor
|
||||
from ..common.base_visualization_nodes import BaseDetectionVisualizationNode
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
_category = "Realtime Nodes/MediaPipe Vision/ObjectDetection/Visualization"
|
||||
|
||||
|
||||
class VisualizeObjectDetections(BaseDetectionVisualizationNode):
|
||||
"""Draws object detection bounding boxes and labels onto images."""
|
||||
|
||||
CATEGORY = _category
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"object_detections": ("OBJECT_DETECTIONS",), # Expect the specific type
|
||||
"object_detections": ("OBJECT_DETECTIONS",), # Expect the specific type
|
||||
"image": ("IMAGE",),
|
||||
},
|
||||
"optional": {
|
||||
"box_color": ("STRING", {"default": "#FF0000"}), # Red
|
||||
"box_color": ("STRING", {"default": "#FF0000"}), # Red
|
||||
"box_thickness": ("INT", {"default": 2, "min": 1, "max": 10}),
|
||||
"label_color": ("STRING", {"default": "#FFFFFF"}), # White
|
||||
"label_bg_color": ("STRING", {"default": "#FF0000"}), # Red BG
|
||||
"label_color": ("STRING", {"default": "#FFFFFF"}), # White
|
||||
"label_bg_color": ("STRING", {"default": "#FF0000"}), # Red BG
|
||||
# Font size might be tricky to implement well, skipping for now
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "draw_visualizations"
|
||||
|
||||
def draw_visualizations(self, object_detections, image,
|
||||
box_color="#FF0000", box_thickness=2,
|
||||
label_color="#FFFFFF", label_bg_color="#FF0000"):
|
||||
def draw_visualizations(
|
||||
self,
|
||||
object_detections,
|
||||
image,
|
||||
box_color="#FF0000",
|
||||
box_thickness=2,
|
||||
label_color="#FFFFFF",
|
||||
label_bg_color="#FF0000",
|
||||
):
|
||||
"""Main function that processes the batch of images and detections."""
|
||||
return self.process_batch(
|
||||
object_detections,
|
||||
object_detections,
|
||||
image,
|
||||
self.draw_objects,
|
||||
box_color=box_color,
|
||||
box_thickness=box_thickness,
|
||||
label_color=label_color,
|
||||
label_bg_color=label_bg_color
|
||||
label_bg_color=label_bg_color,
|
||||
)
|
||||
|
||||
def draw_objects(self, image, width, height, detections,
|
||||
box_color="#FF0000", box_thickness=2,
|
||||
label_color="#FFFFFF", label_bg_color="#FF0000"):
|
||||
|
||||
def draw_objects(
|
||||
self,
|
||||
image,
|
||||
width,
|
||||
height,
|
||||
detections,
|
||||
box_color="#FF0000",
|
||||
box_thickness=2,
|
||||
label_color="#FFFFFF",
|
||||
label_bg_color="#FF0000",
|
||||
):
|
||||
"""Callback function to draw objects on a single image."""
|
||||
# Convert hex colors to BGR tuples for cv2
|
||||
box_color_bgr = self._hex_to_bgr(box_color)
|
||||
|
||||
|
||||
for detection in detections:
|
||||
bbox = detection.bounding_box
|
||||
# Top-left corner
|
||||
x1, y1 = int(bbox.origin_x), int(bbox.origin_y)
|
||||
# Bottom-right corner
|
||||
x2, y2 = int(bbox.origin_x + bbox.width), int(bbox.origin_y + bbox.height)
|
||||
|
||||
|
||||
# Draw bounding box
|
||||
cv2.rectangle(image, (x1, y1), (x2, y2), box_color_bgr, box_thickness)
|
||||
|
||||
|
||||
# Prepare label text (use first category's display name if available)
|
||||
label = "N/A"
|
||||
score = 0.0
|
||||
@@ -71,36 +85,27 @@ class VisualizeObjectDetections(BaseDetectionVisualizationNode):
|
||||
top_category = detection.categories[0]
|
||||
label = top_category.display_name or top_category.category_name or "N/A"
|
||||
score = top_category.score
|
||||
|
||||
|
||||
label_text = f"{label}: {score:.2f}"
|
||||
|
||||
|
||||
# Draw label background and text inside the box top-left
|
||||
try:
|
||||
self.draw_text_with_background(
|
||||
image,
|
||||
label_text,
|
||||
(x1, y1),
|
||||
text_color=label_color,
|
||||
bg_color=label_bg_color
|
||||
)
|
||||
self.draw_text_with_background(image, label_text, (x1, y1), text_color=label_color, bg_color=label_bg_color)
|
||||
except Exception as e:
|
||||
logger.error(f"Error drawing text label: {e}")
|
||||
# Fallback: Draw text without background
|
||||
cv2.putText(
|
||||
image,
|
||||
label_text,
|
||||
(x1 + 2, y1 + 15),
|
||||
cv2.FONT_HERSHEY_SIMPLEX,
|
||||
0.5,
|
||||
self._hex_to_bgr(label_color),
|
||||
1
|
||||
image,
|
||||
label_text,
|
||||
(x1 + 2, y1 + 15),
|
||||
cv2.FONT_HERSHEY_SIMPLEX,
|
||||
0.5,
|
||||
self._hex_to_bgr(label_color),
|
||||
1,
|
||||
)
|
||||
|
||||
# Node registration
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"VisualizeObjectDetections": VisualizeObjectDetections
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"VisualizeObjectDetections": "Visualize Object Detections (MediaPipe)"
|
||||
}
|
||||
# Node registration
|
||||
NODE_CLASS_MAPPINGS = {"VisualizeObjectDetections": VisualizeObjectDetections}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {"VisualizeObjectDetections": "Visualize Object Detections (MediaPipe)"}
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
|
||||
@@ -1,20 +1,25 @@
|
||||
import torch
|
||||
from ....src.mediapipe_vision.pose_landmark.detector import PoseLandmarkDetector
|
||||
from ..common.model_loader import MediaPipeModelLoaderBaseNode
|
||||
from ..common.base_detector_node import BaseMediaPipeDetectorNode
|
||||
import logging
|
||||
|
||||
import torch
|
||||
|
||||
from ....src.mediapipe_vision.pose_landmark.detector import PoseLandmarkDetector
|
||||
from ..common.base_detector_node import BaseMediaPipeDetectorNode
|
||||
from ..common.model_loader import MediaPipeModelLoaderBaseNode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_category = "Realtime Nodes/MediaPipe Vision/Pose"
|
||||
|
||||
|
||||
class MediaPipePoseLandmarkerModelLoaderNode(MediaPipeModelLoaderBaseNode):
|
||||
"""ComfyUI node for loading MediaPipe Pose Landmarker models."""
|
||||
|
||||
TASK_TYPE = "pose_landmarker"
|
||||
RETURN_TYPES = ("POSE_LANDMARKER_MODEL_INFO",)
|
||||
RETURN_NAMES = ("model_info",)
|
||||
CATEGORY = _category
|
||||
DESCRIPTION = "Loads a MediaPipe Pose Landmarker model for detecting human body poses and postures in images. Required before using the Pose Landmarker node."
|
||||
|
||||
|
||||
class MediaPipePoseLandmarkerNode(BaseMediaPipeDetectorNode):
|
||||
"""ComfyUI node for MediaPipe Pose Landmark Detection."""
|
||||
|
||||
@@ -22,7 +27,10 @@ class MediaPipePoseLandmarkerNode(BaseMediaPipeDetectorNode):
|
||||
DETECTOR_CLASS = PoseLandmarkDetector
|
||||
MODEL_INFO_TYPE = "POSE_LANDMARKER_MODEL_INFO"
|
||||
EXPECTED_TASK_TYPE = "pose_landmarker"
|
||||
RETURN_TYPES = ("POSE_LANDMARKS", "MASK_LIST",)
|
||||
RETURN_TYPES = (
|
||||
"POSE_LANDMARKS",
|
||||
"MASK_LIST",
|
||||
)
|
||||
RETURN_NAMES = ("landmarks", "segmentation_masks")
|
||||
FUNCTION = "detect"
|
||||
CATEGORY = _category
|
||||
@@ -32,35 +40,82 @@ class MediaPipePoseLandmarkerNode(BaseMediaPipeDetectorNode):
|
||||
def INPUT_TYPES(cls):
|
||||
# Start with the base inputs from the parent class
|
||||
inputs = super().INPUT_TYPES()
|
||||
|
||||
|
||||
# Add pose-specific parameters
|
||||
inputs["required"].update({
|
||||
"max_results": ("INT", {"default": 1, "min": 1, "max": 4, "step": 1,
|
||||
"tooltip": "Maximum number of people/poses to detect in the image - higher values can detect more people but use more processing power"}),
|
||||
"min_confidence": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
"tooltip": "Minimum confidence threshold for pose detection - lower values detect more poses but may include false positives"}),
|
||||
"min_tracking_confidence": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
"tooltip": "(VIDEO mode) Minimum confidence threshold for pose tracking between frames - lower values maintain tracking longer but may be less accurate"}),
|
||||
"min_presence_confidence": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
"tooltip": "(VIDEO mode) Minimum confidence threshold for pose presence in the image - lower values may include false negatives"}),
|
||||
"output_segmentation_masks": ("BOOLEAN", {"default": False,
|
||||
"tooltip": "When enabled, generates masks that separate people from the background"}),
|
||||
})
|
||||
|
||||
inputs["required"].update(
|
||||
{
|
||||
"max_results": (
|
||||
"INT",
|
||||
{
|
||||
"default": 1,
|
||||
"min": 1,
|
||||
"max": 4,
|
||||
"step": 1,
|
||||
"tooltip": "Maximum number of people/poses to detect in the image - higher values can detect more people but use more processing power",
|
||||
},
|
||||
),
|
||||
"min_confidence": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Minimum confidence threshold for pose detection - lower values detect more poses but may include false positives",
|
||||
},
|
||||
),
|
||||
"min_tracking_confidence": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "(VIDEO mode) Minimum confidence threshold for pose tracking between frames - lower values maintain tracking longer but may be less accurate",
|
||||
},
|
||||
),
|
||||
"min_presence_confidence": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "(VIDEO mode) Minimum confidence threshold for pose presence in the image - lower values may include false negatives",
|
||||
},
|
||||
),
|
||||
"output_segmentation_masks": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "When enabled, generates masks that separate people from the background",
|
||||
},
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
return inputs
|
||||
|
||||
def detect(self, image: torch.Tensor, model_info: dict, max_results: int,
|
||||
min_confidence: float, min_tracking_confidence: float,
|
||||
min_presence_confidence: float, output_segmentation_masks: bool,
|
||||
running_mode: str, delegate: str):
|
||||
def detect(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
model_info: dict,
|
||||
max_results: int,
|
||||
min_confidence: float,
|
||||
min_tracking_confidence: float,
|
||||
min_presence_confidence: float,
|
||||
output_segmentation_masks: bool,
|
||||
running_mode: str,
|
||||
delegate: str,
|
||||
):
|
||||
"""Performs pose landmark detection with the configured parameters."""
|
||||
|
||||
|
||||
# Validate model_info and get model path
|
||||
model_path = self.validate_model_info(model_info)
|
||||
|
||||
|
||||
# Initialize or update detector
|
||||
detector = self.initialize_or_update_detector(model_path)
|
||||
|
||||
|
||||
# Perform detection with all parameters
|
||||
pose_landmarks_batch, segmentation_masks_batch = detector.detect(
|
||||
image,
|
||||
@@ -70,18 +125,19 @@ class MediaPipePoseLandmarkerNode(BaseMediaPipeDetectorNode):
|
||||
min_presence_confidence=min_presence_confidence,
|
||||
output_segmentation_masks=output_segmentation_masks,
|
||||
running_mode=running_mode,
|
||||
delegate=delegate
|
||||
delegate=delegate,
|
||||
)
|
||||
|
||||
|
||||
# Process segmentation masks if needed
|
||||
flat_masks = []
|
||||
if segmentation_masks_batch:
|
||||
for img_masks in segmentation_masks_batch:
|
||||
if img_masks:
|
||||
flat_masks.extend(img_masks)
|
||||
|
||||
|
||||
return (pose_landmarks_batch, flat_masks if output_segmentation_masks else [])
|
||||
|
||||
|
||||
# Define mappings for ComfyUI
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"MediaPipePoseLandmarkerModelLoader": MediaPipePoseLandmarkerModelLoaderNode,
|
||||
@@ -91,4 +147,4 @@ NODE_CLASS_MAPPINGS = {
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"MediaPipePoseLandmarkerModelLoader": "Load Pose Landmarker Model (MediaPipe)",
|
||||
"MediaPipePoseLandmarker": "Pose Landmarker (MediaPipe)",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
# node_wrappers/pose/masking.py
|
||||
import torch
|
||||
import numpy as np
|
||||
import cv2
|
||||
import logging
|
||||
from typing import List, Dict, Tuple
|
||||
from typing import List
|
||||
|
||||
import torch
|
||||
|
||||
from ....src.mediapipe_vision.types import PoseLandmarksResult
|
||||
from ....src.utils.image import create_mask_from_points
|
||||
|
||||
# Removed GetImageDimensionsNode import
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -15,22 +15,24 @@ _category = "Realtime Nodes/MediaPipe Vision/Pose/Masking"
|
||||
# --- Pose Part Definitions (Based on standard MediaPipe Pose Indices - BlazePose) ---
|
||||
# These indices need verification
|
||||
POSE_PART_INDICES = {
|
||||
"HEAD": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], # Example
|
||||
"TORSO": [11, 12, 23, 24], # Example
|
||||
"LEFT_ARM": [11, 13, 15], # Upper arm only
|
||||
"RIGHT_ARM": [12, 14, 16], # Upper arm only
|
||||
"LEFT_FOREARM": [13, 15, 17, 19, 21], # Includes hand points?
|
||||
"RIGHT_FOREARM": [14, 16, 18, 20, 22], # Includes hand points?
|
||||
"LEFT_LEG": [23, 25, 27], # Upper leg
|
||||
"RIGHT_LEG": [24, 26, 28], # Upper leg
|
||||
"LEFT_LOWER_LEG": [25, 27, 29, 31], # Includes foot?
|
||||
"RIGHT_LOWER_LEG": [26, 28, 30, 32], # Includes foot?
|
||||
"HEAD": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], # Example
|
||||
"TORSO": [11, 12, 23, 24], # Example
|
||||
"LEFT_ARM": [11, 13, 15], # Upper arm only
|
||||
"RIGHT_ARM": [12, 14, 16], # Upper arm only
|
||||
"LEFT_FOREARM": [13, 15, 17, 19, 21], # Includes hand points?
|
||||
"RIGHT_FOREARM": [14, 16, 18, 20, 22], # Includes hand points?
|
||||
"LEFT_LEG": [23, 25, 27], # Upper leg
|
||||
"RIGHT_LEG": [24, 26, 28], # Upper leg
|
||||
"LEFT_LOWER_LEG": [25, 27, 29, 31], # Includes foot?
|
||||
"RIGHT_LOWER_LEG": [26, 28, 30, 32], # Includes foot?
|
||||
"FULL_BODY": list(range(33)),
|
||||
}
|
||||
|
||||
# --- Node Definition ---
|
||||
|
||||
# --- Node Definition ---
|
||||
class MaskFromPoseLandmarksNode:
|
||||
"""Creates a mask for a specific body part from Pose Landmarks."""
|
||||
|
||||
CATEGORY = _category
|
||||
DESCRIPTION = "Creates precise masks for specific body parts like head, torso, arms, or legs. Perfect for targeted editing or creating effects on specific body regions."
|
||||
|
||||
@@ -39,22 +41,47 @@ class MaskFromPoseLandmarksNode:
|
||||
part_names = list(POSE_PART_INDICES.keys())
|
||||
return {
|
||||
"required": {
|
||||
"pose_landmarks": ("POSE_LANDMARKS", {"tooltip": "Pose landmark data from the Pose Landmarker node containing the precise locations of body joints"}),
|
||||
"image_for_dimensions": ("IMAGE", {"tooltip": "Reference image to determine the size of the output mask - the mask will match these dimensions"}),
|
||||
"part_name": (part_names, {"default": "FULL_BODY",
|
||||
"tooltip": "Choose which body part to create a mask for - FULL_BODY (entire person), HEAD, TORSO, LEFT_ARM, etc."}),
|
||||
"min_visibility": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.05,
|
||||
"tooltip": "Only include landmarks that are at least this visible (0.0 = include all points regardless of visibility, 1.0 = only fully visible points)"}),
|
||||
"pose_landmarks": (
|
||||
"POSE_LANDMARKS",
|
||||
{"tooltip": "Pose landmark data from the Pose Landmarker node containing the precise locations of body joints"},
|
||||
),
|
||||
"image_for_dimensions": (
|
||||
"IMAGE",
|
||||
{"tooltip": "Reference image to determine the size of the output mask - the mask will match these dimensions"},
|
||||
),
|
||||
"part_name": (
|
||||
part_names,
|
||||
{
|
||||
"default": "FULL_BODY",
|
||||
"tooltip": "Choose which body part to create a mask for - FULL_BODY (entire person), HEAD, TORSO, LEFT_ARM, etc.",
|
||||
},
|
||||
),
|
||||
"min_visibility": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.05,
|
||||
"tooltip": "Only include landmarks that are at least this visible (0.0 = include all points regardless of visibility, 1.0 = only fully visible points)",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("MASK",)
|
||||
FUNCTION = "generate_mask"
|
||||
|
||||
def generate_mask(self, pose_landmarks: List[List[PoseLandmarksResult]], image_for_dimensions: torch.Tensor, part_name: str, min_visibility: float):
|
||||
def generate_mask(
|
||||
self,
|
||||
pose_landmarks: List[List[PoseLandmarksResult]],
|
||||
image_for_dimensions: torch.Tensor,
|
||||
part_name: str,
|
||||
min_visibility: float,
|
||||
):
|
||||
if part_name not in POSE_PART_INDICES:
|
||||
raise ValueError(f"Unknown pose part name: '{part_name}'. Available: {list(POSE_PART_INDICES.keys())}")
|
||||
|
||||
|
||||
# Get dimensions from image
|
||||
if image_for_dimensions.dim() != 4:
|
||||
raise ValueError("image_for_dimensions must be BHWC format.")
|
||||
@@ -65,7 +92,9 @@ class MaskFromPoseLandmarksNode:
|
||||
|
||||
# Validate batch sizes match
|
||||
if len(pose_landmarks) != input_batch_size:
|
||||
logger.warning(f"Batch size mismatch between pose_landmarks ({len(pose_landmarks)}) and image_for_dimensions ({input_batch_size}). Using landmark batch size.")
|
||||
logger.warning(
|
||||
f"Batch size mismatch between pose_landmarks ({len(pose_landmarks)}) and image_for_dimensions ({input_batch_size}). Using landmark batch size."
|
||||
)
|
||||
batch_size = len(pose_landmarks)
|
||||
else:
|
||||
batch_size = input_batch_size
|
||||
@@ -74,7 +103,7 @@ class MaskFromPoseLandmarksNode:
|
||||
output_masks = []
|
||||
|
||||
for i in range(batch_size):
|
||||
landmarks_for_image = pose_landmarks[i] # List[PoseLandmarksResult]
|
||||
landmarks_for_image = pose_landmarks[i] # List[PoseLandmarksResult]
|
||||
combined_points_for_image = []
|
||||
|
||||
# Combine points from all detected poses for the selected part
|
||||
@@ -92,24 +121,25 @@ class MaskFromPoseLandmarksNode:
|
||||
part_points.append((px, py))
|
||||
# No warning for missing index
|
||||
combined_points_for_image.extend(part_points)
|
||||
|
||||
|
||||
# Create mask from all collected points for this image
|
||||
mask_tensor = create_mask_from_points(height, width, combined_points_for_image, device=device)
|
||||
output_masks.append(mask_tensor)
|
||||
|
||||
if not output_masks:
|
||||
logger.warning("No landmarks found to generate any masks. Returning zero mask batch.")
|
||||
output_batch = torch.zeros((input_batch_size, 1, height, width), dtype=torch.float32, device=device)
|
||||
logger.warning("No landmarks found to generate any masks. Returning zero mask batch.")
|
||||
output_batch = torch.zeros((input_batch_size, 1, height, width), dtype=torch.float32, device=device)
|
||||
else:
|
||||
output_batch = torch.stack(output_masks, dim=0)
|
||||
|
||||
output_batch = torch.stack(output_masks, dim=0)
|
||||
|
||||
return (output_batch,)
|
||||
|
||||
# --- Mappings ---
|
||||
|
||||
# --- Mappings ---
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"MaskFromPoseLandmarks": MaskFromPoseLandmarksNode,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"MaskFromPoseLandmarks": "Mask From Pose Landmarks (MediaPipe)",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,19 +1,20 @@
|
||||
# visualize_pose.py
|
||||
import logging
|
||||
from typing import Any, List, Optional
|
||||
|
||||
import mediapipe as mp
|
||||
from typing import Optional, List, Any
|
||||
|
||||
from ....src.mediapipe_vision.types import PoseLandmarksResult # Import specific result type if needed for extraction
|
||||
from ..common.base_visualization_nodes import BaseLandmarkVisualizationNode
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
_category = "Realtime Nodes/MediaPipe Vision/Pose/Visualization"
|
||||
# Define standard pose connections
|
||||
POSE_CONNECTIONS = mp.solutions.pose.POSE_CONNECTIONS
|
||||
|
||||
|
||||
class VisualizePoseLandmarks(BaseLandmarkVisualizationNode):
|
||||
"""Draws Pose landmarks and connections onto images."""
|
||||
|
||||
|
||||
# Override base class properties
|
||||
LANDMARKS_TYPE = "POSE_LANDMARKS"
|
||||
CONNECTIONS = POSE_CONNECTIONS
|
||||
@@ -22,18 +23,14 @@ class VisualizePoseLandmarks(BaseLandmarkVisualizationNode):
|
||||
DEFAULT_POINT_RADIUS = 3
|
||||
DEFAULT_CONNECTION_COLOR = "#FFFFFF" # White
|
||||
DEFAULT_CONNECTION_THICKNESS = 2
|
||||
SUPPORTS_VISIBILITY = True # Pose landmarks have visibility
|
||||
SUPPORTS_VISIBILITY = True # Pose landmarks have visibility
|
||||
CATEGORY = _category
|
||||
|
||||
|
||||
def get_landmark_list(self, detection_result: Any) -> Optional[List[Any]]:
|
||||
return detection_result.landmarks
|
||||
|
||||
|
||||
# Node registration
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"VisualizePoseLandmarks": VisualizePoseLandmarks
|
||||
}
|
||||
NODE_CLASS_MAPPINGS = {"VisualizePoseLandmarks": VisualizePoseLandmarks}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"VisualizePoseLandmarks": "Visualize Pose Landmarks (MediaPipe)"
|
||||
}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {"VisualizePoseLandmarks": "Visualize Pose Landmarks (MediaPipe)"}
|
||||
|
||||
@@ -1,19 +1,20 @@
|
||||
import logging
|
||||
from typing import List, Tuple, Any, Optional
|
||||
|
||||
# Import the base class
|
||||
from ....src.mediapipe_vision.common.position import LandmarkPositionBase
|
||||
from ....src.mediapipe_vision.common.position import LandmarkPositionBase
|
||||
|
||||
# Import the specific landmark type for clarity and potential future use
|
||||
from ....src.mediapipe_vision.types import POSE_LANDMARKS
|
||||
# Import the landmark definitions
|
||||
from ....src.mediapipe_vision.landmark_definitions import POSE_LANDMARK_TOOLTIP
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_category = "Realtime Nodes/MediaPipe Vision/Pose/PoseLandmark/Position"
|
||||
|
||||
|
||||
class PoseLandmarkPositionNode(LandmarkPositionBase):
|
||||
"""Extracts position lists (x, y, z, vis, pres) for a specific landmark index
|
||||
from POSE_LANDMARKS across the batch."""
|
||||
from POSE_LANDMARKS across the batch."""
|
||||
|
||||
CATEGORY = _category
|
||||
# Define the specific input type string this node expects
|
||||
LANDMARKS_TYPE = "POSE_LANDMARKS"
|
||||
@@ -26,14 +27,21 @@ class PoseLandmarkPositionNode(LandmarkPositionBase):
|
||||
"required": {
|
||||
"landmarks": (cls.LANDMARKS_TYPE, {"forceInput": True}),
|
||||
"landmark_index": ("INT", {"default": 0, "min": 0, "max": 32, "tooltip": POSE_LANDMARK_TOOLTIP}),
|
||||
"result_index": ("INT", {"default": 0, "min": 0, "tooltip": "Index of the pose detection to use (0=first detected pose)"}),
|
||||
"use_world_coordinates": ("BOOLEAN", {"default": False, "tooltip": "Use world coordinates if available"}),
|
||||
"result_index": (
|
||||
"INT",
|
||||
{"default": 0, "min": 0, "tooltip": "Index of the pose detection to use (0=first detected pose)"},
|
||||
),
|
||||
"use_world_coordinates": (
|
||||
"BOOLEAN",
|
||||
{"default": False, "tooltip": "Use world coordinates if available"},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
# No need to override INPUT_TYPES or extract_positions, inheriting from base is sufficient.
|
||||
# The base class handles everything using self.LANDMARKS_TYPE.
|
||||
|
||||
|
||||
# --- Node Mappings ---
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"PoseLandmarkPosition": PoseLandmarkPositionNode,
|
||||
@@ -41,4 +49,4 @@ NODE_CLASS_MAPPINGS = {
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"PoseLandmarkPosition": "Pose Landmark Position Extractor (Batch)",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1 +1 @@
|
||||
# This file intentionally left blank
|
||||
# This file intentionally left blank
|
||||
|
||||
@@ -1 +1 @@
|
||||
# This file intentionally left blank
|
||||
# This file intentionally left blank
|
||||
|
||||
@@ -1,49 +1,53 @@
|
||||
import torch
|
||||
import numpy as np
|
||||
import cv2
|
||||
import torch
|
||||
|
||||
from ....src.realtimenodes.control_base import ControlNodeBase
|
||||
from ....src.realtimenodes.mask_controls import MaskControlMixin
|
||||
|
||||
|
||||
class RepulsiveMaskNode(ControlNodeBase, MaskControlMixin):
|
||||
"""Node that maintains a mask that repulses or attracts to input masks"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
inputs = super().INPUT_TYPES()
|
||||
inputs["required"].update({
|
||||
"input_mask": ("MASK",),
|
||||
"x_pos": ("FLOAT", {
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Initial X position (0-1)"
|
||||
}),
|
||||
"y_pos": ("FLOAT", {
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Initial Y position (0-1)"
|
||||
}),
|
||||
"size": ("FLOAT", {
|
||||
"default": 0.2,
|
||||
"min": 0.01,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Size of the mask relative to image size"
|
||||
}),
|
||||
"interaction_type": (["repulse", "attract"], {
|
||||
"default": "repulse",
|
||||
"tooltip": "Whether to move away from or toward input masks"
|
||||
}),
|
||||
"interaction_strength": ("FLOAT", {
|
||||
"default": 0.1,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Strength of the interaction effect"
|
||||
})
|
||||
})
|
||||
inputs["required"].update(
|
||||
{
|
||||
"input_mask": ("MASK",),
|
||||
"x_pos": (
|
||||
"FLOAT",
|
||||
{"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "Initial X position (0-1)"},
|
||||
),
|
||||
"y_pos": (
|
||||
"FLOAT",
|
||||
{"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "Initial Y position (0-1)"},
|
||||
),
|
||||
"size": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.2,
|
||||
"min": 0.01,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Size of the mask relative to image size",
|
||||
},
|
||||
),
|
||||
"interaction_type": (
|
||||
["repulse", "attract"],
|
||||
{"default": "repulse", "tooltip": "Whether to move away from or toward input masks"},
|
||||
),
|
||||
"interaction_strength": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.1,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Strength of the interaction effect",
|
||||
},
|
||||
),
|
||||
}
|
||||
)
|
||||
return inputs
|
||||
|
||||
RETURN_TYPES = ("MASK",)
|
||||
@@ -53,45 +57,45 @@ class RepulsiveMaskNode(ControlNodeBase, MaskControlMixin):
|
||||
def interact(self, mask, input_mask_np, state, strength, height, width, is_attract):
|
||||
if np.max(input_mask_np) < 0.1:
|
||||
return state
|
||||
|
||||
|
||||
y_indices, x_indices = np.nonzero(input_mask_np)
|
||||
if len(y_indices) == 0:
|
||||
return state
|
||||
|
||||
|
||||
weights = input_mask_np[y_indices, x_indices]
|
||||
com_y = np.average(y_indices, weights=weights)
|
||||
com_x = np.average(x_indices, weights=weights)
|
||||
|
||||
|
||||
center_y = state["y"] * height
|
||||
center_x = state["x"] * width
|
||||
|
||||
|
||||
dir_y = center_y - com_y
|
||||
dir_x = center_x - com_x
|
||||
|
||||
|
||||
dist = np.sqrt(dir_x**2 + dir_y**2)
|
||||
if dist < 1:
|
||||
return state
|
||||
|
||||
scale = min(1.0, 20.0/dist)
|
||||
|
||||
scale = min(1.0, 20.0 / dist)
|
||||
dir_y = (dir_y / dist) * scale
|
||||
dir_x = (dir_x / dist) * scale
|
||||
|
||||
|
||||
# Flip direction if attracting
|
||||
if is_attract:
|
||||
dir_y *= -1
|
||||
dir_x *= -1
|
||||
|
||||
|
||||
new_y = state["y"] + (dir_y * strength)
|
||||
new_x = state["x"] + (dir_x * strength)
|
||||
|
||||
|
||||
state["y"] = np.clip(new_y, 0.0, 1.0)
|
||||
state["x"] = np.clip(new_x, 0.0, 1.0)
|
||||
|
||||
|
||||
return state
|
||||
|
||||
def update(self, input_mask, x_pos, y_pos, size, interaction_type, interaction_strength, always_execute=True):
|
||||
was_reset = len(self.state_manager._states) == 0
|
||||
|
||||
|
||||
if was_reset:
|
||||
state = self.get_initial_state(x_pos, y_pos, size)
|
||||
else:
|
||||
@@ -99,71 +103,81 @@ class RepulsiveMaskNode(ControlNodeBase, MaskControlMixin):
|
||||
|
||||
batch_size, height, width = input_mask.shape
|
||||
masks = np.zeros((batch_size, height, width), dtype=np.float32)
|
||||
|
||||
|
||||
for b in range(batch_size):
|
||||
center_y, center_x = int(state["y"] * height), int(state["x"] * width)
|
||||
mask = self.create_circle_mask(height, width, center_y, center_x, state["size"])
|
||||
|
||||
state = self.interact(mask, input_mask[b].cpu().numpy(), state,
|
||||
interaction_strength, height, width,
|
||||
interaction_type == "attract")
|
||||
|
||||
|
||||
state = self.interact(
|
||||
mask,
|
||||
input_mask[b].cpu().numpy(),
|
||||
state,
|
||||
interaction_strength,
|
||||
height,
|
||||
width,
|
||||
interaction_type == "attract",
|
||||
)
|
||||
|
||||
center_y, center_x = int(state["y"] * height), int(state["x"] * width)
|
||||
masks[b] = self.create_circle_mask(height, width, center_y, center_x, state["size"])
|
||||
|
||||
|
||||
self.set_state(state)
|
||||
return (torch.from_numpy(masks),)
|
||||
|
||||
|
||||
class ResizeMaskNode(ControlNodeBase, MaskControlMixin):
|
||||
"""Node that maintains a mask that resizes based on proximity to input masks"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
inputs = super().INPUT_TYPES()
|
||||
inputs["required"].update({
|
||||
"input_mask": ("MASK",),
|
||||
"x_pos": ("FLOAT", {
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Initial X position (0-1)"
|
||||
}),
|
||||
"y_pos": ("FLOAT", {
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Initial Y position (0-1)"
|
||||
}),
|
||||
"size": ("FLOAT", {
|
||||
"default": 0.2,
|
||||
"min": 0.01,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Size of the mask relative to image size"
|
||||
}),
|
||||
"min_size": ("FLOAT", {
|
||||
"default": 0.1,
|
||||
"min": 0.01,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Minimum size the mask can shrink to"
|
||||
}),
|
||||
"max_size": ("FLOAT", {
|
||||
"default": 0.4,
|
||||
"min": 0.01,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Maximum size the mask can grow to"
|
||||
}),
|
||||
"resize_strength": ("FLOAT", {
|
||||
"default": 0.1,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Strength of the resize effect"
|
||||
})
|
||||
})
|
||||
inputs["required"].update(
|
||||
{
|
||||
"input_mask": ("MASK",),
|
||||
"x_pos": (
|
||||
"FLOAT",
|
||||
{"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "Initial X position (0-1)"},
|
||||
),
|
||||
"y_pos": (
|
||||
"FLOAT",
|
||||
{"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "Initial Y position (0-1)"},
|
||||
),
|
||||
"size": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.2,
|
||||
"min": 0.01,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Size of the mask relative to image size",
|
||||
},
|
||||
),
|
||||
"min_size": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.1,
|
||||
"min": 0.01,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Minimum size the mask can shrink to",
|
||||
},
|
||||
),
|
||||
"max_size": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.4,
|
||||
"min": 0.01,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Maximum size the mask can grow to",
|
||||
},
|
||||
),
|
||||
"resize_strength": (
|
||||
"FLOAT",
|
||||
{"default": 0.1, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "Strength of the resize effect"},
|
||||
),
|
||||
}
|
||||
)
|
||||
return inputs
|
||||
|
||||
RETURN_TYPES = ("MASK",)
|
||||
@@ -180,13 +194,13 @@ class ResizeMaskNode(ControlNodeBase, MaskControlMixin):
|
||||
center_y = int(state["y"] * height)
|
||||
center_x = int(state["x"] * width)
|
||||
y_indices, x_indices = np.nonzero(input_mask_np > 0.1)
|
||||
|
||||
|
||||
if len(y_indices) == 0:
|
||||
return state
|
||||
|
||||
distances = np.sqrt((y_indices - center_y)**2 + (x_indices - center_x)**2)
|
||||
|
||||
distances = np.sqrt((y_indices - center_y) ** 2 + (x_indices - center_x) ** 2)
|
||||
min_dist = np.min(distances)
|
||||
|
||||
|
||||
# Map distance directly to size
|
||||
# When dist=0, size=min_size
|
||||
# When dist=100, size=max_size
|
||||
@@ -194,12 +208,12 @@ class ResizeMaskNode(ControlNodeBase, MaskControlMixin):
|
||||
size_range = state["max_size"] - state["min_size"]
|
||||
size = state["min_size"] + (min_dist / dist_range) * size_range
|
||||
state["size"] = np.clip(size, state["min_size"], state["max_size"])
|
||||
|
||||
|
||||
return state
|
||||
|
||||
def update(self, input_mask, x_pos, y_pos, size, min_size, max_size, resize_strength, always_execute=True):
|
||||
was_reset = len(self.state_manager._states) == 0
|
||||
|
||||
|
||||
if was_reset:
|
||||
state = self.get_initial_state(x_pos, y_pos, size, min_size, max_size)
|
||||
else:
|
||||
@@ -207,15 +221,15 @@ class ResizeMaskNode(ControlNodeBase, MaskControlMixin):
|
||||
|
||||
batch_size, height, width = input_mask.shape
|
||||
masks = np.zeros((batch_size, height, width), dtype=np.float32)
|
||||
|
||||
|
||||
for b in range(batch_size):
|
||||
center_y, center_x = int(state["y"] * height), int(state["x"] * width)
|
||||
mask = self.create_circle_mask(height, width, center_y, center_x, state["size"])
|
||||
|
||||
|
||||
state = self.resize(mask, input_mask[b].cpu().numpy(), state, resize_strength, height, width)
|
||||
|
||||
|
||||
center_y, center_x = int(state["y"] * height), int(state["x"] * width)
|
||||
masks[b] = self.create_circle_mask(height, width, center_y, center_x, state["size"])
|
||||
|
||||
|
||||
self.set_state(state)
|
||||
return (torch.from_numpy(masks),)
|
||||
return (torch.from_numpy(masks),)
|
||||
|
||||
@@ -1,41 +1,44 @@
|
||||
from ....src.realtimenodes.control_base import ControlNodeBase
|
||||
import time
|
||||
from enum import Enum
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
import torch
|
||||
import cv2
|
||||
import time
|
||||
|
||||
from ....src.realtimenodes.control_base import ControlNodeBase
|
||||
|
||||
|
||||
class ROIAction(Enum):
|
||||
# Behavioral actions
|
||||
TOGGLE = "toggle" # Toggles between min/max values
|
||||
TOGGLE = "toggle" # Toggles between min/max values
|
||||
MOMENTARY = "momentary" # Outputs max while motion detected
|
||||
TRIGGER = "trigger" # Triggers once per motion event
|
||||
COUNTER = "counter" # Counts motion events
|
||||
TRIGGER = "trigger" # Triggers once per motion event
|
||||
COUNTER = "counter" # Counts motion events
|
||||
# Mathematical actions
|
||||
ADD = "add" # Add value when motion detected
|
||||
SUBTRACT = "subtract" # Subtract value when motion detected
|
||||
MULTIPLY = "multiply" # Multiply by value when motion detected
|
||||
DIVIDE = "divide" # Divide by value when motion detected
|
||||
SET = "set" # Set to specific value when motion detected
|
||||
ADD = "add" # Add value when motion detected
|
||||
SUBTRACT = "subtract" # Subtract value when motion detected
|
||||
MULTIPLY = "multiply" # Multiply by value when motion detected
|
||||
DIVIDE = "divide" # Divide by value when motion detected
|
||||
SET = "set" # Set to specific value when motion detected
|
||||
|
||||
|
||||
class ROINode(ControlNodeBase):
|
||||
"""Defines a single region of interest and its action"""
|
||||
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"mask": ("MASK",), # ROI defined by mask
|
||||
"action": (list(action.value for action in ROIAction),),
|
||||
"value": ("FLOAT", {
|
||||
"default": 0.1,
|
||||
"step": 0.01,
|
||||
"tooltip": "Value to use for mathematical operations"
|
||||
}),
|
||||
"value": (
|
||||
"FLOAT",
|
||||
{"default": 0.1, "step": 0.01, "tooltip": "Value to use for mathematical operations"},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"next_roi": ("ROI",) # Chain to next ROI
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("ROI",)
|
||||
@@ -45,7 +48,7 @@ class ROINode(ControlNodeBase):
|
||||
def define_roi(self, mask, action, value, next_roi=None):
|
||||
# Convert mask to numpy for bounding box calculation
|
||||
mask_np = mask[0].cpu().numpy()
|
||||
|
||||
|
||||
# Find non-zero coordinates in mask
|
||||
coords = np.nonzero(mask_np)
|
||||
if len(coords[0]) == 0: # Empty mask
|
||||
@@ -53,78 +56,73 @@ class ROINode(ControlNodeBase):
|
||||
else:
|
||||
y_min, y_max = coords[0].min(), coords[0].max()
|
||||
x_min, x_max = coords[1].min(), coords[1].max()
|
||||
|
||||
|
||||
roi_data = {
|
||||
"mask": mask_np,
|
||||
"bounds": (y_min, x_min, y_max, x_max),
|
||||
"action": action,
|
||||
"value": value, # Store the operation value
|
||||
"next": next_roi
|
||||
"next": next_roi,
|
||||
}
|
||||
return (roi_data,)
|
||||
|
||||
def update(self, *args, **kwargs):
|
||||
return self.define_roi(*args, **kwargs)
|
||||
|
||||
|
||||
class MotionController(ControlNodeBase):
|
||||
"""Processes motion detection for a chain of ROIs and manages their states"""
|
||||
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
inputs = super().INPUT_TYPES()
|
||||
inputs["required"].update({
|
||||
"image": ("IMAGE",),
|
||||
"roi_chain": ("ROI",),
|
||||
"threshold": ("FLOAT", {
|
||||
"default": 0.1, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
"tooltip": "Motion detection threshold"
|
||||
}),
|
||||
"blur_size": ("INT", {
|
||||
"default": 5, "min": 1, "max": 21, "step": 2,
|
||||
"tooltip": "Size of Gaussian blur kernel"
|
||||
}),
|
||||
"minimum_value": ("FLOAT", {
|
||||
"default": 0.0,
|
||||
"tooltip": "Minimum output value"
|
||||
}),
|
||||
"maximum_value": ("FLOAT", {
|
||||
"default": 1.0,
|
||||
"tooltip": "Maximum output value"
|
||||
}),
|
||||
"starting_value": ("FLOAT", {
|
||||
"default": 0.0,
|
||||
"tooltip": "Initial output value"
|
||||
})
|
||||
})
|
||||
inputs["required"].update(
|
||||
{
|
||||
"image": ("IMAGE",),
|
||||
"roi_chain": ("ROI",),
|
||||
"threshold": (
|
||||
"FLOAT",
|
||||
{"default": 0.1, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "Motion detection threshold"},
|
||||
),
|
||||
"blur_size": (
|
||||
"INT",
|
||||
{"default": 5, "min": 1, "max": 21, "step": 2, "tooltip": "Size of Gaussian blur kernel"},
|
||||
),
|
||||
"minimum_value": ("FLOAT", {"default": 0.0, "tooltip": "Minimum output value"}),
|
||||
"maximum_value": ("FLOAT", {"default": 1.0, "tooltip": "Maximum output value"}),
|
||||
"starting_value": ("FLOAT", {"default": 0.0, "tooltip": "Initial output value"}),
|
||||
}
|
||||
)
|
||||
return inputs
|
||||
|
||||
RETURN_TYPES = ("FLOAT", "MASK")
|
||||
FUNCTION = "process_motion"
|
||||
CATEGORY = "Realtime Nodes/control/motion"
|
||||
|
||||
def process_motion(self, image, roi_chain, threshold, blur_size,
|
||||
minimum_value, maximum_value, starting_value, always_execute=True):
|
||||
def process_motion(self, image, roi_chain, threshold, blur_size, minimum_value, maximum_value, starting_value, always_execute=True):
|
||||
# Get or initialize state with new caching fields
|
||||
state = self.get_state({
|
||||
"prev_frame": None,
|
||||
"prev_frame_blurred": None, # Cache blurred previous frame
|
||||
"roi_states": {},
|
||||
"current_value": starting_value,
|
||||
"last_cleanup": time.time() # Track last state cleanup
|
||||
})
|
||||
|
||||
state = self.get_state(
|
||||
{
|
||||
"prev_frame": None,
|
||||
"prev_frame_blurred": None, # Cache blurred previous frame
|
||||
"roi_states": {},
|
||||
"current_value": starting_value,
|
||||
"last_cleanup": time.time(), # Track last state cleanup
|
||||
}
|
||||
)
|
||||
|
||||
# Convert image tensor to numpy array and prepare current frame
|
||||
current_frame = (image[0] * 255).cpu().numpy().astype(np.uint8)
|
||||
current_gray = cv2.cvtColor(current_frame, cv2.COLOR_RGB2GRAY)
|
||||
current_blurred = cv2.GaussianBlur(current_gray, (blur_size, blur_size), 0)
|
||||
|
||||
|
||||
# Initialize motion mask
|
||||
motion_mask = np.zeros_like(current_gray, dtype=np.float32)
|
||||
|
||||
|
||||
if state["prev_frame"] is not None:
|
||||
# Collect active ROIs for state cleanup
|
||||
active_rois = set()
|
||||
|
||||
|
||||
# Process each ROI in the chain
|
||||
current_roi = roi_chain
|
||||
while current_roi is not None:
|
||||
@@ -133,39 +131,36 @@ class MotionController(ControlNodeBase):
|
||||
action = current_roi["action"]
|
||||
roi_id = str(bounds)
|
||||
active_rois.add(roi_id)
|
||||
|
||||
|
||||
# Extract ROI regions once
|
||||
y_min, x_min, y_max, x_max = bounds
|
||||
roi_region = slice(y_min, y_max+1), slice(x_min, x_max+1)
|
||||
|
||||
roi_region = slice(y_min, y_max + 1), slice(x_min, x_max + 1)
|
||||
|
||||
# Process ROI using cached blurred frames
|
||||
roi_current = current_blurred[roi_region]
|
||||
roi_prev = state["prev_frame_blurred"][roi_region]
|
||||
roi_mask = mask[roi_region]
|
||||
|
||||
|
||||
# Detect motion in ROI
|
||||
diff = cv2.absdiff(roi_current, roi_prev)
|
||||
_, thresh = cv2.threshold(diff, threshold * 255, 255, cv2.THRESH_BINARY)
|
||||
thresh = thresh * (roi_mask > 0.5)
|
||||
|
||||
|
||||
# Update motion mask
|
||||
motion_mask[roi_region] = np.maximum(
|
||||
motion_mask[roi_region],
|
||||
thresh / 255.0
|
||||
)
|
||||
|
||||
motion_mask[roi_region] = np.maximum(motion_mask[roi_region], thresh / 255.0)
|
||||
|
||||
# Calculate motion magnitude
|
||||
roi_area = np.sum(roi_mask > 0.5)
|
||||
motion_detected = np.sum(thresh) / (255 * roi_area) > threshold if roi_area > 0 else False
|
||||
|
||||
|
||||
# Get or initialize ROI state
|
||||
roi_state = state["roi_states"].get(roi_id, {"active": False, "count": 0})
|
||||
value = current_roi["value"]
|
||||
|
||||
|
||||
# Process actions with optimized value updates
|
||||
if motion_detected and not roi_state["active"]:
|
||||
roi_state["active"] = True
|
||||
|
||||
|
||||
# Update current value based on action type
|
||||
if action == ROIAction.ADD.value:
|
||||
state["current_value"] = min(maximum_value, state["current_value"] + value)
|
||||
@@ -184,69 +179,59 @@ class MotionController(ControlNodeBase):
|
||||
state["current_value"] = maximum_value
|
||||
elif action == ROIAction.COUNTER.value:
|
||||
roi_state["count"] += 1
|
||||
state["current_value"] = minimum_value + (
|
||||
roi_state["count"] % (int((maximum_value - minimum_value) + 1))
|
||||
)
|
||||
|
||||
state["current_value"] = minimum_value + (roi_state["count"] % (int((maximum_value - minimum_value) + 1)))
|
||||
|
||||
elif action == ROIAction.MOMENTARY.value:
|
||||
state["current_value"] = maximum_value if motion_detected else minimum_value
|
||||
|
||||
|
||||
elif not motion_detected:
|
||||
roi_state["active"] = False
|
||||
if action == ROIAction.TRIGGER.value:
|
||||
state["current_value"] = minimum_value
|
||||
|
||||
|
||||
state["roi_states"][roi_id] = roi_state
|
||||
current_roi = current_roi["next"]
|
||||
|
||||
|
||||
# Cleanup inactive ROI states periodically (every 60 seconds)
|
||||
if time.time() - state["last_cleanup"] > 60:
|
||||
state["roi_states"] = {k: v for k, v in state["roi_states"].items()
|
||||
if k in active_rois}
|
||||
state["roi_states"] = {k: v for k, v in state["roi_states"].items() if k in active_rois}
|
||||
state["last_cleanup"] = time.time()
|
||||
|
||||
|
||||
# Update state with current frame data
|
||||
state["prev_frame"] = current_gray
|
||||
state["prev_frame_blurred"] = current_blurred
|
||||
self.set_state(state)
|
||||
|
||||
|
||||
# Convert mask to tensor
|
||||
mask_tensor = torch.from_numpy(motion_mask).unsqueeze(0)
|
||||
|
||||
return (state["current_value"], mask_tensor)
|
||||
|
||||
return (state["current_value"], mask_tensor)
|
||||
|
||||
def update(self, *args, **kwargs):
|
||||
return self.process_motion(*args, **kwargs)
|
||||
return self.process_motion(*args, **kwargs)
|
||||
|
||||
|
||||
class RoundingMode(Enum):
|
||||
ROUND = "round" # Standard rounding
|
||||
FLOOR = "floor" # Round down
|
||||
CEIL = "ceil" # Round up
|
||||
ROUND = "round" # Standard rounding
|
||||
FLOOR = "floor" # Round down
|
||||
CEIL = "ceil" # Round up
|
||||
TRUNCATE = "truncate" # Remove decimal part
|
||||
|
||||
|
||||
class IntegerMotionController(MotionController):
|
||||
"""Processes motion detection for ROIs and outputs integer values"""
|
||||
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
inputs = super().INPUT_TYPES()
|
||||
# Convert float parameters to int
|
||||
inputs["required"]["minimum_value"] = ("INT", {
|
||||
"default": 0,
|
||||
"tooltip": "Minimum output value"
|
||||
})
|
||||
inputs["required"]["maximum_value"] = ("INT", {
|
||||
"default": 100,
|
||||
"tooltip": "Maximum output value"
|
||||
})
|
||||
inputs["required"]["starting_value"] = ("INT", {
|
||||
"default": 0,
|
||||
"tooltip": "Initial output value"
|
||||
})
|
||||
inputs["required"]["minimum_value"] = ("INT", {"default": 0, "tooltip": "Minimum output value"})
|
||||
inputs["required"]["maximum_value"] = ("INT", {"default": 100, "tooltip": "Maximum output value"})
|
||||
inputs["required"]["starting_value"] = ("INT", {"default": 0, "tooltip": "Initial output value"})
|
||||
# Add rounding mode parameter
|
||||
inputs["required"]["rounding_mode"] = (
|
||||
[mode.value for mode in RoundingMode],
|
||||
{"default": RoundingMode.ROUND.value}
|
||||
{"default": RoundingMode.ROUND.value},
|
||||
)
|
||||
return inputs
|
||||
|
||||
@@ -265,20 +250,34 @@ class IntegerMotionController(MotionController):
|
||||
else: # TRUNCATE
|
||||
return int(value)
|
||||
|
||||
def process_motion(self, image, roi_chain, threshold, blur_size,
|
||||
minimum_value, maximum_value, starting_value,
|
||||
rounding_mode, always_execute=True):
|
||||
def process_motion(
|
||||
self,
|
||||
image,
|
||||
roi_chain,
|
||||
threshold,
|
||||
blur_size,
|
||||
minimum_value,
|
||||
maximum_value,
|
||||
starting_value,
|
||||
rounding_mode,
|
||||
always_execute=True,
|
||||
):
|
||||
# Call parent class method with float conversion of bounds
|
||||
float_result, mask = super().process_motion(
|
||||
image, roi_chain, threshold, blur_size,
|
||||
float(minimum_value), float(maximum_value), float(starting_value),
|
||||
always_execute
|
||||
image,
|
||||
roi_chain,
|
||||
threshold,
|
||||
blur_size,
|
||||
float(minimum_value),
|
||||
float(maximum_value),
|
||||
float(starting_value),
|
||||
always_execute,
|
||||
)
|
||||
|
||||
|
||||
# Convert result to integer using specified rounding mode
|
||||
int_result = self._apply_rounding(float_result, rounding_mode)
|
||||
|
||||
|
||||
# Ensure result stays within bounds
|
||||
int_result = max(minimum_value, min(maximum_value, int_result))
|
||||
|
||||
return (int_result, mask)
|
||||
|
||||
return (int_result, mask)
|
||||
|
||||
@@ -1,44 +1,46 @@
|
||||
from ....src.realtimenodes.control_base import ControlNodeBase
|
||||
|
||||
|
||||
class SequenceControlBase(ControlNodeBase):
|
||||
"""Base class for sequence-based controls"""
|
||||
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
inputs = super().INPUT_TYPES()
|
||||
inputs["required"].update({
|
||||
"steps_per_item": ("INT", {
|
||||
"default": 30,
|
||||
"min": 1,
|
||||
"max": 1000,
|
||||
"step": 1,
|
||||
"tooltip": "Number of frames to show each item"
|
||||
}),
|
||||
"sequence_mode": (["forward", "reverse", "pingpong", "random"], {
|
||||
"default": "forward",
|
||||
"tooltip": "How to move through the sequence"
|
||||
})
|
||||
})
|
||||
inputs["required"].update(
|
||||
{
|
||||
"steps_per_item": (
|
||||
"INT",
|
||||
{"default": 30, "min": 1, "max": 1000, "step": 1, "tooltip": "Number of frames to show each item"},
|
||||
),
|
||||
"sequence_mode": (
|
||||
["forward", "reverse", "pingpong", "random"],
|
||||
{"default": "forward", "tooltip": "How to move through the sequence"},
|
||||
),
|
||||
}
|
||||
)
|
||||
return inputs
|
||||
|
||||
def update_sequence_base(self, values: list, steps_per_item: int, sequence_mode: str, always_execute: bool = True):
|
||||
# Initialize or get state
|
||||
state = self.get_state({
|
||||
"current_index": 0,
|
||||
"step_counter": 0,
|
||||
"direction": 1 # 1 for forward, -1 for reverse (used in pingpong)
|
||||
})
|
||||
|
||||
state = self.get_state(
|
||||
{
|
||||
"current_index": 0,
|
||||
"step_counter": 0,
|
||||
"direction": 1, # 1 for forward, -1 for reverse (used in pingpong)
|
||||
}
|
||||
)
|
||||
|
||||
if not values:
|
||||
return (values[0] if values else None,)
|
||||
|
||||
|
||||
# Update step counter
|
||||
state["step_counter"] += 1
|
||||
|
||||
|
||||
# Check if we should move to next item
|
||||
if state["step_counter"] >= steps_per_item:
|
||||
state["step_counter"] = 0
|
||||
|
||||
|
||||
# Update index based on sequence mode
|
||||
if sequence_mode == "forward":
|
||||
state["current_index"] = (state["current_index"] + 1) % len(values)
|
||||
@@ -52,17 +54,19 @@ class SequenceControlBase(ControlNodeBase):
|
||||
state["current_index"] = next_index
|
||||
elif sequence_mode == "random":
|
||||
import random
|
||||
|
||||
state["current_index"] = random.randint(0, len(values) - 1)
|
||||
|
||||
|
||||
# Save state
|
||||
self.set_state(state)
|
||||
|
||||
|
||||
# Return current value
|
||||
return (values[state["current_index"]],)
|
||||
|
||||
|
||||
class FloatSequence(SequenceControlBase):
|
||||
"""Cycles through a sequence of float values"""
|
||||
|
||||
|
||||
RETURN_TYPES = ("FLOAT",)
|
||||
FUNCTION = "update"
|
||||
CATEGORY = "Realtime Nodes/control/sequence"
|
||||
@@ -70,22 +74,18 @@ class FloatSequence(SequenceControlBase):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
inputs = super().INPUT_TYPES()
|
||||
inputs["required"].update({
|
||||
"values": ("STRING", {
|
||||
"default": "0.0, 0.5, 1.0",
|
||||
"tooltip": "List of float values (comma separated)"
|
||||
})
|
||||
})
|
||||
inputs["required"].update({"values": ("STRING", {"default": "0.0, 0.5, 1.0", "tooltip": "List of float values (comma separated)"})})
|
||||
return inputs
|
||||
|
||||
|
||||
def update(self, values: str, steps_per_item: int, sequence_mode: str, always_execute: bool = True):
|
||||
# Parse comma-separated float values
|
||||
float_values = [float(v.strip()) for v in values.split(',') if v.strip()]
|
||||
float_values = [float(v.strip()) for v in values.split(",") if v.strip()]
|
||||
return self.update_sequence_base(float_values, steps_per_item, sequence_mode, always_execute)
|
||||
|
||||
|
||||
class IntSequence(SequenceControlBase):
|
||||
"""Cycles through a sequence of integer values"""
|
||||
|
||||
|
||||
RETURN_TYPES = ("INT",)
|
||||
FUNCTION = "update"
|
||||
CATEGORY = "Realtime Nodes/control/sequence"
|
||||
@@ -93,22 +93,18 @@ class IntSequence(SequenceControlBase):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
inputs = super().INPUT_TYPES()
|
||||
inputs["required"].update({
|
||||
"values": ("STRING", {
|
||||
"default": "0, 5, 10",
|
||||
"tooltip": "List of integer values (comma separated)"
|
||||
})
|
||||
})
|
||||
inputs["required"].update({"values": ("STRING", {"default": "0, 5, 10", "tooltip": "List of integer values (comma separated)"})})
|
||||
return inputs
|
||||
|
||||
|
||||
def update(self, values: str, steps_per_item: int, sequence_mode: str, always_execute: bool = True):
|
||||
# Parse comma-separated integer values
|
||||
int_values = [int(v.strip()) for v in values.split(',') if v.strip()]
|
||||
int_values = [int(v.strip()) for v in values.split(",") if v.strip()]
|
||||
return self.update_sequence_base(int_values, steps_per_item, sequence_mode, always_execute)
|
||||
|
||||
|
||||
class StringSequence(SequenceControlBase):
|
||||
"""Cycles through a sequence of strings"""
|
||||
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
FUNCTION = "update"
|
||||
CATEGORY = "Realtime Nodes/control/sequence"
|
||||
@@ -116,16 +112,17 @@ class StringSequence(SequenceControlBase):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
inputs = super().INPUT_TYPES()
|
||||
inputs["required"].update({
|
||||
"values": ("STRING", {
|
||||
"multiline": True,
|
||||
"default": "first\nsecond\nthird",
|
||||
"tooltip": "List of strings (one per line)"
|
||||
})
|
||||
})
|
||||
inputs["required"].update(
|
||||
{
|
||||
"values": (
|
||||
"STRING",
|
||||
{"multiline": True, "default": "first\nsecond\nthird", "tooltip": "List of strings (one per line)"},
|
||||
)
|
||||
}
|
||||
)
|
||||
return inputs
|
||||
|
||||
|
||||
def update(self, values: str, steps_per_item: int, sequence_mode: str, always_execute: bool = True):
|
||||
# Split into string values
|
||||
string_values = [v.strip() for v in values.split('\n') if v.strip()]
|
||||
return self.update_sequence_base(string_values, steps_per_item, sequence_mode, always_execute)
|
||||
string_values = [v.strip() for v in values.split("\n") if v.strip()]
|
||||
return self.update_sequence_base(string_values, steps_per_item, sequence_mode, always_execute)
|
||||
|
||||
@@ -1,21 +1,18 @@
|
||||
from ....src.realtimenodes.control_base import ControlNodeBase
|
||||
import copy
|
||||
|
||||
from ....src.realtimenodes.control_base import ControlNodeBase
|
||||
from ....src.utils.general import AlwaysEqualProxy
|
||||
|
||||
|
||||
class StateResetNode(ControlNodeBase):
|
||||
"""Node that resets all control node states when triggered"""
|
||||
|
||||
|
||||
OUTPUT_NODE = True
|
||||
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
inputs = super().INPUT_TYPES()
|
||||
inputs["required"].update({
|
||||
"trigger": ("BOOLEAN", {
|
||||
"default": False,
|
||||
"tooltip": "Set to True to reset all states"
|
||||
})
|
||||
})
|
||||
inputs["required"].update({"trigger": ("BOOLEAN", {"default": False, "tooltip": "Set to True to reset all states"})})
|
||||
return inputs
|
||||
|
||||
RETURN_TYPES = ("BOOLEAN",)
|
||||
@@ -28,21 +25,21 @@ class StateResetNode(ControlNodeBase):
|
||||
return (True,)
|
||||
return (False,)
|
||||
|
||||
|
||||
class StateTestNode(ControlNodeBase):
|
||||
"""Simple node that maintains a counter to test state management"""
|
||||
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
inputs = super().INPUT_TYPES()
|
||||
inputs["required"].update({
|
||||
"increment": ("INT", {
|
||||
"default": 1,
|
||||
"min": 1,
|
||||
"max": 100,
|
||||
"step": 1,
|
||||
"tooltip": "Amount to increment counter by"
|
||||
})
|
||||
})
|
||||
inputs["required"].update(
|
||||
{
|
||||
"increment": (
|
||||
"INT",
|
||||
{"default": 1, "min": 1, "max": 100, "step": 1, "tooltip": "Amount to increment counter by"},
|
||||
)
|
||||
}
|
||||
)
|
||||
return inputs
|
||||
|
||||
RETURN_TYPES = ("INT",)
|
||||
@@ -50,87 +47,112 @@ class StateTestNode(ControlNodeBase):
|
||||
CATEGORY = "Realtime Nodes/control/utility"
|
||||
|
||||
def update(self, increment, always_execute=True):
|
||||
state = self.get_state({
|
||||
"counter": 0
|
||||
})
|
||||
|
||||
state = self.get_state({"counter": 0})
|
||||
|
||||
state["counter"] += increment
|
||||
|
||||
self.set_state(state)
|
||||
|
||||
return (state["counter"],)
|
||||
|
||||
return (state["counter"],)
|
||||
|
||||
|
||||
class GetStateNode(ControlNodeBase):
|
||||
"""
|
||||
Node that retrieves a value from the global state using a user-specified key.
|
||||
"""
|
||||
|
||||
CATEGORY = "utils"
|
||||
RETURN_TYPES = (AlwaysEqualProxy("*"),)
|
||||
RETURN_NAMES = ("value",)
|
||||
FUNCTION = "update"
|
||||
DESCRIPTION = "Retrieve a value from the global state using the given key. If the key is not found, return the default value."
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
inputs = super().INPUT_TYPES()
|
||||
inputs["required"].update({
|
||||
"key": ("STRING", {"default": "default_key", "tooltip": "The key to retrieve the value from. If not provided, the default value will be returned."}),
|
||||
"default_value": (AlwaysEqualProxy("*"), {"tooltip": "The value to return if the key is not found."}),
|
||||
"use_default": ("BOOLEAN", {"default": False, "tooltip": "If True, the default value will be returned if the key is not found."}),
|
||||
})
|
||||
inputs["required"].update(
|
||||
{
|
||||
"key": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "default_key",
|
||||
"tooltip": "The key to retrieve the value from. If not provided, the default value will be returned.",
|
||||
},
|
||||
),
|
||||
"default_value": (AlwaysEqualProxy("*"), {"tooltip": "The value to return if the key is not found."}),
|
||||
"use_default": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "If True, the default value will be returned if the key is not found.",
|
||||
},
|
||||
),
|
||||
}
|
||||
)
|
||||
return inputs
|
||||
|
||||
|
||||
def update(self, key: str, default_value, use_default: bool, always_execute=True):
|
||||
"""
|
||||
Retrieve a value from the global state using the given key.
|
||||
"""
|
||||
if not key or use_default:
|
||||
return (default_value,)
|
||||
|
||||
|
||||
# Get the shared state dictionary
|
||||
shared_state = self.state_manager.get_state("__shared_keys__", {})
|
||||
|
||||
|
||||
# Check if the key exists
|
||||
if key in shared_state:
|
||||
return (shared_state[key],)
|
||||
|
||||
|
||||
# Return default value if key not found
|
||||
return (default_value,)
|
||||
|
||||
|
||||
|
||||
class SetStateNode(ControlNodeBase):
|
||||
"""
|
||||
Node that stores a value in the global state with a user-specified key.
|
||||
The value will be accessible in future workflow runs through GetStateNode.
|
||||
"""
|
||||
|
||||
CATEGORY = "utils"
|
||||
RETURN_TYPES = (AlwaysEqualProxy("*"),)
|
||||
RETURN_NAMES = ("value",)
|
||||
FUNCTION = "update"
|
||||
OUTPUT_NODE = True
|
||||
DESCRIPTION = "Store a value in the global state with the given key. The value will be accessible in future workflow runs through GetStateNode."
|
||||
DESCRIPTION = (
|
||||
"Store a value in the global state with the given key. The value will be accessible in future workflow runs through GetStateNode."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
inputs = super().INPUT_TYPES()
|
||||
inputs["required"].update({
|
||||
"key": ("STRING", {"default": "default_key", "tooltip": "The key to store the value under. If not provided, the value will not be stored."}),
|
||||
"value": (AlwaysEqualProxy("*"), {"tooltip": "The value to store in the global state."}),
|
||||
})
|
||||
inputs["required"].update(
|
||||
{
|
||||
"key": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "default_key",
|
||||
"tooltip": "The key to store the value under. If not provided, the value will not be stored.",
|
||||
},
|
||||
),
|
||||
"value": (AlwaysEqualProxy("*"), {"tooltip": "The value to store in the global state."}),
|
||||
}
|
||||
)
|
||||
return inputs
|
||||
|
||||
|
||||
def update(self, key: str, value, always_execute=True):
|
||||
"""
|
||||
Store a value in the global state with the given key.
|
||||
"""
|
||||
if not key:
|
||||
return (value,)
|
||||
|
||||
|
||||
try:
|
||||
shared_state = self.state_manager.get_state("__shared_keys__", {})
|
||||
shared_state[key] = copy.deepcopy(value)
|
||||
self.state_manager.set_state("__shared_keys__", shared_state)
|
||||
except Exception as e:
|
||||
print(f"[State Node] Error storing value: {str(e)}")
|
||||
|
||||
return (value,)
|
||||
|
||||
return (value,)
|
||||
|
||||
@@ -1,25 +1,31 @@
|
||||
from ....src.realtimenodes.control_base import ControlNodeBase
|
||||
from ....src.realtimenodes.patterns.movement_patterns import MOVEMENT_PATTERNS
|
||||
|
||||
|
||||
class ValueControlBase(ControlNodeBase):
|
||||
"""Base class for float and integer control nodes"""
|
||||
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
inputs = super().INPUT_TYPES()
|
||||
inputs["required"].update({
|
||||
"steps_per_cycle": ("INT", {
|
||||
"default": 30,
|
||||
"min": 1,
|
||||
"max": 1000,
|
||||
"step": 1,
|
||||
"tooltip": "Number of steps to complete one full cycle"
|
||||
}),
|
||||
"movement_type": (list(MOVEMENT_PATTERNS.keys()), {
|
||||
"default": "sine",
|
||||
"tooltip": "Pattern of value changes over time"
|
||||
}),
|
||||
})
|
||||
inputs["required"].update(
|
||||
{
|
||||
"steps_per_cycle": (
|
||||
"INT",
|
||||
{
|
||||
"default": 30,
|
||||
"min": 1,
|
||||
"max": 1000,
|
||||
"step": 1,
|
||||
"tooltip": "Number of steps to complete one full cycle",
|
||||
},
|
||||
),
|
||||
"movement_type": (
|
||||
list(MOVEMENT_PATTERNS.keys()),
|
||||
{"default": "sine", "tooltip": "Pattern of value changes over time"},
|
||||
),
|
||||
}
|
||||
)
|
||||
return inputs
|
||||
|
||||
def update(self, *args, **kwargs):
|
||||
@@ -27,11 +33,8 @@ class ValueControlBase(ControlNodeBase):
|
||||
return self.update_value_base(*args, **kwargs)
|
||||
|
||||
def update_value_base(self, maximum_value, minimum_value, starting_value, steps_per_cycle, movement_type, always_execute=True):
|
||||
state = self.get_state({
|
||||
"current_value": None,
|
||||
"phase": 0.0
|
||||
})
|
||||
|
||||
state = self.get_state({"current_value": None, "phase": 0.0})
|
||||
|
||||
# Initialize if this is the first run
|
||||
if state["current_value"] is None:
|
||||
state["current_value"] = starting_value
|
||||
@@ -39,147 +42,175 @@ class ValueControlBase(ControlNodeBase):
|
||||
return (starting_value,)
|
||||
|
||||
# Update phase
|
||||
state["phase"] = (state["phase"] + 1/steps_per_cycle) % 1.0
|
||||
|
||||
state["phase"] = (state["phase"] + 1 / steps_per_cycle) % 1.0
|
||||
|
||||
# Calculate new value using pattern
|
||||
pattern = MOVEMENT_PATTERNS[movement_type]
|
||||
new_value = pattern.calculate(state["phase"], minimum_value, maximum_value)
|
||||
|
||||
|
||||
# Update state
|
||||
state["current_value"] = new_value
|
||||
self.set_state(state)
|
||||
|
||||
|
||||
return (new_value,)
|
||||
|
||||
|
||||
class FloatControl(ValueControlBase):
|
||||
|
||||
DESCRIPTION = "Generates a floating point value that changes over time according to various patterns."
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
inputs = super().INPUT_TYPES()
|
||||
inputs["required"].update({
|
||||
"maximum_value": ("FLOAT", {
|
||||
"default": 1.0,
|
||||
"min": -10000.0,
|
||||
"max": 10000.0,
|
||||
"step": 0.1,
|
||||
"tooltip": "Maximum value that can be output"
|
||||
}),
|
||||
"minimum_value": ("FLOAT", {
|
||||
"default": 0.0,
|
||||
"min": -10000.0,
|
||||
"max": 10000.0,
|
||||
"step": 0.1,
|
||||
"tooltip": "Minimum value that can be output"
|
||||
}),
|
||||
"starting_value": ("FLOAT", {
|
||||
"default": 0.5,
|
||||
"min": -10000.0,
|
||||
"max": 10000.0,
|
||||
"step": 0.1,
|
||||
"tooltip": "Initial value when the node first executes"
|
||||
}),
|
||||
})
|
||||
inputs["required"].update(
|
||||
{
|
||||
"maximum_value": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 1.0,
|
||||
"min": -10000.0,
|
||||
"max": 10000.0,
|
||||
"step": 0.1,
|
||||
"tooltip": "Maximum value that can be output",
|
||||
},
|
||||
),
|
||||
"minimum_value": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.0,
|
||||
"min": -10000.0,
|
||||
"max": 10000.0,
|
||||
"step": 0.1,
|
||||
"tooltip": "Minimum value that can be output",
|
||||
},
|
||||
),
|
||||
"starting_value": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.5,
|
||||
"min": -10000.0,
|
||||
"max": 10000.0,
|
||||
"step": 0.1,
|
||||
"tooltip": "Initial value when the node first executes",
|
||||
},
|
||||
),
|
||||
}
|
||||
)
|
||||
return inputs
|
||||
|
||||
RETURN_TYPES = ("FLOAT",)
|
||||
FUNCTION = "update_value"
|
||||
CATEGORY = "Realtime Nodes/control/value"
|
||||
|
||||
|
||||
def update_value(self, maximum_value, minimum_value, starting_value, steps_per_cycle, movement_type, always_execute=True):
|
||||
return self.update_value_base(maximum_value, minimum_value, starting_value, steps_per_cycle, movement_type, always_execute)
|
||||
|
||||
class IntControl(ValueControlBase):
|
||||
|
||||
|
||||
class IntControl(ValueControlBase):
|
||||
DESCRIPTION = "Generates an integer value that changes over time according to various patterns."
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
inputs = super().INPUT_TYPES()
|
||||
inputs["required"].update({
|
||||
"maximum_value": ("INT", {
|
||||
"default": 100,
|
||||
"min": -10000,
|
||||
"max": 10000,
|
||||
"step": 1,
|
||||
"tooltip": "Maximum value that can be output"
|
||||
}),
|
||||
"minimum_value": ("INT", {
|
||||
"default": 0,
|
||||
"min": -10000,
|
||||
"max": 10000,
|
||||
"step": 1,
|
||||
"tooltip": "Minimum value that can be output"
|
||||
}),
|
||||
"starting_value": ("INT", {
|
||||
"default": 50,
|
||||
"min": -10000,
|
||||
"max": 10000,
|
||||
"step": 1,
|
||||
"tooltip": "Initial value when the node first executes"
|
||||
}),
|
||||
})
|
||||
inputs["required"].update(
|
||||
{
|
||||
"maximum_value": (
|
||||
"INT",
|
||||
{
|
||||
"default": 100,
|
||||
"min": -10000,
|
||||
"max": 10000,
|
||||
"step": 1,
|
||||
"tooltip": "Maximum value that can be output",
|
||||
},
|
||||
),
|
||||
"minimum_value": (
|
||||
"INT",
|
||||
{
|
||||
"default": 0,
|
||||
"min": -10000,
|
||||
"max": 10000,
|
||||
"step": 1,
|
||||
"tooltip": "Minimum value that can be output",
|
||||
},
|
||||
),
|
||||
"starting_value": (
|
||||
"INT",
|
||||
{
|
||||
"default": 50,
|
||||
"min": -10000,
|
||||
"max": 10000,
|
||||
"step": 1,
|
||||
"tooltip": "Initial value when the node first executes",
|
||||
},
|
||||
),
|
||||
}
|
||||
)
|
||||
return inputs
|
||||
|
||||
RETURN_TYPES = ("INT",)
|
||||
FUNCTION = "update_value"
|
||||
CATEGORY = "Realtime Nodes/control/value"
|
||||
|
||||
|
||||
def update_value(self, maximum_value, minimum_value, starting_value, steps_per_cycle, movement_type, always_execute=True):
|
||||
result = self.update_value_base(maximum_value, minimum_value, starting_value, steps_per_cycle, movement_type, always_execute)
|
||||
return (int(round(result[0])),)
|
||||
|
||||
|
||||
class StringControl(ControlNodeBase):
|
||||
|
||||
DESCRIPTION = "Generates string outputs that change over time according to various patterns."
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
inputs = super().INPUT_TYPES()
|
||||
inputs["required"].update({
|
||||
"strings": ("STRING", {
|
||||
"multiline": True,
|
||||
"default": "first string\nsecond string\nthird string",
|
||||
"tooltip": "List of strings to cycle through (one per line)"
|
||||
}),
|
||||
"steps_per_cycle": ("INT", {
|
||||
"default": 30,
|
||||
"min": 1,
|
||||
"max": 1000,
|
||||
"step": 1,
|
||||
"tooltip": "Number of steps to complete one full cycle"
|
||||
}),
|
||||
"movement_type": (list(MOVEMENT_PATTERNS.keys()), {
|
||||
"default": "sine",
|
||||
"tooltip": "Pattern of value changes over time"
|
||||
}),
|
||||
})
|
||||
inputs["required"].update(
|
||||
{
|
||||
"strings": (
|
||||
"STRING",
|
||||
{
|
||||
"multiline": True,
|
||||
"default": "first string\nsecond string\nthird string",
|
||||
"tooltip": "List of strings to cycle through (one per line)",
|
||||
},
|
||||
),
|
||||
"steps_per_cycle": (
|
||||
"INT",
|
||||
{
|
||||
"default": 30,
|
||||
"min": 1,
|
||||
"max": 1000,
|
||||
"step": 1,
|
||||
"tooltip": "Number of steps to complete one full cycle",
|
||||
},
|
||||
),
|
||||
"movement_type": (
|
||||
list(MOVEMENT_PATTERNS.keys()),
|
||||
{"default": "sine", "tooltip": "Pattern of value changes over time"},
|
||||
),
|
||||
}
|
||||
)
|
||||
return inputs
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
FUNCTION = "update"
|
||||
CATEGORY = "Realtime Nodes/control/value"
|
||||
|
||||
|
||||
def update(self, strings, steps_per_cycle, movement_type, always_execute=True):
|
||||
# Split the input strings into a list
|
||||
string_list = [s.strip() for s in strings.split('\n') if s.strip()]
|
||||
string_list = [s.strip() for s in strings.split("\n") if s.strip()]
|
||||
if not string_list:
|
||||
return ("",)
|
||||
|
||||
state = self.get_state({
|
||||
"phase": 0.0
|
||||
})
|
||||
|
||||
|
||||
state = self.get_state({"phase": 0.0})
|
||||
|
||||
# Update phase
|
||||
state["phase"] = (state["phase"] + 1/steps_per_cycle) % 1.0
|
||||
|
||||
state["phase"] = (state["phase"] + 1 / steps_per_cycle) % 1.0
|
||||
|
||||
# Calculate index using pattern
|
||||
pattern = MOVEMENT_PATTERNS[movement_type]
|
||||
index_float = pattern.calculate(state["phase"], 0, len(string_list) - 1)
|
||||
index = int(round(index_float)) % len(string_list)
|
||||
|
||||
|
||||
# Update state
|
||||
self.set_state(state)
|
||||
|
||||
return (string_list[index],)
|
||||
|
||||
return (string_list[index],)
|
||||
|
||||
@@ -1,31 +1,41 @@
|
||||
from ....src.realtimenodes.control_base import ControlNodeBase
|
||||
from ....src.utils.general import AlwaysEqualProxy
|
||||
|
||||
|
||||
class LazyCondition(ControlNodeBase):
|
||||
DESCRIPTION = "Uses lazy evaluation to truly skip execution of unused paths. Maintains state of the last value to circumvent feedback loops."
|
||||
|
||||
DESCRIPTION = (
|
||||
"Uses lazy evaluation to truly skip execution of unused paths. Maintains state of the last value to circumvent feedback loops."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {
|
||||
"required": {
|
||||
"condition": (AlwaysEqualProxy("*"), {
|
||||
"tooltip": "When truthy (non-zero, non-empty, non-None), evaluates and returns if_true path. When falsy, returns either fallback or previous state of if_true.",
|
||||
"forceInput": True,
|
||||
}),
|
||||
"if_true": (AlwaysEqualProxy("*"), {
|
||||
"lazy": True,
|
||||
"tooltip": "The path that should only be evaluated when condition is truthy"
|
||||
}),
|
||||
"fallback": (AlwaysEqualProxy("*"), {
|
||||
"tooltip": "Alternative value to use when condition is falsy or no previous state of if_true"
|
||||
}),
|
||||
"use_fallback": ("BOOLEAN", {
|
||||
"default": False,
|
||||
"tooltip": "When False, uses last successful if_true result (if one exists). When True, uses fallback value"
|
||||
}),
|
||||
"condition": (
|
||||
AlwaysEqualProxy("*"),
|
||||
{
|
||||
"tooltip": "When truthy (non-zero, non-empty, non-None), evaluates and returns if_true path. When falsy, returns either fallback or previous state of if_true.",
|
||||
"forceInput": True,
|
||||
},
|
||||
),
|
||||
"if_true": (
|
||||
AlwaysEqualProxy("*"),
|
||||
{"lazy": True, "tooltip": "The path that should only be evaluated when condition is truthy"},
|
||||
),
|
||||
"fallback": (
|
||||
AlwaysEqualProxy("*"),
|
||||
{"tooltip": "Alternative value to use when condition is falsy or no previous state of if_true"},
|
||||
),
|
||||
"use_fallback": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "When False, uses last successful if_true result (if one exists). When True, uses fallback value",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
RETURN_TYPES = (AlwaysEqualProxy("*"),)
|
||||
FUNCTION = "update"
|
||||
CATEGORY = "Realtime Nodes/control/utility"
|
||||
@@ -39,14 +49,12 @@ class LazyCondition(ControlNodeBase):
|
||||
|
||||
def update(self, condition, if_true, fallback, use_fallback):
|
||||
"""Route to either if_true output or fallback value, maintaining state of last if_true."""
|
||||
state = self.get_state({
|
||||
"prev_output": None
|
||||
})
|
||||
state = self.get_state({"prev_output": None})
|
||||
|
||||
if condition: # Let Python handle truthiness
|
||||
# Update last state when we run if_true path
|
||||
state["prev_output"] = if_true if if_true is not None else fallback
|
||||
if hasattr(if_true, 'detach'):
|
||||
if hasattr(if_true, "detach"):
|
||||
state["prev_output"] = if_true.detach().clone()
|
||||
self.set_state(state)
|
||||
return (if_true,)
|
||||
@@ -56,32 +64,42 @@ class LazyCondition(ControlNodeBase):
|
||||
else:
|
||||
return (state["prev_output"],)
|
||||
|
||||
|
||||
class LogicOperator(ControlNodeBase):
|
||||
DESCRIPTION = "Performs logical operations (AND, OR, NOT, XOR) on inputs based on their truthiness"
|
||||
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {
|
||||
"required": {
|
||||
"operation": (["AND", "OR", "NOT", "XOR"], {
|
||||
"default": "AND",
|
||||
"tooltip": "Logical operation to perform"
|
||||
}),
|
||||
"input_a": (AlwaysEqualProxy("*"), {
|
||||
"tooltip": "First input to evaluate for truthiness",
|
||||
"forceInput": True,
|
||||
}),
|
||||
"always_execute": ("BOOLEAN", {
|
||||
"default": True,
|
||||
}),
|
||||
"operation": (
|
||||
["AND", "OR", "NOT", "XOR"],
|
||||
{"default": "AND", "tooltip": "Logical operation to perform"},
|
||||
),
|
||||
"input_a": (
|
||||
AlwaysEqualProxy("*"),
|
||||
{
|
||||
"tooltip": "First input to evaluate for truthiness",
|
||||
"forceInput": True,
|
||||
},
|
||||
),
|
||||
"always_execute": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": True,
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"input_b": (AlwaysEqualProxy("*"), {
|
||||
"tooltip": "Second input to evaluate for truthiness (not used for NOT operation)",
|
||||
}),
|
||||
}
|
||||
"input_b": (
|
||||
AlwaysEqualProxy("*"),
|
||||
{
|
||||
"tooltip": "Second input to evaluate for truthiness (not used for NOT operation)",
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
RETURN_TYPES = ("BOOLEAN",)
|
||||
FUNCTION = "update"
|
||||
CATEGORY = "Realtime Nodes/control/logic"
|
||||
@@ -89,19 +107,19 @@ class LogicOperator(ControlNodeBase):
|
||||
def update(self, operation, input_a, always_execute=True, input_b=None):
|
||||
"""Perform the selected logical operation on inputs based on their truthiness."""
|
||||
a = bool(input_a)
|
||||
|
||||
|
||||
if operation == "NOT":
|
||||
return (not a,)
|
||||
|
||||
|
||||
# For all other operations, we need input_b
|
||||
b = bool(input_b)
|
||||
|
||||
|
||||
if operation == "AND":
|
||||
return (a and b,)
|
||||
elif operation == "OR":
|
||||
return (a or b,)
|
||||
elif operation == "XOR":
|
||||
return (a != b,)
|
||||
|
||||
|
||||
# Should never get here, but just in case
|
||||
return (False,)
|
||||
return (False,)
|
||||
|
||||
@@ -1 +1 @@
|
||||
# This file intentionally left blank
|
||||
# This file intentionally left blank
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import torch
|
||||
import numpy as np
|
||||
import mediapipe as mp
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
|
||||
class HandTrackingNode:
|
||||
@classmethod
|
||||
@@ -9,66 +10,51 @@ class HandTrackingNode:
|
||||
return {
|
||||
"required": {
|
||||
"image": ("IMAGE",),
|
||||
"detection_confidence": ("FLOAT", {
|
||||
"default": 0.5,
|
||||
"min": 0.1,
|
||||
"max": 1.0,
|
||||
"step": 0.05
|
||||
}),
|
||||
"tracking_confidence": ("FLOAT", {
|
||||
"default": 0.5,
|
||||
"min": 0.1,
|
||||
"max": 1.0,
|
||||
"step": 0.05
|
||||
}),
|
||||
"detection_confidence": ("FLOAT", {"default": 0.5, "min": 0.1, "max": 1.0, "step": 0.05}),
|
||||
"tracking_confidence": ("FLOAT", {"default": 0.5, "min": 0.1, "max": 1.0, "step": 0.05}),
|
||||
},
|
||||
"optional": {
|
||||
"max_num_hands": ("INT", {
|
||||
"default": 2,
|
||||
"min": 1,
|
||||
"max": 4,
|
||||
"step": 1
|
||||
}),
|
||||
"static_image_mode": ("BOOLEAN", {
|
||||
"default": False
|
||||
}),
|
||||
"draw_debug": ("BOOLEAN", {
|
||||
"default": False
|
||||
})
|
||||
}
|
||||
"max_num_hands": ("INT", {"default": 2, "min": 1, "max": 4, "step": 1}),
|
||||
"static_image_mode": ("BOOLEAN", {"default": False}),
|
||||
"draw_debug": ("BOOLEAN", {"default": False}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "HAND_DATA")
|
||||
RETURN_NAMES = ("debug_image", "hand_data")
|
||||
FUNCTION = "track_hands"
|
||||
CATEGORY = "image/processing"
|
||||
DESCRIPTION = "(((EXPERIMENTAL))) Track hands in an image using MediaPipe. Returns a debug image with landmarks and a dictionary of hand data."
|
||||
DESCRIPTION = (
|
||||
"(((EXPERIMENTAL))) Track hands in an image using MediaPipe. Returns a debug image with landmarks and a dictionary of hand data."
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
self.last_detection_conf = 0.5
|
||||
self.last_tracking_conf = 0.5
|
||||
self.last_max_hands = 2
|
||||
self.last_static_mode = True
|
||||
|
||||
|
||||
self.mp_hands = mp.solutions.hands
|
||||
self.hands = self.mp_hands.Hands(
|
||||
static_image_mode=self.last_static_mode,
|
||||
max_num_hands=self.last_max_hands,
|
||||
min_detection_confidence=self.last_detection_conf,
|
||||
min_tracking_confidence=self.last_tracking_conf
|
||||
min_tracking_confidence=self.last_tracking_conf,
|
||||
)
|
||||
|
||||
def track_hands(self, image, detection_confidence, tracking_confidence, max_num_hands=2, static_image_mode=True, draw_debug=True):
|
||||
# Check if we need to reinitialize MediaPipe hands
|
||||
if (detection_confidence != self.last_detection_conf or
|
||||
tracking_confidence != self.last_tracking_conf or
|
||||
max_num_hands != self.last_max_hands or
|
||||
static_image_mode != self.last_static_mode):
|
||||
|
||||
if (
|
||||
detection_confidence != self.last_detection_conf
|
||||
or tracking_confidence != self.last_tracking_conf
|
||||
or max_num_hands != self.last_max_hands
|
||||
or static_image_mode != self.last_static_mode
|
||||
):
|
||||
self.hands = self.mp_hands.Hands(
|
||||
static_image_mode=static_image_mode,
|
||||
max_num_hands=max_num_hands,
|
||||
min_detection_confidence=detection_confidence,
|
||||
min_tracking_confidence=tracking_confidence
|
||||
min_tracking_confidence=tracking_confidence,
|
||||
)
|
||||
self.last_detection_conf = detection_confidence
|
||||
self.last_tracking_conf = tracking_confidence
|
||||
@@ -83,61 +69,53 @@ class HandTrackingNode:
|
||||
# Process each image in batch
|
||||
processed_images = []
|
||||
hand_data_batch = []
|
||||
|
||||
|
||||
for i in range(batch_size):
|
||||
single_image = image_np[i]
|
||||
image_rgb = cv2.cvtColor(single_image, cv2.COLOR_BGR2RGB)
|
||||
results = self.hands.process(image_rgb)
|
||||
|
||||
hand_data = {
|
||||
"left_hand": None,
|
||||
"right_hand": None,
|
||||
"num_hands": 0,
|
||||
"hands_present": False
|
||||
}
|
||||
|
||||
|
||||
hand_data = {"left_hand": None, "right_hand": None, "num_hands": 0, "hands_present": False}
|
||||
|
||||
if results.multi_hand_landmarks:
|
||||
hand_data["num_hands"] = len(results.multi_hand_landmarks)
|
||||
hand_data["hands_present"] = True
|
||||
|
||||
|
||||
# Create single debug image for all hands
|
||||
debug_image = single_image.copy() if draw_debug else single_image
|
||||
|
||||
|
||||
# Process all detected hands
|
||||
for hand_idx, (hand_landmarks, handedness) in enumerate(zip(results.multi_hand_landmarks, results.multi_handedness)):
|
||||
hand_type = handedness.classification[0].label.lower()
|
||||
landmarks = np.array([[lm.x, lm.y, lm.z] for lm in hand_landmarks.landmark])
|
||||
|
||||
|
||||
# Store hand data based on type
|
||||
if hand_type == "left" and hand_data["left_hand"] is None:
|
||||
hand_data["left_hand"] = landmarks
|
||||
elif hand_type == "right" and hand_data["right_hand"] is None:
|
||||
hand_data["right_hand"] = landmarks
|
||||
|
||||
|
||||
# Draw landmarks for this hand on the shared debug image
|
||||
if draw_debug:
|
||||
mp.solutions.drawing_utils.draw_landmarks(
|
||||
debug_image,
|
||||
hand_landmarks,
|
||||
self.mp_hands.HAND_CONNECTIONS
|
||||
)
|
||||
|
||||
mp.solutions.drawing_utils.draw_landmarks(debug_image, hand_landmarks, self.mp_hands.HAND_CONNECTIONS)
|
||||
|
||||
processed_images.append(debug_image)
|
||||
else:
|
||||
processed_images.append(single_image)
|
||||
|
||||
|
||||
hand_data_batch.append(hand_data)
|
||||
|
||||
# Convert back to tensors
|
||||
processed_batch = np.stack(processed_images, axis=0)
|
||||
output_tensor = torch.from_numpy(processed_batch).float() / 255.0
|
||||
|
||||
|
||||
return (output_tensor, hand_data_batch)
|
||||
|
||||
def __del__(self):
|
||||
if hasattr(self, 'hands'):
|
||||
if hasattr(self, "hands"):
|
||||
self.hands.close()
|
||||
|
||||
|
||||
class HandMaskNode:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
@@ -156,57 +134,57 @@ class HandMaskNode:
|
||||
def create_mask(self, image, hand_data, mask_type):
|
||||
batch_size, height, width, channels = image.shape
|
||||
masks = np.zeros((batch_size, height, width), dtype=np.float32)
|
||||
|
||||
|
||||
# Finger indices in MediaPipe hand landmarks
|
||||
finger_indices = {
|
||||
"thumb": [1,2,3,4],
|
||||
"index": [5,6,7,8],
|
||||
"middle": [9,10,11,12],
|
||||
"ring": [13,14,15,16],
|
||||
"pinky": [17,18,19,20]
|
||||
"thumb": [1, 2, 3, 4],
|
||||
"index": [5, 6, 7, 8],
|
||||
"middle": [9, 10, 11, 12],
|
||||
"ring": [13, 14, 15, 16],
|
||||
"pinky": [17, 18, 19, 20],
|
||||
}
|
||||
palm_indices = [0,1,5,9,13,17] # Wrist and base of each finger
|
||||
|
||||
palm_indices = [0, 1, 5, 9, 13, 17] # Wrist and base of each finger
|
||||
|
||||
for i in range(batch_size):
|
||||
frame_hand_data = hand_data[i]
|
||||
if not frame_hand_data["hands_present"]:
|
||||
continue
|
||||
|
||||
|
||||
frame_mask = np.zeros((height, width), dtype=np.float32)
|
||||
|
||||
|
||||
for hand_key in ["right_hand", "left_hand"]:
|
||||
hand = frame_hand_data[hand_key]
|
||||
if hand is None:
|
||||
continue
|
||||
|
||||
|
||||
points = hand[:, :2] # Take only x,y
|
||||
points = (points * np.array([width, height])).astype(np.int32)
|
||||
|
||||
|
||||
if mask_type == "full_hand":
|
||||
hull = cv2.convexHull(points)
|
||||
cv2.fillConvexPoly(frame_mask, hull, 1.0)
|
||||
|
||||
|
||||
elif mask_type == "palm_only":
|
||||
palm_points = points[palm_indices]
|
||||
hull = cv2.convexHull(palm_points)
|
||||
cv2.fillConvexPoly(frame_mask, hull, 1.0)
|
||||
|
||||
|
||||
elif mask_type == "fingers_only":
|
||||
for finger in finger_indices.values():
|
||||
finger_points = points[finger]
|
||||
cv2.fillConvexPoly(frame_mask, cv2.convexHull(finger_points), 1.0)
|
||||
|
||||
|
||||
elif mask_type in finger_indices:
|
||||
finger_points = points[finger_indices[mask_type]]
|
||||
cv2.fillConvexPoly(frame_mask, cv2.convexHull(finger_points), 1.0)
|
||||
|
||||
|
||||
masks[i] = frame_mask
|
||||
|
||||
|
||||
# # Debug output
|
||||
# if frame_hand_data["right_hand"] is not None:
|
||||
# print(f"Right hand points range: x({np.min(points[:, 0]):.1f}-{np.max(points[:, 0]):.1f}), y({np.min(points[:, 1]):.1f}-{np.max(points[:, 1]):.1f})")
|
||||
# if frame_hand_data["left_hand"] is not None:
|
||||
# print(f"Left hand points range: x({np.min(points[:, 0]):.1f}-{np.max(points[:, 0]):.1f}), y({np.min(points[:, 1]):.1f}-{np.max(points[:, 1]):.1f})")
|
||||
|
||||
|
||||
mask_tensor = torch.from_numpy(masks)
|
||||
return (image, mask_tensor)
|
||||
return (image, mask_tensor)
|
||||
|
||||
@@ -1 +1 @@
|
||||
# This file intentionally left blank
|
||||
# This file intentionally left blank
|
||||
|
||||
@@ -1,164 +1,152 @@
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import numpy as np
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
import torch
|
||||
import random
|
||||
from ....src.utils.general import AlwaysEqualProxy
|
||||
|
||||
# # Add package root to Python path # <-- REMOVED
|
||||
# sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from ....src.realtimenodes.control_base import ControlNodeBase
|
||||
from ....src.utils.similar_image_filter import SimilarImageFilter
|
||||
|
||||
|
||||
class FPSMonitor(ControlNodeBase):
|
||||
"""Generates an FPS overlay as an image and mask"""
|
||||
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "MASK")
|
||||
FUNCTION = "update"
|
||||
CATEGORY = "Realtime Nodes/control/utility"
|
||||
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
inputs = super().INPUT_TYPES()
|
||||
inputs["required"].update({
|
||||
"width": ("INT", {
|
||||
"default": 512,
|
||||
"min": 64,
|
||||
"max": 4096,
|
||||
"step": 1
|
||||
}),
|
||||
"height": ("INT", {
|
||||
"default": 512,
|
||||
"min": 64,
|
||||
"max": 4096,
|
||||
"step": 1
|
||||
}),
|
||||
"text_color": ("INT", {
|
||||
"default": 255,
|
||||
"min": 0,
|
||||
"max": 255,
|
||||
"step": 1
|
||||
}),
|
||||
"text_size": ("FLOAT", {
|
||||
"default": 1.0,
|
||||
"min": 0.1,
|
||||
"max": 5.0,
|
||||
"step": 0.1
|
||||
}),
|
||||
"window_size": ("INT", {
|
||||
"default": 60,
|
||||
"min": 1,
|
||||
"max": 300,
|
||||
"step": 1,
|
||||
"tooltip": "Number of frames to average over"
|
||||
})
|
||||
})
|
||||
inputs["required"].update(
|
||||
{
|
||||
"width": ("INT", {"default": 512, "min": 64, "max": 4096, "step": 1}),
|
||||
"height": ("INT", {"default": 512, "min": 64, "max": 4096, "step": 1}),
|
||||
"text_color": ("INT", {"default": 255, "min": 0, "max": 255, "step": 1}),
|
||||
"text_size": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 5.0, "step": 0.1}),
|
||||
"window_size": (
|
||||
"INT",
|
||||
{"default": 60, "min": 1, "max": 300, "step": 1, "tooltip": "Number of frames to average over"},
|
||||
),
|
||||
}
|
||||
)
|
||||
return inputs
|
||||
|
||||
def update(self, width: int, height: int, text_color: int, text_size: float, window_size: int, always_execute=True):
|
||||
current_time = time.time()
|
||||
state = self.get_state({
|
||||
"last_time": current_time,
|
||||
"frame_times": [],
|
||||
"last_fps": 0,
|
||||
"last_avg_fps": 0,
|
||||
"cached_image": None,
|
||||
"cached_mask": None,
|
||||
"last_dims": (0, 0),
|
||||
"last_color": 0,
|
||||
"last_size": 0
|
||||
})
|
||||
|
||||
state = self.get_state(
|
||||
{
|
||||
"last_time": current_time,
|
||||
"frame_times": [],
|
||||
"last_fps": 0,
|
||||
"last_avg_fps": 0,
|
||||
"cached_image": None,
|
||||
"cached_mask": None,
|
||||
"last_dims": (0, 0),
|
||||
"last_color": 0,
|
||||
"last_size": 0,
|
||||
}
|
||||
)
|
||||
|
||||
# Calculate FPS
|
||||
delta_time = current_time - state["last_time"]
|
||||
current_fps = 1.0 / delta_time if delta_time > 0 else 0
|
||||
|
||||
|
||||
# Update rolling window
|
||||
state["frame_times"].append(delta_time)
|
||||
if len(state["frame_times"]) > window_size:
|
||||
state["frame_times"].pop(0)
|
||||
|
||||
|
||||
# Calculate average FPS
|
||||
avg_delta = sum(state["frame_times"]) / len(state["frame_times"])
|
||||
average_fps = 1.0 / avg_delta if avg_delta > 0 else 0
|
||||
|
||||
|
||||
# Check if we need to redraw
|
||||
needs_redraw = (
|
||||
abs(current_fps - state["last_fps"]) > 1.0 or # FPS changed significantly
|
||||
abs(average_fps - state["last_avg_fps"]) > 0.5 or # Avg FPS changed significantly
|
||||
state["last_dims"] != (width, height) or # Dimensions changed
|
||||
state["last_color"] != text_color or # Color changed
|
||||
state["last_size"] != text_size or # Text size changed
|
||||
state["cached_image"] is None # No cache exists
|
||||
abs(current_fps - state["last_fps"]) > 1.0 # FPS changed significantly
|
||||
or abs(average_fps - state["last_avg_fps"]) > 0.5 # Avg FPS changed significantly
|
||||
or state["last_dims"] != (width, height) # Dimensions changed
|
||||
or state["last_color"] != text_color # Color changed
|
||||
or state["last_size"] != text_size # Text size changed
|
||||
or state["cached_image"] is None # No cache exists
|
||||
)
|
||||
|
||||
|
||||
if needs_redraw:
|
||||
# Create text
|
||||
fps_text = f"FPS: {current_fps:.1f} (avg: {average_fps:.1f})"
|
||||
|
||||
|
||||
# Create image and mask using numpy/cv2
|
||||
image = np.zeros((height, width, 3), dtype=np.uint8)
|
||||
mask = np.zeros((height, width), dtype=np.uint8)
|
||||
|
||||
|
||||
# Position text in top-left with padding
|
||||
padding = 10
|
||||
pos = (padding, padding + 20) # Add vertical offset for cv2's text positioning
|
||||
|
||||
|
||||
# Draw text - much simpler with cv2!
|
||||
cv2.putText(image, fps_text, pos, cv2.FONT_HERSHEY_SIMPLEX, text_size, (text_color, text_color, text_color), 1)
|
||||
cv2.putText(mask, fps_text, pos, cv2.FONT_HERSHEY_SIMPLEX, text_size, 255, 1)
|
||||
|
||||
|
||||
# Convert to float32 and normalize
|
||||
image = image.astype(np.float32) / 255.0
|
||||
mask = mask.astype(np.float32) / 255.0
|
||||
|
||||
|
||||
# Convert to torch tensors with correct shapes
|
||||
state["cached_image"] = torch.from_numpy(image[None, ...]) # BHWC
|
||||
state["cached_mask"] = torch.from_numpy(mask[None, ...]) # BHW
|
||||
|
||||
state["cached_mask"] = torch.from_numpy(mask[None, ...]) # BHW
|
||||
|
||||
# Update cache parameters
|
||||
state["last_fps"] = current_fps
|
||||
state["last_avg_fps"] = average_fps
|
||||
state["last_dims"] = (width, height)
|
||||
state["last_color"] = text_color
|
||||
state["last_size"] = text_size
|
||||
|
||||
|
||||
state["last_time"] = current_time
|
||||
self.set_state(state)
|
||||
|
||||
|
||||
return (state["cached_image"], state["cached_mask"])
|
||||
|
||||
|
||||
class SimilarityFilter(ControlNodeBase):
|
||||
DESCRIPTION = "Filters out similar consecutive images and outputs a signal to control downstream execution."
|
||||
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {
|
||||
"required": {
|
||||
"image": ("IMAGE", {
|
||||
"tooltip": "Input image to compare with previous frame"
|
||||
}),
|
||||
"always_execute": ("BOOLEAN", {
|
||||
"default": False,
|
||||
}),
|
||||
"threshold": ("FLOAT", {
|
||||
"default": 0.98,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Similarity threshold (0-1). Higher values mean more frames are considered similar"
|
||||
}),
|
||||
"max_skip_frames": ("INT", {
|
||||
"default": 10,
|
||||
"min": 1,
|
||||
"max": 100,
|
||||
"step": 1,
|
||||
"tooltip": "Maximum number of consecutive frames to skip before forcing execution"
|
||||
}),
|
||||
"image": ("IMAGE", {"tooltip": "Input image to compare with previous frame"}),
|
||||
"always_execute": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
},
|
||||
),
|
||||
"threshold": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.98,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Similarity threshold (0-1). Higher values mean more frames are considered similar",
|
||||
},
|
||||
),
|
||||
"max_skip_frames": (
|
||||
"INT",
|
||||
{
|
||||
"default": 10,
|
||||
"min": 1,
|
||||
"max": 100,
|
||||
"step": 1,
|
||||
"tooltip": "Maximum number of consecutive frames to skip before forcing execution",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "BOOLEAN")
|
||||
RETURN_NAMES = ("image", "should_execute")
|
||||
FUNCTION = "update"
|
||||
@@ -170,12 +158,9 @@ class SimilarityFilter(ControlNodeBase):
|
||||
|
||||
def update(self, image, threshold=0.98, max_skip_frames=10, always_execute=False):
|
||||
print(f"[DEBUG] Input image object: {hex(id(image))}, shape: {image.shape}, device: {image.device}")
|
||||
|
||||
|
||||
# Get state with defaults
|
||||
state = self.get_state({
|
||||
"prev_image": None,
|
||||
"skip_count": 0
|
||||
})
|
||||
state = self.get_state({"prev_image": None, "skip_count": 0})
|
||||
|
||||
# First frame case
|
||||
if state["prev_image"] is None:
|
||||
@@ -187,7 +172,7 @@ class SimilarityFilter(ControlNodeBase):
|
||||
# Update filter parameters
|
||||
self._similarity_filter.set_threshold(threshold)
|
||||
self._similarity_filter.set_max_skip_frame(max_skip_frames)
|
||||
|
||||
|
||||
# Use filter to check similarity
|
||||
result = self._similarity_filter(image)
|
||||
should_execute = result is not None
|
||||
@@ -200,7 +185,7 @@ class SimilarityFilter(ControlNodeBase):
|
||||
state["skip_count"] = 0
|
||||
self.set_state(state)
|
||||
return (image, True) # Force execution after max skips
|
||||
|
||||
|
||||
# Skip frame - return previous image and False for execution
|
||||
state["skip_count"] += 1
|
||||
self.set_state(state)
|
||||
@@ -211,12 +196,3 @@ class SimilarityFilter(ControlNodeBase):
|
||||
state["skip_count"] = 0
|
||||
self.set_state(state)
|
||||
return (image, True)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+603
-529
File diff suppressed because it is too large
Load Diff
+4
-1
@@ -2,7 +2,7 @@
|
||||
name = "comfyui_realtimenodes"
|
||||
description = "These nodes are for real-time applications of ComfyUI."
|
||||
|
||||
version = "1.3.0"
|
||||
version = "1.3.1"
|
||||
|
||||
license = {file = "LICENSE"}
|
||||
dependencies = ["opencv-python"]
|
||||
@@ -15,3 +15,6 @@ Repository = "https://github.com/ryanontheinside/ComfyUI_RealTimeNodes"
|
||||
PublisherId = "ryanontheinside"
|
||||
DisplayName = "ComfyUI_RealTimeNodes"
|
||||
Icon = "⏱️"
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 140
|
||||
+1
-1
@@ -1 +1 @@
|
||||
# This file intentionally left blank
|
||||
# This file intentionally left blank
|
||||
|
||||
@@ -9,4 +9,4 @@ with support for single values, batches, and point collections.
|
||||
from .coordinate_system import CoordinateSystem, Point, PointList
|
||||
from .drawing_engine import DrawingEngine
|
||||
|
||||
__all__ = ['CoordinateSystem', 'Point', 'PointList', 'DrawingEngine']
|
||||
__all__ = ["CoordinateSystem", "Point", "PointList", "DrawingEngine"]
|
||||
|
||||
@@ -1,28 +1,25 @@
|
||||
import logging
|
||||
from typing import Optional, List, Any, Tuple, Union
|
||||
import math
|
||||
import torch
|
||||
import numpy as np
|
||||
import cv2
|
||||
import collections
|
||||
import typing # Add typing for Dict hint
|
||||
import logging
|
||||
import math
|
||||
import typing # Add typing for Dict hint
|
||||
from typing import Optional
|
||||
|
||||
#TODO: consider moving base nodes to src
|
||||
# TODO: consider moving base nodes to src
|
||||
|
||||
# Import from the new consolidated utilities
|
||||
from ...src.utils.math import scale_value
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_POSITION_HISTORY = 50 # Matches MAX_LANDMARK_HISTORY for consistency
|
||||
FLOAT_EQUALITY_TOLERANCE = 1e-6 # Same tolerance as base_delta_nodes
|
||||
MAX_POSITION_HISTORY = 50 # Matches MAX_LANDMARK_HISTORY for consistency
|
||||
FLOAT_EQUALITY_TOLERANCE = 1e-6 # Same tolerance as base_delta_nodes
|
||||
|
||||
|
||||
# --- Base Position Delta Logic (Handling Multiple Histories) ---
|
||||
class BaseCoordinateDelta:
|
||||
"""Base class for nodes calculating delta from position history.
|
||||
Handles single floats or lists of coordinates.
|
||||
Handles single floats or lists of coordinates.
|
||||
"""
|
||||
|
||||
FUNCTION = "execute"
|
||||
|
||||
def __init__(self):
|
||||
@@ -39,23 +36,21 @@ class BaseCoordinateDelta:
|
||||
def _get_position_delta(self, x: float, y: float, z: float, window_size: int, item_index: int = 0) -> Optional[float]:
|
||||
"""Calculates delta using the history deque for the given item_index."""
|
||||
history_deque = self._get_or_create_history(item_index)
|
||||
|
||||
|
||||
window_size = max(1, min(window_size, MAX_POSITION_HISTORY - 1))
|
||||
current_pos = (x, y, z)
|
||||
history_deque.append(current_pos)
|
||||
|
||||
|
||||
if len(history_deque) > window_size:
|
||||
pos_now = history_deque[-1]
|
||||
pos_past = history_deque[-(window_size + 1)] # Correct indexing
|
||||
pos_past = history_deque[-(window_size + 1)] # Correct indexing
|
||||
if pos_now is None or pos_past is None:
|
||||
logger.warning(f"{self.__class__.__name__}[{item_index}]: Missing position data in history window.")
|
||||
return None
|
||||
delta = math.sqrt((pos_now[0] - pos_past[0])**2 +
|
||||
(pos_now[1] - pos_past[1])**2 +
|
||||
(pos_now[2] - pos_past[2])**2)
|
||||
logger.warning(f"{self.__class__.__name__}[{item_index}]: Missing position data in history window.")
|
||||
return None
|
||||
delta = math.sqrt((pos_now[0] - pos_past[0]) ** 2 + (pos_now[1] - pos_past[1]) ** 2 + (pos_now[2] - pos_past[2]) ** 2)
|
||||
return delta
|
||||
else:
|
||||
# logger.debug(f"{self.__class__.__name__}[{item_index}]: Not enough history ({len(history_deque)}/{window_size+1}).")
|
||||
return None # Not enough history yet
|
||||
|
||||
return None # Not enough history yet
|
||||
|
||||
# Remove _handle_input_coords, logic moved to execute
|
||||
|
||||
@@ -5,44 +5,47 @@ This module provides a centralized system for handling coordinates across differ
|
||||
spaces (normalized, pixel) with support for both single values and batches.
|
||||
"""
|
||||
|
||||
import torch
|
||||
import numpy as np
|
||||
from typing import Union, List, Tuple, Optional, Dict, Any
|
||||
import logging
|
||||
from typing import List, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CoordinateSystem:
|
||||
"""Unified coordinate system handling for coordinate space conversions."""
|
||||
|
||||
|
||||
# Constants for coordinate spaces
|
||||
NORMALIZED = "normalized" # 0.0-1.0 range
|
||||
PIXEL = "pixel" # Pixel coordinates
|
||||
|
||||
PIXEL = "pixel" # Pixel coordinates
|
||||
|
||||
@staticmethod
|
||||
def normalize(coords: Union[float, List[float], torch.Tensor],
|
||||
dimensions: Union[int, Tuple[int, ...]],
|
||||
input_space: str = PIXEL) -> Union[float, List[float], torch.Tensor]:
|
||||
def normalize(
|
||||
coords: Union[float, List[float], torch.Tensor],
|
||||
dimensions: Union[int, Tuple[int, ...]],
|
||||
input_space: str = PIXEL,
|
||||
) -> Union[float, List[float], torch.Tensor]:
|
||||
"""
|
||||
Normalize coordinates to 0.0-1.0 range.
|
||||
|
||||
|
||||
Args:
|
||||
coords: Coordinates to normalize
|
||||
dimensions: Dimensions to normalize by (width, height)
|
||||
input_space: Current coordinate space
|
||||
|
||||
|
||||
Returns:
|
||||
Normalized coordinates (0.0-1.0)
|
||||
"""
|
||||
if input_space == CoordinateSystem.NORMALIZED:
|
||||
return coords
|
||||
|
||||
|
||||
# Handle different dimension specifications
|
||||
if isinstance(dimensions, tuple):
|
||||
dim_size = dimensions[0] if len(dimensions) == 1 else dimensions
|
||||
else:
|
||||
dim_size = dimensions
|
||||
|
||||
|
||||
# Handle different input types
|
||||
if isinstance(coords, list):
|
||||
if isinstance(dim_size, tuple):
|
||||
@@ -67,31 +70,33 @@ class CoordinateSystem:
|
||||
return coords.float() / dim_size
|
||||
else:
|
||||
return float(coords) / dim_size
|
||||
|
||||
|
||||
@staticmethod
|
||||
def denormalize(coords: Union[float, List[float], torch.Tensor],
|
||||
dimensions: Union[int, Tuple[int, ...]],
|
||||
output_space: str = PIXEL) -> Union[float, List[float], torch.Tensor]:
|
||||
def denormalize(
|
||||
coords: Union[float, List[float], torch.Tensor],
|
||||
dimensions: Union[int, Tuple[int, ...]],
|
||||
output_space: str = PIXEL,
|
||||
) -> Union[float, List[float], torch.Tensor]:
|
||||
"""
|
||||
Convert normalized coordinates to target space.
|
||||
|
||||
|
||||
Args:
|
||||
coords: Normalized coordinates (0.0-1.0)
|
||||
dimensions: Target dimensions
|
||||
output_space: Target coordinate space
|
||||
|
||||
|
||||
Returns:
|
||||
Denormalized coordinates in target space
|
||||
"""
|
||||
if output_space == CoordinateSystem.NORMALIZED:
|
||||
return coords
|
||||
|
||||
|
||||
# Handle different dimension specifications
|
||||
if isinstance(dimensions, tuple):
|
||||
dim_size = dimensions[0] if len(dimensions) == 1 else dimensions
|
||||
else:
|
||||
dim_size = dimensions
|
||||
|
||||
|
||||
# Handle different input types
|
||||
if isinstance(coords, list):
|
||||
if isinstance(dim_size, tuple):
|
||||
@@ -117,69 +122,68 @@ class CoordinateSystem:
|
||||
return torch.clamp(coords * dim_size, 0, dim_size - 1)
|
||||
else:
|
||||
return float(max(0, min(dim_size - 1, coords * dim_size)))
|
||||
|
||||
|
||||
@staticmethod
|
||||
def convert(coords: Union[float, List[float], torch.Tensor],
|
||||
dimensions: Union[int, Tuple[int, ...]],
|
||||
from_space: str,
|
||||
to_space: str) -> Union[float, List[float], torch.Tensor]:
|
||||
def convert(
|
||||
coords: Union[float, List[float], torch.Tensor],
|
||||
dimensions: Union[int, Tuple[int, ...]],
|
||||
from_space: str,
|
||||
to_space: str,
|
||||
) -> Union[float, List[float], torch.Tensor]:
|
||||
"""
|
||||
Convert between coordinate spaces.
|
||||
|
||||
|
||||
Args:
|
||||
coords: Coordinates to convert
|
||||
dimensions: Target dimensions
|
||||
from_space: Source coordinate space
|
||||
to_space: Target coordinate space
|
||||
|
||||
|
||||
Returns:
|
||||
Converted coordinates
|
||||
"""
|
||||
if from_space == to_space:
|
||||
return coords
|
||||
|
||||
|
||||
# First normalize if needed
|
||||
if from_space != CoordinateSystem.NORMALIZED:
|
||||
normalized = CoordinateSystem.normalize(coords, dimensions, from_space)
|
||||
else:
|
||||
normalized = coords
|
||||
|
||||
|
||||
# Then denormalize to target space if needed
|
||||
if to_space != CoordinateSystem.NORMALIZED:
|
||||
return CoordinateSystem.denormalize(normalized, dimensions, to_space)
|
||||
else:
|
||||
return normalized
|
||||
|
||||
|
||||
@staticmethod
|
||||
def get_dimensions_from_tensor(tensor: torch.Tensor) -> Tuple[int, int]:
|
||||
"""
|
||||
Extract dimensions from tensor in BHWC format.
|
||||
|
||||
|
||||
Args:
|
||||
tensor: Input tensor in BHWC format
|
||||
|
||||
|
||||
Returns:
|
||||
Tuple of (width, height)
|
||||
"""
|
||||
if tensor.dim() != 4:
|
||||
raise ValueError(f"Expected 4D tensor (BHWC), got shape {tensor.shape}")
|
||||
|
||||
|
||||
_, height, width, _ = tensor.shape
|
||||
return (width, height)
|
||||
|
||||
|
||||
class Point:
|
||||
"""
|
||||
Represents a 2D or 3D point with coordinate space awareness.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
x: float,
|
||||
y: float,
|
||||
z: Optional[float] = None,
|
||||
space: str = CoordinateSystem.NORMALIZED):
|
||||
|
||||
def __init__(self, x: float, y: float, z: Optional[float] = None, space: str = CoordinateSystem.NORMALIZED):
|
||||
"""
|
||||
Initialize a point with coordinates and space information.
|
||||
|
||||
|
||||
Args:
|
||||
x: X coordinate
|
||||
y: Y coordinate
|
||||
@@ -191,85 +195,85 @@ class Point:
|
||||
self.z = z
|
||||
self.space = space
|
||||
self.is_3d = z is not None
|
||||
|
||||
def to_normalized(self, dimensions: Tuple[int, ...]) -> 'Point':
|
||||
|
||||
def to_normalized(self, dimensions: Tuple[int, ...]) -> "Point":
|
||||
"""
|
||||
Convert point to normalized space.
|
||||
|
||||
|
||||
Args:
|
||||
dimensions: Dimensions for conversion
|
||||
|
||||
|
||||
Returns:
|
||||
New point in normalized space
|
||||
"""
|
||||
if self.space == CoordinateSystem.NORMALIZED:
|
||||
return self
|
||||
|
||||
|
||||
norm_x = CoordinateSystem.normalize(self.x, dimensions[0], self.space)
|
||||
norm_y = CoordinateSystem.normalize(self.y, dimensions[1], self.space)
|
||||
norm_z = None
|
||||
|
||||
|
||||
if self.is_3d and len(dimensions) > 2:
|
||||
norm_z = CoordinateSystem.normalize(self.z, dimensions[2], self.space)
|
||||
|
||||
|
||||
return Point(norm_x, norm_y, norm_z, CoordinateSystem.NORMALIZED)
|
||||
|
||||
def to_pixel(self, dimensions: Tuple[int, ...]) -> 'Point':
|
||||
|
||||
def to_pixel(self, dimensions: Tuple[int, ...]) -> "Point":
|
||||
"""
|
||||
Convert point to pixel space.
|
||||
|
||||
|
||||
Args:
|
||||
dimensions: Dimensions for conversion
|
||||
|
||||
|
||||
Returns:
|
||||
New point in pixel space
|
||||
"""
|
||||
if self.space == CoordinateSystem.PIXEL:
|
||||
return self
|
||||
|
||||
|
||||
# First normalize if not already
|
||||
if self.space != CoordinateSystem.NORMALIZED:
|
||||
normalized = self.to_normalized(dimensions)
|
||||
else:
|
||||
normalized = self
|
||||
|
||||
|
||||
pixel_x = CoordinateSystem.denormalize(normalized.x, dimensions[0], CoordinateSystem.PIXEL)
|
||||
pixel_y = CoordinateSystem.denormalize(normalized.y, dimensions[1], CoordinateSystem.PIXEL)
|
||||
pixel_z = None
|
||||
|
||||
|
||||
if self.is_3d and len(dimensions) > 2:
|
||||
pixel_z = CoordinateSystem.denormalize(normalized.z, dimensions[2], CoordinateSystem.PIXEL)
|
||||
|
||||
|
||||
return Point(pixel_x, pixel_y, pixel_z, CoordinateSystem.PIXEL)
|
||||
|
||||
def to_space(self, dimensions: Tuple[int, ...], target_space: str) -> 'Point':
|
||||
|
||||
def to_space(self, dimensions: Tuple[int, ...], target_space: str) -> "Point":
|
||||
"""
|
||||
Convert point to target space.
|
||||
|
||||
|
||||
Args:
|
||||
dimensions: Dimensions for conversion
|
||||
target_space: Target coordinate space
|
||||
|
||||
|
||||
Returns:
|
||||
New point in target space
|
||||
"""
|
||||
if self.space == target_space:
|
||||
return self
|
||||
|
||||
|
||||
if target_space == CoordinateSystem.NORMALIZED:
|
||||
return self.to_normalized(dimensions)
|
||||
elif target_space == CoordinateSystem.PIXEL:
|
||||
return self.to_pixel(dimensions)
|
||||
else:
|
||||
raise ValueError(f"Unsupported target space: {target_space}")
|
||||
|
||||
|
||||
def to_tuple(self, include_z: bool = False) -> Tuple[float, ...]:
|
||||
"""
|
||||
Convert point to tuple representation.
|
||||
|
||||
|
||||
Args:
|
||||
include_z: Whether to include Z coordinate if available
|
||||
|
||||
|
||||
Returns:
|
||||
Tuple of coordinates
|
||||
"""
|
||||
@@ -277,14 +281,14 @@ class Point:
|
||||
return (self.x, self.y, self.z)
|
||||
else:
|
||||
return (self.x, self.y)
|
||||
|
||||
|
||||
def to_list(self, include_z: bool = False) -> List[float]:
|
||||
"""
|
||||
Convert point to list representation.
|
||||
|
||||
|
||||
Args:
|
||||
include_z: Whether to include Z coordinate if available
|
||||
|
||||
|
||||
Returns:
|
||||
List of coordinates
|
||||
"""
|
||||
@@ -297,126 +301,127 @@ class Point:
|
||||
else:
|
||||
return f"Point({self.x}, {self.y}, space='{self.space}')"
|
||||
|
||||
|
||||
class PointList:
|
||||
"""
|
||||
Represents a list of 2D or 3D points with coordinate space awareness.
|
||||
Useful for handling batches of points or landmark lists.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
points: List[Point] = None,
|
||||
space: str = CoordinateSystem.NORMALIZED):
|
||||
|
||||
def __init__(self, points: List[Point] = None, space: str = CoordinateSystem.NORMALIZED):
|
||||
"""
|
||||
Initialize a point list.
|
||||
|
||||
|
||||
Args:
|
||||
points: List of Point objects
|
||||
space: Coordinate space for all points
|
||||
"""
|
||||
self.points = points or []
|
||||
self.space = space
|
||||
|
||||
|
||||
@classmethod
|
||||
def from_coordinates(cls,
|
||||
x_coords: List[float],
|
||||
y_coords: List[float],
|
||||
z_coords: Optional[List[float]] = None,
|
||||
space: str = CoordinateSystem.NORMALIZED) -> 'PointList':
|
||||
def from_coordinates(
|
||||
cls,
|
||||
x_coords: List[float],
|
||||
y_coords: List[float],
|
||||
z_coords: Optional[List[float]] = None,
|
||||
space: str = CoordinateSystem.NORMALIZED,
|
||||
) -> "PointList":
|
||||
"""
|
||||
Create a PointList from separate coordinate lists.
|
||||
|
||||
|
||||
Args:
|
||||
x_coords: List of X coordinates
|
||||
y_coords: List of Y coordinates
|
||||
z_coords: List of Z coordinates (optional)
|
||||
space: Coordinate space
|
||||
|
||||
|
||||
Returns:
|
||||
New PointList object
|
||||
"""
|
||||
if len(x_coords) != len(y_coords):
|
||||
raise ValueError(f"Coordinate lists must have same length: x={len(x_coords)}, y={len(y_coords)}")
|
||||
|
||||
|
||||
has_z = z_coords is not None
|
||||
if has_z and len(z_coords) != len(x_coords):
|
||||
raise ValueError(f"Z coordinates must match length of X/Y: x={len(x_coords)}, z={len(z_coords)}")
|
||||
|
||||
|
||||
points = []
|
||||
for i in range(len(x_coords)):
|
||||
z = z_coords[i] if has_z else None
|
||||
points.append(Point(x_coords[i], y_coords[i], z, space))
|
||||
|
||||
|
||||
return cls(points, space)
|
||||
|
||||
def to_normalized(self, dimensions: Tuple[int, ...]) -> 'PointList':
|
||||
|
||||
def to_normalized(self, dimensions: Tuple[int, ...]) -> "PointList":
|
||||
"""
|
||||
Convert all points to normalized space.
|
||||
|
||||
|
||||
Args:
|
||||
dimensions: Dimensions for conversion
|
||||
|
||||
|
||||
Returns:
|
||||
New PointList in normalized space
|
||||
"""
|
||||
if self.space == CoordinateSystem.NORMALIZED:
|
||||
return self
|
||||
|
||||
|
||||
normalized_points = [point.to_normalized(dimensions) for point in self.points]
|
||||
return PointList(normalized_points, CoordinateSystem.NORMALIZED)
|
||||
|
||||
def to_pixel(self, dimensions: Tuple[int, ...]) -> 'PointList':
|
||||
|
||||
def to_pixel(self, dimensions: Tuple[int, ...]) -> "PointList":
|
||||
"""
|
||||
Convert all points to pixel space.
|
||||
|
||||
|
||||
Args:
|
||||
dimensions: Dimensions for conversion
|
||||
|
||||
|
||||
Returns:
|
||||
New PointList in pixel space
|
||||
"""
|
||||
if self.space == CoordinateSystem.PIXEL:
|
||||
return self
|
||||
|
||||
|
||||
pixel_points = [point.to_pixel(dimensions) for point in self.points]
|
||||
return PointList(pixel_points, CoordinateSystem.PIXEL)
|
||||
|
||||
def to_space(self, dimensions: Tuple[int, ...], target_space: str) -> 'PointList':
|
||||
|
||||
def to_space(self, dimensions: Tuple[int, ...], target_space: str) -> "PointList":
|
||||
"""
|
||||
Convert all points to target space.
|
||||
|
||||
|
||||
Args:
|
||||
dimensions: Dimensions for conversion
|
||||
target_space: Target coordinate space
|
||||
|
||||
|
||||
Returns:
|
||||
New PointList in target space
|
||||
"""
|
||||
if self.space == target_space:
|
||||
return self
|
||||
|
||||
|
||||
if target_space == CoordinateSystem.NORMALIZED:
|
||||
return self.to_normalized(dimensions)
|
||||
elif target_space == CoordinateSystem.PIXEL:
|
||||
return self.to_pixel(dimensions)
|
||||
else:
|
||||
raise ValueError(f"Unsupported target space: {target_space}")
|
||||
|
||||
|
||||
def get_x_list(self) -> List[float]:
|
||||
"""Get list of X coordinates."""
|
||||
return [point.x for point in self.points]
|
||||
|
||||
|
||||
def get_y_list(self) -> List[float]:
|
||||
"""Get list of Y coordinates."""
|
||||
return [point.y for point in self.points]
|
||||
|
||||
|
||||
def get_z_list(self) -> List[Optional[float]]:
|
||||
"""Get list of Z coordinates (may contain None values)."""
|
||||
return [point.z for point in self.points]
|
||||
|
||||
|
||||
def __len__(self) -> int:
|
||||
"""Get number of points in the list."""
|
||||
return len(self.points)
|
||||
|
||||
|
||||
def __getitem__(self, idx: int) -> Point:
|
||||
"""Access point by index."""
|
||||
return self.points[idx]
|
||||
return self.points[idx]
|
||||
|
||||
+136
-117
@@ -1,26 +1,28 @@
|
||||
import torch
|
||||
import numpy as np
|
||||
import cv2
|
||||
from typing import Union, List, Tuple, Dict, Optional, Any
|
||||
import logging
|
||||
from typing import List, Tuple, Union
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from .coordinate_system import CoordinateSystem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DrawingEngine:
|
||||
"""High-performance drawing engine for real-time applications.
|
||||
Optimized for minimal tensor-numpy conversions and efficient batch operations.
|
||||
"""
|
||||
|
||||
|
||||
@staticmethod
|
||||
def hex_to_bgr(hex_color: str) -> Tuple[int, int, int]:
|
||||
"""Converts hex color to BGR tuple with validation."""
|
||||
hex_color = hex_color.lstrip('#')
|
||||
hex_color = hex_color.lstrip("#")
|
||||
if len(hex_color) != 6:
|
||||
return (0, 0, 0) # Default black
|
||||
try:
|
||||
rgb = tuple(int(hex_color[i:i+2], 16) for i in (0, 2, 4))
|
||||
rgb = tuple(int(hex_color[i : i + 2], 16) for i in (0, 2, 4))
|
||||
return (rgb[2], rgb[1], rgb[0]) # RGB to BGR
|
||||
except ValueError:
|
||||
return (0, 0, 0)
|
||||
@@ -30,7 +32,7 @@ class DrawingEngine:
|
||||
"""Fast preparation of single image from batch for drawing - minimal conversion."""
|
||||
if image.dim() != 4:
|
||||
raise ValueError("Input image must be BHWC format")
|
||||
|
||||
|
||||
device, dtype = image.device, image.dtype
|
||||
# Get single image, convert to float32 for processing
|
||||
img_tensor = image[batch_idx].to(torch.float32)
|
||||
@@ -44,17 +46,19 @@ class DrawingEngine:
|
||||
"""Fast conversion from numpy back to tensor with proper device/dtype."""
|
||||
return torch.from_numpy(img_np.astype(np.float32) / 255.0).to(device=device, dtype=dtype)
|
||||
|
||||
def draw_points(self,
|
||||
image: torch.Tensor,
|
||||
points_x: Union[float, List[float]],
|
||||
points_y: Union[float, List[float]],
|
||||
radius: int = 5,
|
||||
color: str = "#FF0000",
|
||||
is_normalized: bool = True,
|
||||
batch_mapping: str = "broadcast") -> torch.Tensor:
|
||||
def draw_points(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
points_x: Union[float, List[float]],
|
||||
points_y: Union[float, List[float]],
|
||||
radius: int = 5,
|
||||
color: str = "#FF0000",
|
||||
is_normalized: bool = True,
|
||||
batch_mapping: str = "broadcast",
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Draw points on images efficiently.
|
||||
|
||||
|
||||
Args:
|
||||
image: BHWC tensor
|
||||
points_x/y: Coordinate values (normalized 0-1 or pixel)
|
||||
@@ -62,30 +66,30 @@ class DrawingEngine:
|
||||
color: Hex color
|
||||
is_normalized: If True, coords are 0-1 normalized
|
||||
batch_mapping: How to map points to batch ("broadcast", "one-to-one", "all-on-first")
|
||||
|
||||
|
||||
Returns:
|
||||
Modified image tensor
|
||||
"""
|
||||
batch_size, height, width, _ = image.shape
|
||||
output_batch = image.clone() # Avoid modifying input
|
||||
bgr_color = self.hex_to_bgr(color)
|
||||
|
||||
|
||||
# Standardize input to lists for unified processing
|
||||
x_list = points_x if isinstance(points_x, list) else [points_x]
|
||||
y_list = points_y if isinstance(points_y, list) else [points_y]
|
||||
|
||||
|
||||
# Determine batch mapping strategy
|
||||
points_len = len(x_list)
|
||||
if batch_mapping == "one-to-one" and points_len != batch_size and points_len != 1:
|
||||
logger.warning(f"One-to-one mapping requested but point list length ({points_len}) != batch size ({batch_size})")
|
||||
batch_mapping = "all-on-first"
|
||||
|
||||
|
||||
# Process each image in batch
|
||||
for b in range(batch_size):
|
||||
# Skip processing based on mapping strategy
|
||||
if batch_mapping == "all-on-first" and b > 0:
|
||||
continue
|
||||
|
||||
|
||||
# Get points for this batch item based on mapping strategy
|
||||
if batch_mapping == "one-to-one" and points_len == batch_size:
|
||||
# One point per image
|
||||
@@ -99,19 +103,19 @@ class DrawingEngine:
|
||||
# All points on first image only
|
||||
curr_x = x_list
|
||||
curr_y = y_list
|
||||
|
||||
|
||||
# Skip if no points to draw
|
||||
if not curr_x:
|
||||
continue
|
||||
|
||||
|
||||
# Single efficient conversion to numpy
|
||||
img_np, device, dtype = self.prep_batch_image(output_batch, b)
|
||||
|
||||
|
||||
# Draw all points for this image
|
||||
for xi, yi in zip(curr_x, curr_y):
|
||||
if not isinstance(xi, (float, int)) or not isinstance(yi, (float, int)):
|
||||
continue
|
||||
|
||||
|
||||
# Convert to pixel coordinates if needed
|
||||
if is_normalized:
|
||||
px = int(CoordinateSystem.denormalize(xi, width, CoordinateSystem.PIXEL))
|
||||
@@ -119,34 +123,36 @@ class DrawingEngine:
|
||||
else:
|
||||
px = int(max(0, min(width - 1, xi)))
|
||||
py = int(max(0, min(height - 1, yi)))
|
||||
|
||||
|
||||
# Draw circle
|
||||
cv2.circle(img_np, (px, py), radius, bgr_color, -1)
|
||||
|
||||
|
||||
# Convert back to tensor and update batch
|
||||
output_batch[b] = self.tensor_from_numpy(img_np, device, dtype)
|
||||
|
||||
|
||||
return output_batch
|
||||
|
||||
def draw_lines(self,
|
||||
image: torch.Tensor,
|
||||
x1: Union[float, List[float]],
|
||||
y1: Union[float, List[float]],
|
||||
x2: Union[float, List[float]],
|
||||
y2: Union[float, List[float]],
|
||||
thickness: int = 2,
|
||||
color: str = "#00FF00",
|
||||
is_normalized: bool = True,
|
||||
batch_mapping: str = "broadcast",
|
||||
draw_endpoints: bool = False,
|
||||
point_radius: int = 3,
|
||||
draw_label: bool = False,
|
||||
label_text: str = "",
|
||||
label_position: str = "Midpoint",
|
||||
font_scale: float = 0.5) -> torch.Tensor:
|
||||
|
||||
def draw_lines(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
x1: Union[float, List[float]],
|
||||
y1: Union[float, List[float]],
|
||||
x2: Union[float, List[float]],
|
||||
y2: Union[float, List[float]],
|
||||
thickness: int = 2,
|
||||
color: str = "#00FF00",
|
||||
is_normalized: bool = True,
|
||||
batch_mapping: str = "broadcast",
|
||||
draw_endpoints: bool = False,
|
||||
point_radius: int = 3,
|
||||
draw_label: bool = False,
|
||||
label_text: str = "",
|
||||
label_position: str = "Midpoint",
|
||||
font_scale: float = 0.5,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Draw lines on images efficiently.
|
||||
|
||||
|
||||
Args:
|
||||
image: BHWC tensor
|
||||
x1/y1/x2/y2: Line endpoint coordinates
|
||||
@@ -156,36 +162,36 @@ class DrawingEngine:
|
||||
batch_mapping: How to map lines to batch
|
||||
draw_endpoints: Whether to draw circles at endpoints
|
||||
Other params: Styling options
|
||||
|
||||
|
||||
Returns:
|
||||
Modified image tensor
|
||||
"""
|
||||
batch_size, height, width, _ = image.shape
|
||||
output_batch = image.clone()
|
||||
bgr_color = self.hex_to_bgr(color)
|
||||
|
||||
|
||||
# Standardize input to lists
|
||||
x1_list = x1 if isinstance(x1, list) else [x1]
|
||||
y1_list = y1 if isinstance(y1, list) else [y1]
|
||||
x2_list = x2 if isinstance(x2, list) else [x2]
|
||||
y2_list = y2 if isinstance(y2, list) else [y2]
|
||||
|
||||
|
||||
# Validate input lengths
|
||||
line_len = len(x1_list)
|
||||
if not all(len(coord) == line_len for coord in [y1_list, x2_list, y2_list]):
|
||||
raise ValueError("All coordinate lists must have the same length")
|
||||
|
||||
|
||||
# Determine batch mapping
|
||||
if batch_mapping == "one-to-one" and line_len != batch_size and line_len != 1:
|
||||
logger.warning(f"One-to-one mapping requested but line list length ({line_len}) != batch size ({batch_size})")
|
||||
batch_mapping = "all-on-first"
|
||||
|
||||
|
||||
# Process each image in batch
|
||||
for b in range(batch_size):
|
||||
# Skip processing based on mapping strategy
|
||||
if batch_mapping == "all-on-first" and b > 0:
|
||||
continue
|
||||
|
||||
|
||||
# Get lines for this batch item
|
||||
if batch_mapping == "one-to-one" and line_len == batch_size:
|
||||
indices = [b]
|
||||
@@ -193,22 +199,22 @@ class DrawingEngine:
|
||||
indices = range(line_len)
|
||||
else: # "all-on-first"
|
||||
indices = range(line_len)
|
||||
|
||||
|
||||
# Skip if no lines to draw
|
||||
if not indices:
|
||||
continue
|
||||
|
||||
|
||||
# Single efficient conversion to numpy
|
||||
img_np, device, dtype = self.prep_batch_image(output_batch, b)
|
||||
|
||||
|
||||
# Draw all lines for this image
|
||||
for i in indices:
|
||||
_x1, _y1 = x1_list[i], y1_list[i]
|
||||
_x2, _y2 = x2_list[i], y2_list[i]
|
||||
|
||||
|
||||
if not all(isinstance(c, (float, int)) for c in [_x1, _y1, _x2, _y2]):
|
||||
continue
|
||||
|
||||
|
||||
# Convert to pixel coordinates if needed
|
||||
if is_normalized:
|
||||
px1 = int(CoordinateSystem.denormalize(_x1, width, CoordinateSystem.PIXEL))
|
||||
@@ -220,15 +226,15 @@ class DrawingEngine:
|
||||
py1 = int(max(0, min(height - 1, _y1)))
|
||||
px2 = int(max(0, min(width - 1, _x2)))
|
||||
py2 = int(max(0, min(height - 1, _y2)))
|
||||
|
||||
|
||||
# Draw line
|
||||
cv2.line(img_np, (px1, py1), (px2, py2), bgr_color, thickness)
|
||||
|
||||
|
||||
# Optional features
|
||||
if draw_endpoints:
|
||||
cv2.circle(img_np, (px1, py1), point_radius, bgr_color, -1)
|
||||
cv2.circle(img_np, (px2, py2), point_radius, bgr_color, -1)
|
||||
|
||||
|
||||
if draw_label and label_text:
|
||||
# Calculate label position
|
||||
if label_position == "Midpoint":
|
||||
@@ -238,43 +244,47 @@ class DrawingEngine:
|
||||
label_x, label_y = px1, py1
|
||||
else: # "End"
|
||||
label_x, label_y = px2, py2
|
||||
|
||||
|
||||
# Get text size
|
||||
text_size, baseline = cv2.getTextSize(label_text, cv2.FONT_HERSHEY_SIMPLEX, font_scale, 1)
|
||||
|
||||
|
||||
# Draw background rectangle
|
||||
cv2.rectangle(img_np,
|
||||
(label_x - 5, label_y - text_size[1] - 5),
|
||||
(label_x + text_size[0] + 5, label_y + 5),
|
||||
(50, 50, 50), -1)
|
||||
|
||||
cv2.rectangle(
|
||||
img_np,
|
||||
(label_x - 5, label_y - text_size[1] - 5),
|
||||
(label_x + text_size[0] + 5, label_y + 5),
|
||||
(50, 50, 50),
|
||||
-1,
|
||||
)
|
||||
|
||||
# Draw text
|
||||
cv2.putText(img_np, label_text, (label_x, label_y),
|
||||
cv2.FONT_HERSHEY_SIMPLEX, font_scale, (255, 255, 255), 1)
|
||||
|
||||
cv2.putText(img_np, label_text, (label_x, label_y), cv2.FONT_HERSHEY_SIMPLEX, font_scale, (255, 255, 255), 1)
|
||||
|
||||
# Convert back to tensor and update batch
|
||||
output_batch[b] = self.tensor_from_numpy(img_np, device, dtype)
|
||||
|
||||
|
||||
return output_batch
|
||||
|
||||
def draw_polygon(self,
|
||||
image: torch.Tensor,
|
||||
points_x: List[float],
|
||||
points_y: List[float],
|
||||
color: str = "#0000FF",
|
||||
thickness: int = 2,
|
||||
fill: bool = False,
|
||||
is_normalized: bool = True,
|
||||
batch_mapping: str = "broadcast",
|
||||
draw_vertices: bool = False,
|
||||
vertex_radius: int = 3,
|
||||
draw_label: bool = False,
|
||||
label_text: str = "",
|
||||
label_position: str = "Center",
|
||||
font_scale: float = 0.5) -> torch.Tensor:
|
||||
|
||||
def draw_polygon(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
points_x: List[float],
|
||||
points_y: List[float],
|
||||
color: str = "#0000FF",
|
||||
thickness: int = 2,
|
||||
fill: bool = False,
|
||||
is_normalized: bool = True,
|
||||
batch_mapping: str = "broadcast",
|
||||
draw_vertices: bool = False,
|
||||
vertex_radius: int = 3,
|
||||
draw_label: bool = False,
|
||||
label_text: str = "",
|
||||
label_position: str = "Center",
|
||||
font_scale: float = 0.5,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Draw polygon on images efficiently.
|
||||
|
||||
|
||||
Args:
|
||||
image: BHWC tensor
|
||||
points_x/y: Lists of polygon vertex coordinates
|
||||
@@ -284,46 +294,46 @@ class DrawingEngine:
|
||||
is_normalized: If True, coords are 0-1 normalized
|
||||
batch_mapping: How to map polygons to batch
|
||||
Other params: Styling options
|
||||
|
||||
|
||||
Returns:
|
||||
Modified image tensor
|
||||
"""
|
||||
batch_size, height, width, _ = image.shape
|
||||
output_batch = image.clone()
|
||||
bgr_color = self.hex_to_bgr(color)
|
||||
|
||||
|
||||
# Validate input lengths
|
||||
if len(points_x) != len(points_y):
|
||||
raise ValueError("X and Y coordinate lists must have the same length")
|
||||
|
||||
|
||||
# Ensure lists have at least 3 points for a polygon
|
||||
if len(points_x) < 3:
|
||||
logger.warning("Polygon needs at least 3 points. Drawing nothing.")
|
||||
return output_batch
|
||||
|
||||
|
||||
# Determine batch mapping
|
||||
if batch_mapping == "one-to-one" and batch_size > 1:
|
||||
logger.warning(f"One-to-one mapping not supported for polygons. Using broadcast.")
|
||||
batch_mapping = "broadcast"
|
||||
|
||||
|
||||
# Process each image in batch
|
||||
for b in range(batch_size):
|
||||
# Skip processing based on mapping strategy
|
||||
if batch_mapping == "all-on-first" and b > 0:
|
||||
continue
|
||||
|
||||
|
||||
# Single efficient conversion to numpy
|
||||
img_np, device, dtype = self.prep_batch_image(output_batch, b)
|
||||
|
||||
|
||||
# Convert polygon points to pixel coordinates
|
||||
pixel_points = []
|
||||
for i in range(len(points_x)):
|
||||
x_coord = points_x[i]
|
||||
y_coord = points_y[i]
|
||||
|
||||
|
||||
if not isinstance(x_coord, (float, int)) or not isinstance(y_coord, (float, int)):
|
||||
continue
|
||||
|
||||
|
||||
# Convert to pixel coordinates if needed
|
||||
if is_normalized:
|
||||
px = int(CoordinateSystem.denormalize(x_coord, width, CoordinateSystem.PIXEL))
|
||||
@@ -331,47 +341,56 @@ class DrawingEngine:
|
||||
else:
|
||||
px = int(max(0, min(width - 1, x_coord)))
|
||||
py = int(max(0, min(height - 1, y_coord)))
|
||||
|
||||
|
||||
pixel_points.append((px, py))
|
||||
|
||||
|
||||
# Draw vertices if requested
|
||||
if draw_vertices:
|
||||
cv2.circle(img_np, (px, py), vertex_radius, bgr_color, -1)
|
||||
|
||||
|
||||
# Skip if no valid points
|
||||
if len(pixel_points) < 3:
|
||||
continue
|
||||
|
||||
|
||||
# Convert to numpy array for OpenCV
|
||||
pts = np.array(pixel_points, dtype=np.int32)
|
||||
|
||||
|
||||
# Draw filled or outline polygon
|
||||
if fill:
|
||||
cv2.fillPoly(img_np, [pts], bgr_color)
|
||||
else:
|
||||
cv2.polylines(img_np, [pts], True, bgr_color, thickness)
|
||||
|
||||
|
||||
# Draw label if requested
|
||||
if draw_label and label_text:
|
||||
# Calculate label position (center of polygon)
|
||||
center_x = int(sum(p[0] for p in pixel_points) / len(pixel_points))
|
||||
center_y = int(sum(p[1] for p in pixel_points) / len(pixel_points))
|
||||
|
||||
|
||||
# Get text size
|
||||
text_size, baseline = cv2.getTextSize(label_text, cv2.FONT_HERSHEY_SIMPLEX, font_scale, 1)
|
||||
|
||||
|
||||
# Draw background rectangle
|
||||
cv2.rectangle(img_np,
|
||||
(center_x - text_size[0]//2 - 5, center_y - text_size[1]//2 - 5),
|
||||
(center_x + text_size[0]//2 + 5, center_y + text_size[1]//2 + 5),
|
||||
(50, 50, 50), -1)
|
||||
|
||||
cv2.rectangle(
|
||||
img_np,
|
||||
(center_x - text_size[0] // 2 - 5, center_y - text_size[1] // 2 - 5),
|
||||
(center_x + text_size[0] // 2 + 5, center_y + text_size[1] // 2 + 5),
|
||||
(50, 50, 50),
|
||||
-1,
|
||||
)
|
||||
|
||||
# Draw text
|
||||
cv2.putText(img_np, label_text,
|
||||
(center_x - text_size[0]//2, center_y + text_size[1]//2),
|
||||
cv2.FONT_HERSHEY_SIMPLEX, font_scale, (255, 255, 255), 1)
|
||||
|
||||
cv2.putText(
|
||||
img_np,
|
||||
label_text,
|
||||
(center_x - text_size[0] // 2, center_y + text_size[1] // 2),
|
||||
cv2.FONT_HERSHEY_SIMPLEX,
|
||||
font_scale,
|
||||
(255, 255, 255),
|
||||
1,
|
||||
)
|
||||
|
||||
# Convert back to tensor and update batch
|
||||
output_batch[b] = self.tensor_from_numpy(img_np, device, dtype)
|
||||
|
||||
return output_batch
|
||||
|
||||
return output_batch
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Source code for MediaPipe Vision tasks implementation."""
|
||||
|
||||
# Import utility modules
|
||||
from . import utils
|
||||
from . import utils
|
||||
|
||||
@@ -1,31 +1,21 @@
|
||||
import logging
|
||||
from typing import Optional, List, Any, Tuple, Union
|
||||
import math
|
||||
import torch
|
||||
import numpy as np
|
||||
import cv2
|
||||
import collections
|
||||
import typing # Add typing for Dict hint
|
||||
|
||||
#TODO: consider moving base nodes to src
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
# TODO: consider moving base nodes to src
|
||||
# Assuming types are correctly imported relative to this file's location
|
||||
from ..types import (
|
||||
FACE_LANDMARKS, HAND_LANDMARKS, POSE_LANDMARKS,
|
||||
LandmarkPoint, FaceLandmarksResult, HandLandmarksResult, PoseLandmarksResult
|
||||
)
|
||||
from ..types import LandmarkPoint
|
||||
|
||||
# Import from the new consolidated utilities
|
||||
from ...utils.math import scale_value
|
||||
from ...utils.general import AlwaysEqualProxy
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_POSITION_HISTORY = 50 # Matches MAX_LANDMARK_HISTORY for consistency
|
||||
FLOAT_EQUALITY_TOLERANCE = 1e-6 # Same tolerance as base_delta_nodes
|
||||
#TODO refactor to use CoordinateSystem
|
||||
MAX_POSITION_HISTORY = 50 # Matches MAX_LANDMARK_HISTORY for consistency
|
||||
FLOAT_EQUALITY_TOLERANCE = 1e-6 # Same tolerance as base_delta_nodes
|
||||
|
||||
|
||||
# TODO refactor to use CoordinateSystem
|
||||
class LandmarkPositionBase:
|
||||
#NOTE: use x,y,and z for the landmark position rather than LandmarkPoint object for max flexibility
|
||||
# NOTE: use x,y,and z for the landmark position rather than LandmarkPoint object for max flexibility
|
||||
"""
|
||||
Base class for extracting position lists from specific landmark types.
|
||||
Outputs values corresponding to the input batch size, using defaults for missing items.
|
||||
@@ -33,27 +23,30 @@ class LandmarkPositionBase:
|
||||
Processes only the detected object at the specified result_index per batch item.
|
||||
RETURN_TYPES declare the semantic type (FLOAT/BOOLEAN); the actual return value may be a list.
|
||||
"""
|
||||
|
||||
FUNCTION = "extract_positions"
|
||||
# Use base types for semantic declaration
|
||||
RETURN_TYPES = ("FLOAT", "FLOAT", "FLOAT", "FLOAT", "FLOAT", "BOOLEAN")
|
||||
RETURN_NAMES = ("x", "y", "z", "visibility", "presence", "is_valid") # Singular names
|
||||
CATEGORY = "Realtime Nodes/MediaPipe Vision/Common/Utils"
|
||||
RETURN_NAMES = ("x", "y", "z", "visibility", "presence", "is_valid") # Singular names
|
||||
CATEGORY = "Realtime Nodes/MediaPipe Vision/Common/Utils"
|
||||
|
||||
DEFAULT_VALUES = (0.0, 0.0, 0.0, 0.0, 0.0)
|
||||
LANDMARKS_TYPE = "ANY_LANDMARKS" # Placeholder
|
||||
LANDMARKS_TYPE = "ANY_LANDMARKS" # Placeholder
|
||||
|
||||
# INPUT_TYPES are defined in subclasses
|
||||
|
||||
def extract_positions(self, landmarks: list, landmark_index: int, result_index: int = 0, use_world_coordinates: bool = False) -> Tuple[List[float], List[float], List[float], List[float], List[float], List[bool]]:
|
||||
def extract_positions(
|
||||
self, landmarks: list, landmark_index: int, result_index: int = 0, use_world_coordinates: bool = False
|
||||
) -> Tuple[List[float], List[float], List[float], List[float], List[float], List[bool]]:
|
||||
"""Extracts position and properties for the specified landmark across the entire batch,
|
||||
padding with defaults and providing validity flags for missing items."""
|
||||
padding with defaults and providing validity flags for missing items."""
|
||||
|
||||
x_coords = []
|
||||
y_coords = []
|
||||
z_coords = []
|
||||
vis_coords = []
|
||||
pres_coords = []
|
||||
is_valid_list = [] # New list to track validity
|
||||
is_valid_list = [] # New list to track validity
|
||||
|
||||
if not isinstance(landmarks, list):
|
||||
raise TypeError(f"{self.__class__.__name__}: Input landmarks is not a list. Received type: {type(landmarks)}.")
|
||||
@@ -63,12 +56,12 @@ class LandmarkPositionBase:
|
||||
# Iterate through the batch (List[...])
|
||||
for batch_idx in range(batch_size):
|
||||
item_list = landmarks[batch_idx]
|
||||
|
||||
|
||||
current_x, current_y, current_z, current_vis, current_pres = self.DEFAULT_VALUES
|
||||
is_valid = False # Assume invalid until proven otherwise
|
||||
is_valid = False # Assume invalid until proven otherwise
|
||||
landmark_point: Optional[LandmarkPoint] = None
|
||||
target_landmarks_list: Optional[List[LandmarkPoint]] = None
|
||||
|
||||
|
||||
# Check if item_list is valid and contains at least one result item
|
||||
if isinstance(item_list, list) and item_list:
|
||||
# Get the result at the specified index if it exists
|
||||
@@ -77,21 +70,25 @@ class LandmarkPositionBase:
|
||||
|
||||
if result_item is not None:
|
||||
try:
|
||||
# --- Determine Structure and Extract Landmark List ---
|
||||
# --- Determine Structure and Extract Landmark List ---
|
||||
# Case 1: result_item is the list itself (FACE_LANDMARKS)
|
||||
if isinstance(result_item, list):
|
||||
target_landmarks_list = result_item
|
||||
if use_world_coordinates and self.LANDMARKS_TYPE == "FACE_LANDMARKS":
|
||||
logger.warning(f"World coordinates requested but not available for FACE_LANDMARKS. Using image coordinates.")
|
||||
logger.warning(
|
||||
f"World coordinates requested but not available for FACE_LANDMARKS. Using image coordinates."
|
||||
)
|
||||
|
||||
# Case 2: result_item is an object with .landmarks (HAND/POSE_LANDMARKS)
|
||||
elif hasattr(result_item, 'landmarks'):
|
||||
landmarks_attr = getattr(result_item, 'landmarks', None)
|
||||
elif hasattr(result_item, "landmarks"):
|
||||
landmarks_attr = getattr(result_item, "landmarks", None)
|
||||
if landmarks_attr is not None:
|
||||
use_world = (use_world_coordinates and
|
||||
self.LANDMARKS_TYPE != "FACE_LANDMARKS" and
|
||||
hasattr(result_item, 'world_landmarks') and
|
||||
getattr(result_item, 'world_landmarks', None))
|
||||
use_world = (
|
||||
use_world_coordinates
|
||||
and self.LANDMARKS_TYPE != "FACE_LANDMARKS"
|
||||
and hasattr(result_item, "world_landmarks")
|
||||
and getattr(result_item, "world_landmarks", None)
|
||||
)
|
||||
if use_world:
|
||||
target_landmarks_list = result_item.world_landmarks
|
||||
else:
|
||||
@@ -101,22 +98,27 @@ class LandmarkPositionBase:
|
||||
# --- Find Landmark Point by Index ---
|
||||
if isinstance(target_landmarks_list, list) and target_landmarks_list:
|
||||
landmark_point = next((lm for lm in target_landmarks_list if lm.index == landmark_index), None)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"{self.__class__.__name__}: Unexpected error processing batch item {batch_idx}: {e}", exc_info=True)
|
||||
# Keep landmark_point as None, is_valid as False
|
||||
logger.error(
|
||||
f"{self.__class__.__name__}: Unexpected error processing batch item {batch_idx}: {e}",
|
||||
exc_info=True,
|
||||
)
|
||||
# Keep landmark_point as None, is_valid as False
|
||||
else:
|
||||
logger.debug(f"{self.__class__.__name__}: Result index {result_index} exceeds available detections for batch item {batch_idx} (has {len(item_list)})")
|
||||
logger.debug(
|
||||
f"{self.__class__.__name__}: Result index {result_index} exceeds available detections for batch item {batch_idx} (has {len(item_list)})"
|
||||
)
|
||||
# else: item_list was not a list or was empty, keep defaults/invalid
|
||||
|
||||
# --- Extract Values or Use Defaults, Set Validity ---
|
||||
if landmark_point:
|
||||
is_valid = True # Mark as valid since we found the point
|
||||
is_valid = True # Mark as valid since we found the point
|
||||
current_x = landmark_point.x
|
||||
current_y = landmark_point.y
|
||||
current_z = landmark_point.z
|
||||
current_vis = getattr(landmark_point, 'visibility', None)
|
||||
current_pres = getattr(landmark_point, 'presence', None)
|
||||
current_vis = getattr(landmark_point, "visibility", None)
|
||||
current_pres = getattr(landmark_point, "presence", None)
|
||||
current_vis = current_vis if current_vis is not None else 0.0
|
||||
current_pres = current_pres if current_pres is not None else 0.0
|
||||
# else: defaults are used, is_valid remains False
|
||||
@@ -132,5 +134,5 @@ class LandmarkPositionBase:
|
||||
# Check if each list has only one element, if so return single values instead of lists
|
||||
if len(x_coords) == 1:
|
||||
return (x_coords[0], y_coords[0], z_coords[0], vis_coords[0], pres_coords[0], is_valid_list[0])
|
||||
|
||||
|
||||
return (x_coords, y_coords, z_coords, vis_coords, pres_coords, is_valid_list)
|
||||
|
||||
@@ -1 +1 @@
|
||||
"""Face Detection implementation using MediaPipe."""
|
||||
"""Face Detection implementation using MediaPipe."""
|
||||
|
||||
@@ -3,16 +3,20 @@
|
||||
This module contains the implementation of MediaPipe Face Detection functionality.
|
||||
"""
|
||||
|
||||
from typing import List
|
||||
|
||||
import mediapipe as mp
|
||||
import numpy as np
|
||||
import torch
|
||||
from mediapipe.tasks import python
|
||||
from mediapipe.tasks.python import vision
|
||||
from mediapipe.tasks.python.core.base_options import BaseOptions
|
||||
from typing import List, Tuple
|
||||
|
||||
from ...utils.timing import TimestampProvider # Import from utils
|
||||
|
||||
# Import new types
|
||||
from ..types import FaceDetectionResult, BoundingBox, FaceKeypoint
|
||||
from ...utils.timing import TimestampProvider # Import from utils
|
||||
from ..types import BoundingBox, FaceDetectionResult, FaceKeypoint
|
||||
|
||||
|
||||
class FaceDetector:
|
||||
"""Detects faces in an image using MediaPipe FaceDetector."""
|
||||
@@ -28,28 +32,33 @@ class FaceDetector:
|
||||
|
||||
self.model_path = model_path
|
||||
self._detector_instance = None
|
||||
self._current_options = None # Store current options for comparison
|
||||
self._timestamp_provider = None # Initialize TimestampProvider later
|
||||
self._current_options = None # Store current options for comparison
|
||||
self._timestamp_provider = None # Initialize TimestampProvider later
|
||||
|
||||
def _initialize_detector(self, min_detection_confidence: float, running_mode: str, delegate: str):
|
||||
"""Initializes the FaceDetector."""
|
||||
# Map string mode to MediaPipe enum
|
||||
mode_enum = vision.RunningMode.IMAGE if running_mode == "image" else vision.RunningMode.VIDEO
|
||||
delegate_enum = BaseOptions.Delegate.CPU if delegate.lower() == 'cpu' else BaseOptions.Delegate.GPU
|
||||
|
||||
delegate_enum = BaseOptions.Delegate.CPU if delegate.lower() == "cpu" else BaseOptions.Delegate.GPU
|
||||
|
||||
base_options = python.BaseOptions(model_asset_path=self.model_path, delegate=delegate_enum)
|
||||
options = vision.FaceDetectorOptions(
|
||||
base_options=base_options,
|
||||
running_mode=mode_enum, # Use enum mode
|
||||
min_detection_confidence=min_detection_confidence
|
||||
running_mode=mode_enum, # Use enum mode
|
||||
min_detection_confidence=min_detection_confidence,
|
||||
)
|
||||
self._current_options = options # Store for comparison
|
||||
self._current_options = options # Store for comparison
|
||||
# Only create timestamp provider for VIDEO mode
|
||||
self._timestamp_provider = TimestampProvider() if mode_enum == vision.RunningMode.VIDEO else None
|
||||
self._timestamp_provider = TimestampProvider() if mode_enum == vision.RunningMode.VIDEO else None
|
||||
return vision.FaceDetector.create_from_options(options)
|
||||
|
||||
def detect(self, image: torch.Tensor, min_detection_confidence: float = 0.5,
|
||||
running_mode: str = "video", delegate: str = 'cpu') -> List[List[FaceDetectionResult]]: # Added running_mode
|
||||
def detect(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
min_detection_confidence: float = 0.5,
|
||||
running_mode: str = "video",
|
||||
delegate: str = "cpu",
|
||||
) -> List[List[FaceDetectionResult]]: # Added running_mode
|
||||
"""Detects faces in the input image tensor.
|
||||
|
||||
Returns:
|
||||
@@ -63,15 +72,15 @@ class FaceDetector:
|
||||
new_options_tuple = (min_detection_confidence, running_mode, delegate.lower())
|
||||
current_options_tuple = None
|
||||
if self._current_options:
|
||||
current_options_tuple = (
|
||||
self._current_options.min_detection_confidence,
|
||||
"image" if self._current_options.running_mode == vision.RunningMode.IMAGE else "video", # Map enum back to string
|
||||
'cpu' if self._current_options.base_options.delegate == BaseOptions.Delegate.CPU else 'gpu'
|
||||
current_options_tuple = (
|
||||
self._current_options.min_detection_confidence,
|
||||
"image" if self._current_options.running_mode == vision.RunningMode.IMAGE else "video", # Map enum back to string
|
||||
"cpu" if self._current_options.base_options.delegate == BaseOptions.Delegate.CPU else "gpu",
|
||||
)
|
||||
|
||||
if self._detector_instance is None or new_options_tuple != current_options_tuple:
|
||||
# Close previous instance if it exists and has a close method
|
||||
if self._detector_instance and hasattr(self._detector_instance, 'close'):
|
||||
if self._detector_instance and hasattr(self._detector_instance, "close"):
|
||||
self._detector_instance.close()
|
||||
self._detector_instance = self._initialize_detector(min_detection_confidence, running_mode, delegate)
|
||||
|
||||
@@ -82,7 +91,7 @@ class FaceDetector:
|
||||
detect_func = self._detector_instance.detect_for_video if is_video_mode else self._detector_instance.detect
|
||||
|
||||
for i in range(batch_size):
|
||||
img_tensor = image[i] # HWC
|
||||
img_tensor = image[i] # HWC
|
||||
np_image = (img_tensor.cpu().numpy() * 255).astype(np.uint8)
|
||||
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=np_image)
|
||||
|
||||
@@ -93,39 +102,23 @@ class FaceDetector:
|
||||
else:
|
||||
# Call image mode function
|
||||
detection_result = detect_func(mp_image)
|
||||
|
||||
|
||||
current_image_detections = []
|
||||
if detection_result and detection_result.detections: # Check if result is not None
|
||||
if detection_result and detection_result.detections: # Check if result is not None
|
||||
for detection in detection_result.detections:
|
||||
bbox_mp = detection.bounding_box
|
||||
keypoints_mp = detection.keypoints
|
||||
score = detection.categories[0].score if detection.categories else None # Face detection usually has one score
|
||||
|
||||
bbox = BoundingBox(
|
||||
origin_x=bbox_mp.origin_x,
|
||||
origin_y=bbox_mp.origin_y,
|
||||
width=bbox_mp.width,
|
||||
height=bbox_mp.height
|
||||
)
|
||||
|
||||
score = detection.categories[0].score if detection.categories else None # Face detection usually has one score
|
||||
|
||||
bbox = BoundingBox(origin_x=bbox_mp.origin_x, origin_y=bbox_mp.origin_y, width=bbox_mp.width, height=bbox_mp.height)
|
||||
|
||||
keypoints = None
|
||||
if keypoints_mp:
|
||||
keypoints = [
|
||||
FaceKeypoint(
|
||||
label=kp.label,
|
||||
x=kp.x,
|
||||
y=kp.y
|
||||
)
|
||||
for kp in keypoints_mp
|
||||
]
|
||||
keypoints = [FaceKeypoint(label=kp.label, x=kp.x, y=kp.y) for kp in keypoints_mp]
|
||||
|
||||
current_image_detections.append(FaceDetectionResult(bounding_box=bbox, keypoints=keypoints, score=score))
|
||||
|
||||
current_image_detections.append(FaceDetectionResult(
|
||||
bounding_box=bbox,
|
||||
keypoints=keypoints,
|
||||
score=score
|
||||
))
|
||||
|
||||
batch_results.append(current_image_detections)
|
||||
|
||||
# Check MediaPipe documentation if resource cleanup is required here.
|
||||
return batch_results
|
||||
return batch_results
|
||||
|
||||
@@ -1 +1 @@
|
||||
"""Face Landmark Detection implementation using MediaPipe."""
|
||||
"""Face Landmark Detection implementation using MediaPipe."""
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import mediapipe as mp
|
||||
import numpy as np
|
||||
import torch
|
||||
from mediapipe.tasks import python
|
||||
from mediapipe.tasks.python import vision
|
||||
from mediapipe.tasks.python.core.base_options import BaseOptions
|
||||
from typing import List, Tuple, Optional, Dict
|
||||
|
||||
from ...utils.timing import TimestampProvider # Import TimestampProvider
|
||||
|
||||
# Import new types
|
||||
from ..types import LandmarkPoint, FaceLandmarksResult
|
||||
from ...utils.timing import TimestampProvider # Import TimestampProvider
|
||||
from ..types import LandmarkPoint
|
||||
|
||||
|
||||
class FaceLandmarkDetector:
|
||||
"""Detects face landmarks and blendshapes using MediaPipe FaceLandmarker."""
|
||||
@@ -22,38 +26,53 @@ class FaceLandmarkDetector:
|
||||
raise ValueError("A valid model_path must be provided.")
|
||||
|
||||
self.model_path = model_path
|
||||
self._detector_instance = None
|
||||
self._current_options = None # Store current options for comparison
|
||||
self._timestamp_provider = None # Added
|
||||
self._detector_instance = None
|
||||
self._current_options = None # Store current options for comparison
|
||||
self._timestamp_provider = None # Added
|
||||
|
||||
def _initialize_detector(self, num_faces: int, min_detection_confidence: float,
|
||||
min_presence_confidence: float, min_tracking_confidence: float,
|
||||
output_blendshapes: bool, output_transform_matrix: bool,
|
||||
running_mode: str, delegate: str): # Added running_mode
|
||||
def _initialize_detector(
|
||||
self,
|
||||
num_faces: int,
|
||||
min_detection_confidence: float,
|
||||
min_presence_confidence: float,
|
||||
min_tracking_confidence: float,
|
||||
output_blendshapes: bool,
|
||||
output_transform_matrix: bool,
|
||||
running_mode: str,
|
||||
delegate: str,
|
||||
): # Added running_mode
|
||||
"""Initializes the FaceLandmarker detector."""
|
||||
mode_enum = vision.RunningMode.IMAGE if running_mode == "image" else vision.RunningMode.VIDEO # Added
|
||||
delegate_enum = BaseOptions.Delegate.CPU if delegate.lower() == 'cpu' else BaseOptions.Delegate.GPU
|
||||
mode_enum = vision.RunningMode.IMAGE if running_mode == "image" else vision.RunningMode.VIDEO # Added
|
||||
delegate_enum = BaseOptions.Delegate.CPU if delegate.lower() == "cpu" else BaseOptions.Delegate.GPU
|
||||
base_options = python.BaseOptions(model_asset_path=self.model_path, delegate=delegate_enum)
|
||||
options = vision.FaceLandmarkerOptions(
|
||||
base_options=base_options,
|
||||
running_mode=mode_enum, # Use enum mode
|
||||
running_mode=mode_enum, # Use enum mode
|
||||
num_faces=num_faces,
|
||||
min_face_detection_confidence=min_detection_confidence,
|
||||
# min_face_presence_confidence and min_tracking_confidence are for VIDEO/LIVE_STREAM
|
||||
min_face_presence_confidence=min_presence_confidence,
|
||||
min_tracking_confidence=min_tracking_confidence,
|
||||
min_face_presence_confidence=min_presence_confidence,
|
||||
min_tracking_confidence=min_tracking_confidence,
|
||||
output_face_blendshapes=output_blendshapes,
|
||||
output_facial_transformation_matrixes=output_transform_matrix
|
||||
output_facial_transformation_matrixes=output_transform_matrix,
|
||||
)
|
||||
self._current_options = options # Store for comparison
|
||||
self._current_options = options # Store for comparison
|
||||
# Only create timestamp provider for VIDEO mode
|
||||
self._timestamp_provider = TimestampProvider() if mode_enum == vision.RunningMode.VIDEO else None
|
||||
self._timestamp_provider = TimestampProvider() if mode_enum == vision.RunningMode.VIDEO else None
|
||||
return vision.FaceLandmarker.create_from_options(options)
|
||||
|
||||
def detect(self, image: torch.Tensor, num_faces: int = 1, min_detection_confidence: float = 0.5,
|
||||
min_presence_confidence: float = 0.5, min_tracking_confidence: float = 0.5,
|
||||
output_blendshapes: bool = False, output_transform_matrix: bool = False,
|
||||
running_mode: str = "video", delegate: str = 'cpu') -> Tuple[List[List[LandmarkPoint]], Optional[List[Dict[str, float]]], Optional[List[np.ndarray]]]: # Added running_mode
|
||||
def detect(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
num_faces: int = 1,
|
||||
min_detection_confidence: float = 0.5,
|
||||
min_presence_confidence: float = 0.5,
|
||||
min_tracking_confidence: float = 0.5,
|
||||
output_blendshapes: bool = False,
|
||||
output_transform_matrix: bool = False,
|
||||
running_mode: str = "video",
|
||||
delegate: str = "cpu",
|
||||
) -> Tuple[List[List[LandmarkPoint]], Optional[List[Dict[str, float]]], Optional[List[np.ndarray]]]: # Added running_mode
|
||||
"""Detects face landmarks in the input image tensor.
|
||||
|
||||
Returns:
|
||||
@@ -64,32 +83,46 @@ class FaceLandmarkDetector:
|
||||
"""
|
||||
if image.dim() != 4:
|
||||
raise ValueError("Input tensor must be in BHWC format.")
|
||||
|
||||
# Check if options changed
|
||||
|
||||
# Check if options changed
|
||||
# Add running_mode to tuple check
|
||||
new_options_tuple = (num_faces, min_detection_confidence, min_presence_confidence, min_tracking_confidence,
|
||||
output_blendshapes, output_transform_matrix, running_mode, delegate.lower())
|
||||
new_options_tuple = (
|
||||
num_faces,
|
||||
min_detection_confidence,
|
||||
min_presence_confidence,
|
||||
min_tracking_confidence,
|
||||
output_blendshapes,
|
||||
output_transform_matrix,
|
||||
running_mode,
|
||||
delegate.lower(),
|
||||
)
|
||||
current_options_tuple = None
|
||||
if self._current_options:
|
||||
current_options_tuple = (
|
||||
self._current_options.num_faces,
|
||||
self._current_options.min_face_detection_confidence,
|
||||
self._current_options.min_face_presence_confidence,
|
||||
self._current_options.min_tracking_confidence,
|
||||
self._current_options.min_face_presence_confidence,
|
||||
self._current_options.min_tracking_confidence,
|
||||
self._current_options.output_face_blendshapes,
|
||||
self._current_options.output_facial_transformation_matrixes,
|
||||
"image" if self._current_options.running_mode == vision.RunningMode.IMAGE else "video", # Map enum back to string
|
||||
'cpu' if self._current_options.base_options.delegate == BaseOptions.Delegate.CPU else 'gpu'
|
||||
"image" if self._current_options.running_mode == vision.RunningMode.IMAGE else "video", # Map enum back to string
|
||||
"cpu" if self._current_options.base_options.delegate == BaseOptions.Delegate.CPU else "gpu",
|
||||
)
|
||||
|
||||
if self._detector_instance is None or new_options_tuple != current_options_tuple:
|
||||
if self._detector_instance and hasattr(self._detector_instance, 'close'): # Close previous instance if options changed
|
||||
self._detector_instance.close()
|
||||
if self._detector_instance and hasattr(self._detector_instance, "close"): # Close previous instance if options changed
|
||||
self._detector_instance.close()
|
||||
# Pass new params to initializer
|
||||
self._detector_instance = self._initialize_detector(num_faces, min_detection_confidence,
|
||||
min_presence_confidence, min_tracking_confidence,
|
||||
output_blendshapes, output_transform_matrix,
|
||||
running_mode, delegate) # Pass running_mode
|
||||
self._detector_instance = self._initialize_detector(
|
||||
num_faces,
|
||||
min_detection_confidence,
|
||||
min_presence_confidence,
|
||||
min_tracking_confidence,
|
||||
output_blendshapes,
|
||||
output_transform_matrix,
|
||||
running_mode,
|
||||
delegate,
|
||||
) # Pass running_mode
|
||||
|
||||
batch_size = image.shape[0]
|
||||
batch_results_landmarks = []
|
||||
@@ -99,9 +132,9 @@ class FaceLandmarkDetector:
|
||||
# Determine the correct detection function based on mode
|
||||
is_video_mode = self._current_options.running_mode == vision.RunningMode.VIDEO
|
||||
detect_func = self._detector_instance.detect_for_video if is_video_mode else self._detector_instance.detect
|
||||
|
||||
|
||||
for i in range(batch_size):
|
||||
img_tensor = image[i]
|
||||
img_tensor = image[i]
|
||||
np_image = (img_tensor.cpu().numpy() * 255).astype(np.uint8)
|
||||
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=np_image)
|
||||
|
||||
@@ -110,24 +143,30 @@ class FaceLandmarkDetector:
|
||||
detection_result = detect_func(mp_image, timestamp_ms)
|
||||
else:
|
||||
detection_result = detect_func(mp_image)
|
||||
|
||||
|
||||
current_image_landmarks = []
|
||||
current_image_blendshapes = [] if output_blendshapes else None
|
||||
current_image_matrices = [] if output_transform_matrix else None
|
||||
if detection_result and detection_result.face_landmarks: # Added check for detection_result
|
||||
if detection_result and detection_result.face_landmarks: # Added check for detection_result
|
||||
for face_idx, face_landmarks_mp in enumerate(detection_result.face_landmarks):
|
||||
landmarks = [LandmarkPoint(index=lm_idx, x=lm.x, y=lm.y, z=lm.z) for lm_idx, lm in enumerate(face_landmarks_mp)]
|
||||
current_image_landmarks.append(landmarks)
|
||||
if output_blendshapes and detection_result.face_blendshapes and face_idx < len(detection_result.face_blendshapes):
|
||||
blendshapes = {bs.category_name: bs.score for bs in detection_result.face_blendshapes[face_idx]}
|
||||
current_image_blendshapes.append(blendshapes)
|
||||
if output_transform_matrix and detection_result.facial_transformation_matrixes and face_idx < len(detection_result.facial_transformation_matrixes):
|
||||
current_image_matrices.append(detection_result.facial_transformation_matrixes[face_idx])
|
||||
blendshapes = {bs.category_name: bs.score for bs in detection_result.face_blendshapes[face_idx]}
|
||||
current_image_blendshapes.append(blendshapes)
|
||||
if (
|
||||
output_transform_matrix
|
||||
and detection_result.facial_transformation_matrixes
|
||||
and face_idx < len(detection_result.facial_transformation_matrixes)
|
||||
):
|
||||
current_image_matrices.append(detection_result.facial_transformation_matrixes[face_idx])
|
||||
batch_results_landmarks.append(current_image_landmarks)
|
||||
if output_blendshapes: batch_results_blendshapes.append(current_image_blendshapes)
|
||||
if output_transform_matrix: batch_results_matrices.append(current_image_matrices)
|
||||
|
||||
if output_blendshapes:
|
||||
batch_results_blendshapes.append(current_image_blendshapes)
|
||||
if output_transform_matrix:
|
||||
batch_results_matrices.append(current_image_matrices)
|
||||
|
||||
# Close detector after batch? Landmarker might need closing per batch? Check docs.
|
||||
# For IMAGE mode, closing after batch seems okay.
|
||||
# self._detector_instance.close() # Close is handled if re-initialized
|
||||
return batch_results_landmarks, batch_results_blendshapes, batch_results_matrices
|
||||
return batch_results_landmarks, batch_results_blendshapes, batch_results_matrices
|
||||
|
||||
@@ -1 +1 @@
|
||||
"""Face Stylization implementation using MediaPipe."""
|
||||
"""Face Stylization implementation using MediaPipe."""
|
||||
|
||||
@@ -3,9 +3,10 @@ import numpy as np
|
||||
import torch
|
||||
from mediapipe.tasks import python
|
||||
from mediapipe.tasks.python import vision
|
||||
from typing import List, Optional
|
||||
|
||||
from ...utils.timing import TimestampProvider
|
||||
|
||||
|
||||
class FaceStylizer:
|
||||
"""Applies stylization to faces using MediaPipe FaceStylizer."""
|
||||
|
||||
@@ -27,12 +28,12 @@ class FaceStylizer:
|
||||
options = vision.FaceStylizerOptions(
|
||||
base_options=base_options
|
||||
# running_mode=vision.RunningMode.VIDEO # Not supported
|
||||
)
|
||||
)
|
||||
self.current_options = ()
|
||||
self._timestamp_provider = TimestampProvider()
|
||||
return vision.FaceStylizer.create_from_options(options)
|
||||
|
||||
def stylize(self, image: torch.Tensor) -> torch.Tensor: # Output is IMAGE
|
||||
def stylize(self, image: torch.Tensor) -> torch.Tensor: # Output is IMAGE
|
||||
"""Applies face stylization to the input image tensor(s)."""
|
||||
if image.dim() != 4:
|
||||
raise ValueError("Input tensor must be in BHWC format.")
|
||||
@@ -44,24 +45,24 @@ class FaceStylizer:
|
||||
batch_results = []
|
||||
|
||||
for i in range(batch_size):
|
||||
img_tensor = image[i] # HWC
|
||||
img_tensor = image[i] # HWC
|
||||
np_image = (img_tensor.cpu().numpy() * 255).astype(np.uint8)
|
||||
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=np_image)
|
||||
|
||||
stylization_result = self.detector_instance.stylize(mp_image)
|
||||
|
||||
|
||||
output_image_tensor = None
|
||||
if stylization_result:
|
||||
# Convert MediaPipe Image back to NumPy (HWC, uint8)
|
||||
stylized_np = stylization_result.numpy_view()
|
||||
# Convert to float tensor (HWC, 0-1)
|
||||
output_image_tensor = torch.from_numpy(stylized_np).float().div(255.0)
|
||||
# Convert MediaPipe Image back to NumPy (HWC, uint8)
|
||||
stylized_np = stylization_result.numpy_view()
|
||||
# Convert to float tensor (HWC, 0-1)
|
||||
output_image_tensor = torch.from_numpy(stylized_np).float().div(255.0)
|
||||
else:
|
||||
# Handle cases where stylization fails (e.g., no face detected)
|
||||
# Return original image tensor for this batch item
|
||||
output_image_tensor = img_tensor
|
||||
|
||||
# Handle cases where stylization fails (e.g., no face detected)
|
||||
# Return original image tensor for this batch item
|
||||
output_image_tensor = img_tensor
|
||||
|
||||
batch_results.append(output_image_tensor)
|
||||
|
||||
output_batch_tensor = torch.stack(batch_results, dim=0)
|
||||
return output_batch_tensor
|
||||
return output_batch_tensor
|
||||
|
||||
@@ -1 +1 @@
|
||||
"""Gesture Recognition implementation using MediaPipe."""
|
||||
"""Gesture Recognition implementation using MediaPipe."""
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
from typing import List
|
||||
|
||||
import mediapipe as mp
|
||||
import numpy as np
|
||||
import torch
|
||||
from mediapipe.tasks import python
|
||||
from mediapipe.tasks.python import vision
|
||||
from mediapipe.tasks.python.core.base_options import BaseOptions
|
||||
from typing import List
|
||||
|
||||
from ...utils.timing import TimestampProvider # Import TimestampProvider
|
||||
|
||||
# Import new types
|
||||
from ..types import GestureRecognitionResult, GestureCategory
|
||||
from ...utils.timing import TimestampProvider # Import TimestampProvider
|
||||
from ..types import GestureCategory, GestureRecognitionResult
|
||||
|
||||
|
||||
class GestureRecognizer:
|
||||
"""Recognizes hand gestures using MediaPipe GestureRecognizer."""
|
||||
@@ -18,57 +22,81 @@ class GestureRecognizer:
|
||||
self.model_path = model_path
|
||||
self._detector_instance = None
|
||||
self._current_options = None
|
||||
self._timestamp_provider = None # Added
|
||||
self._timestamp_provider = None # Added
|
||||
|
||||
def _initialize_detector(self, num_hands: int, min_detection_confidence: float,
|
||||
min_tracking_confidence: float, min_presence_confidence: float,
|
||||
running_mode: str, delegate: str): # Added running_mode
|
||||
mode_enum = vision.RunningMode.IMAGE if running_mode == "image" else vision.RunningMode.VIDEO # Added
|
||||
delegate_enum = BaseOptions.Delegate.CPU if delegate.lower() == 'cpu' else BaseOptions.Delegate.GPU
|
||||
def _initialize_detector(
|
||||
self,
|
||||
num_hands: int,
|
||||
min_detection_confidence: float,
|
||||
min_tracking_confidence: float,
|
||||
min_presence_confidence: float,
|
||||
running_mode: str,
|
||||
delegate: str,
|
||||
): # Added running_mode
|
||||
mode_enum = vision.RunningMode.IMAGE if running_mode == "image" else vision.RunningMode.VIDEO # Added
|
||||
delegate_enum = BaseOptions.Delegate.CPU if delegate.lower() == "cpu" else BaseOptions.Delegate.GPU
|
||||
base_options = python.BaseOptions(model_asset_path=self.model_path, delegate=delegate_enum)
|
||||
# Note: GestureRecognizerOptions has min_tracking_confidence, useful for VIDEO mode
|
||||
options = vision.GestureRecognizerOptions(
|
||||
base_options=base_options,
|
||||
running_mode=mode_enum, # Use enum mode
|
||||
running_mode=mode_enum, # Use enum mode
|
||||
num_hands=num_hands,
|
||||
min_hand_detection_confidence=min_detection_confidence,
|
||||
min_tracking_confidence=min_tracking_confidence,
|
||||
min_hand_presence_confidence=min_presence_confidence
|
||||
min_tracking_confidence=min_tracking_confidence,
|
||||
min_hand_presence_confidence=min_presence_confidence,
|
||||
# Add other relevant VIDEO mode parameters if needed (e.g., canned_gestures_classifier_options)
|
||||
)
|
||||
self._current_options = options
|
||||
# Only create timestamp provider for VIDEO mode
|
||||
self._timestamp_provider = TimestampProvider() if mode_enum == vision.RunningMode.VIDEO else None
|
||||
self._timestamp_provider = TimestampProvider() if mode_enum == vision.RunningMode.VIDEO else None
|
||||
return vision.GestureRecognizer.create_from_options(options)
|
||||
|
||||
def recognize(self, image: torch.Tensor, num_hands: int = 2, min_detection_confidence: float = 0.5,
|
||||
min_tracking_confidence: float = 0.5, min_presence_confidence: float = 0.5,
|
||||
running_mode: str = "video", delegate: str = 'cpu') -> List[List[GestureRecognitionResult]]: # Added running_mode
|
||||
def recognize(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
num_hands: int = 2,
|
||||
min_detection_confidence: float = 0.5,
|
||||
min_tracking_confidence: float = 0.5,
|
||||
min_presence_confidence: float = 0.5,
|
||||
running_mode: str = "video",
|
||||
delegate: str = "cpu",
|
||||
) -> List[List[GestureRecognitionResult]]: # Added running_mode
|
||||
"""Recognizes hand gestures in the input image tensor."""
|
||||
if image.dim() != 4:
|
||||
raise ValueError("Input tensor must be in BHWC format.")
|
||||
|
||||
|
||||
# Add running_mode to tuple check
|
||||
new_options_tuple = (num_hands, min_detection_confidence, min_tracking_confidence,
|
||||
min_presence_confidence, running_mode, delegate.lower())
|
||||
new_options_tuple = (
|
||||
num_hands,
|
||||
min_detection_confidence,
|
||||
min_tracking_confidence,
|
||||
min_presence_confidence,
|
||||
running_mode,
|
||||
delegate.lower(),
|
||||
)
|
||||
current_options_tuple = None
|
||||
if self._current_options:
|
||||
current_options_tuple = (
|
||||
self._current_options.num_hands,
|
||||
self._current_options.min_hand_detection_confidence,
|
||||
self._current_options.min_tracking_confidence,
|
||||
self._current_options.min_hand_presence_confidence,
|
||||
"image" if self._current_options.running_mode == vision.RunningMode.IMAGE else "video", # Map enum back to string
|
||||
'cpu' if self._current_options.base_options.delegate == BaseOptions.Delegate.CPU else 'gpu'
|
||||
)
|
||||
current_options_tuple = (
|
||||
self._current_options.num_hands,
|
||||
self._current_options.min_hand_detection_confidence,
|
||||
self._current_options.min_tracking_confidence,
|
||||
self._current_options.min_hand_presence_confidence,
|
||||
"image" if self._current_options.running_mode == vision.RunningMode.IMAGE else "video", # Map enum back to string
|
||||
"cpu" if self._current_options.base_options.delegate == BaseOptions.Delegate.CPU else "gpu",
|
||||
)
|
||||
|
||||
if self._detector_instance is None or new_options_tuple != current_options_tuple:
|
||||
if self._detector_instance and hasattr(self._detector_instance, 'close'):
|
||||
self._detector_instance.close()
|
||||
if self._detector_instance and hasattr(self._detector_instance, "close"):
|
||||
self._detector_instance.close()
|
||||
# Pass new params to initializer
|
||||
self._detector_instance = self._initialize_detector(num_hands, min_detection_confidence,
|
||||
min_tracking_confidence, min_presence_confidence,
|
||||
running_mode, delegate) # Pass running_mode
|
||||
self._detector_instance = self._initialize_detector(
|
||||
num_hands,
|
||||
min_detection_confidence,
|
||||
min_tracking_confidence,
|
||||
min_presence_confidence,
|
||||
running_mode,
|
||||
delegate,
|
||||
) # Pass running_mode
|
||||
|
||||
batch_size = image.shape[0]
|
||||
batch_results = []
|
||||
@@ -77,7 +105,7 @@ class GestureRecognizer:
|
||||
recognize_func = self._detector_instance.recognize_for_video if is_video_mode else self._detector_instance.recognize
|
||||
|
||||
for i in range(batch_size):
|
||||
img_tensor = image[i]
|
||||
img_tensor = image[i]
|
||||
np_image = (img_tensor.cpu().numpy() * 255).astype(np.uint8)
|
||||
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=np_image)
|
||||
|
||||
@@ -86,17 +114,26 @@ class GestureRecognizer:
|
||||
recognition_result = recognize_func(mp_image, timestamp_ms)
|
||||
else:
|
||||
recognition_result = recognize_func(mp_image)
|
||||
|
||||
|
||||
current_image_gestures = []
|
||||
if recognition_result and recognition_result.gestures: # Added check for recognition_result
|
||||
if recognition_result and recognition_result.gestures: # Added check for recognition_result
|
||||
for hand_idx, hand_gestures in enumerate(recognition_result.gestures):
|
||||
gesture_categories = [GestureCategory(index=cat.index, score=cat.score, display_name=cat.display_name, category_name=cat.category_name) for cat in hand_gestures]
|
||||
gesture_categories = [
|
||||
GestureCategory(
|
||||
index=cat.index,
|
||||
score=cat.score,
|
||||
display_name=cat.display_name,
|
||||
category_name=cat.category_name,
|
||||
)
|
||||
for cat in hand_gestures
|
||||
]
|
||||
handedness = None
|
||||
if recognition_result.handedness and hand_idx < len(recognition_result.handedness):
|
||||
handedness_cats = recognition_result.handedness[hand_idx]
|
||||
if handedness_cats: handedness = handedness_cats[0].display_name
|
||||
if handedness_cats:
|
||||
handedness = handedness_cats[0].display_name
|
||||
current_image_gestures.append(GestureRecognitionResult(gestures=gesture_categories, handedness=handedness))
|
||||
batch_results.append(current_image_gestures)
|
||||
|
||||
# self._detector_instance.close() # Close handled on re-init
|
||||
return batch_results
|
||||
return batch_results
|
||||
|
||||
@@ -1 +1 @@
|
||||
"""Hand Landmark Detection implementation using MediaPipe."""
|
||||
"""Hand Landmark Detection implementation using MediaPipe."""
|
||||
|
||||
@@ -1,17 +1,21 @@
|
||||
from typing import List
|
||||
|
||||
import mediapipe as mp
|
||||
import numpy as np
|
||||
import torch
|
||||
from mediapipe.tasks import python
|
||||
from mediapipe.tasks.python import vision
|
||||
from mediapipe.tasks.python.core.base_options import BaseOptions
|
||||
from typing import List, Tuple, Optional
|
||||
|
||||
from ...utils.timing import TimestampProvider # Import TimestampProvider
|
||||
|
||||
# Import new types
|
||||
from ..types import LandmarkPoint, HandLandmarksResult
|
||||
from ...utils.timing import TimestampProvider # Import TimestampProvider
|
||||
from ..types import HandLandmarksResult, LandmarkPoint
|
||||
|
||||
|
||||
class HandLandmarkDetector:
|
||||
"""Detects hand landmarks using MediaPipe HandLandmarker."""
|
||||
|
||||
|
||||
def __init__(self, model_path: str):
|
||||
"""Initialize the detector with the model path.
|
||||
|
||||
@@ -21,36 +25,49 @@ class HandLandmarkDetector:
|
||||
if not model_path:
|
||||
raise ValueError("A valid model_path must be provided.")
|
||||
# Verify file exists? Optional, MediaPipe might handle this.
|
||||
|
||||
|
||||
self.model_path = model_path
|
||||
self._detector_instance = None
|
||||
self._current_options = None
|
||||
self._timestamp_provider = None # Added
|
||||
self._timestamp_provider = None # Added
|
||||
|
||||
def _initialize_detector(self, num_hands: int, min_detection_confidence: float,
|
||||
min_presence_confidence: float, min_tracking_confidence: float,
|
||||
running_mode: str, delegate: str): # Added running_mode
|
||||
def _initialize_detector(
|
||||
self,
|
||||
num_hands: int,
|
||||
min_detection_confidence: float,
|
||||
min_presence_confidence: float,
|
||||
min_tracking_confidence: float,
|
||||
running_mode: str,
|
||||
delegate: str,
|
||||
): # Added running_mode
|
||||
"""Initializes the HandLandmarker detector."""
|
||||
mode_enum = vision.RunningMode.IMAGE if running_mode == "image" else vision.RunningMode.VIDEO # Added
|
||||
delegate_enum = BaseOptions.Delegate.CPU if delegate.lower() == 'cpu' else BaseOptions.Delegate.GPU
|
||||
mode_enum = vision.RunningMode.IMAGE if running_mode == "image" else vision.RunningMode.VIDEO # Added
|
||||
delegate_enum = BaseOptions.Delegate.CPU if delegate.lower() == "cpu" else BaseOptions.Delegate.GPU
|
||||
base_options = python.BaseOptions(model_asset_path=self.model_path, delegate=delegate_enum)
|
||||
options = vision.HandLandmarkerOptions(
|
||||
base_options=base_options,
|
||||
running_mode=mode_enum, # Use enum mode
|
||||
running_mode=mode_enum, # Use enum mode
|
||||
num_hands=num_hands,
|
||||
min_hand_detection_confidence=min_detection_confidence,
|
||||
# min_hand_presence_confidence and min_tracking_confidence are for VIDEO/LIVE_STREAM
|
||||
min_hand_presence_confidence=min_presence_confidence,
|
||||
min_tracking_confidence=min_tracking_confidence
|
||||
min_tracking_confidence=min_tracking_confidence,
|
||||
)
|
||||
self._current_options = options
|
||||
# Only create timestamp provider for VIDEO mode
|
||||
self._timestamp_provider = TimestampProvider() if mode_enum == vision.RunningMode.VIDEO else None
|
||||
self._timestamp_provider = TimestampProvider() if mode_enum == vision.RunningMode.VIDEO else None
|
||||
return vision.HandLandmarker.create_from_options(options)
|
||||
|
||||
def detect(self, image: torch.Tensor, num_hands: int = 2, min_detection_confidence: float = 0.5,
|
||||
min_presence_confidence: float = 0.5, min_tracking_confidence: float = 0.5,
|
||||
running_mode: str = "video", delegate: str = 'cpu') -> List[List[HandLandmarksResult]]: # Added running_mode
|
||||
def detect(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
num_hands: int = 2,
|
||||
min_detection_confidence: float = 0.5,
|
||||
min_presence_confidence: float = 0.5,
|
||||
min_tracking_confidence: float = 0.5,
|
||||
running_mode: str = "video",
|
||||
delegate: str = "cpu",
|
||||
) -> List[List[HandLandmarksResult]]: # Added running_mode
|
||||
"""Detects hand landmarks in the input image tensor.
|
||||
|
||||
Returns:
|
||||
@@ -60,72 +77,78 @@ class HandLandmarkDetector:
|
||||
raise ValueError("Input tensor must be in BHWC format.")
|
||||
|
||||
# Add running_mode to tuple check
|
||||
new_options_tuple = (num_hands, min_detection_confidence,
|
||||
min_presence_confidence, min_tracking_confidence, running_mode, delegate.lower())
|
||||
new_options_tuple = (
|
||||
num_hands,
|
||||
min_detection_confidence,
|
||||
min_presence_confidence,
|
||||
min_tracking_confidence,
|
||||
running_mode,
|
||||
delegate.lower(),
|
||||
)
|
||||
current_options_tuple = None
|
||||
if self._current_options:
|
||||
current_options_tuple = (
|
||||
self._current_options.num_hands,
|
||||
self._current_options.min_hand_detection_confidence,
|
||||
self._current_options.min_hand_presence_confidence,
|
||||
self._current_options.min_tracking_confidence,
|
||||
"image" if self._current_options.running_mode == vision.RunningMode.IMAGE else "video", # Map enum back to string
|
||||
'cpu' if self._current_options.base_options.delegate == BaseOptions.Delegate.CPU else 'gpu'
|
||||
)
|
||||
current_options_tuple = (
|
||||
self._current_options.num_hands,
|
||||
self._current_options.min_hand_detection_confidence,
|
||||
self._current_options.min_hand_presence_confidence,
|
||||
self._current_options.min_tracking_confidence,
|
||||
"image" if self._current_options.running_mode == vision.RunningMode.IMAGE else "video", # Map enum back to string
|
||||
"cpu" if self._current_options.base_options.delegate == BaseOptions.Delegate.CPU else "gpu",
|
||||
)
|
||||
|
||||
if self._detector_instance is None or new_options_tuple != current_options_tuple:
|
||||
if self._detector_instance and hasattr(self._detector_instance, 'close'):
|
||||
self._detector_instance.close()
|
||||
# Pass new params to initializer
|
||||
self._detector_instance = self._initialize_detector(num_hands, min_detection_confidence,
|
||||
min_presence_confidence, min_tracking_confidence,
|
||||
running_mode, delegate) # Pass running_mode
|
||||
if self._detector_instance and hasattr(self._detector_instance, "close"):
|
||||
self._detector_instance.close()
|
||||
# Pass new params to initializer
|
||||
self._detector_instance = self._initialize_detector(
|
||||
num_hands,
|
||||
min_detection_confidence,
|
||||
min_presence_confidence,
|
||||
min_tracking_confidence,
|
||||
running_mode,
|
||||
delegate,
|
||||
) # Pass running_mode
|
||||
|
||||
batch_size = image.shape[0]
|
||||
batch_results = []
|
||||
# Determine the correct detection function based on mode
|
||||
is_video_mode = self._current_options.running_mode == vision.RunningMode.VIDEO
|
||||
detect_func = self._detector_instance.detect_for_video if is_video_mode else self._detector_instance.detect
|
||||
|
||||
|
||||
for i in range(batch_size):
|
||||
img_tensor = image[i] # HWC
|
||||
img_tensor = image[i] # HWC
|
||||
np_image = (img_tensor.cpu().numpy() * 255).astype(np.uint8)
|
||||
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=np_image)
|
||||
|
||||
|
||||
if is_video_mode:
|
||||
timestamp_ms = self._timestamp_provider.next()
|
||||
detection_result = detect_func(mp_image, timestamp_ms)
|
||||
else:
|
||||
detection_result = detect_func(mp_image)
|
||||
|
||||
|
||||
current_image_results = []
|
||||
if detection_result and detection_result.hand_landmarks: # Added check for detection_result
|
||||
if detection_result and detection_result.hand_landmarks: # Added check for detection_result
|
||||
for hand_idx, hand_landmarks_mp in enumerate(detection_result.hand_landmarks):
|
||||
|
||||
landmarks = [
|
||||
LandmarkPoint(index=lm_idx, x=lm.x, y=lm.y, z=lm.z)
|
||||
for lm_idx, lm in enumerate(hand_landmarks_mp)
|
||||
]
|
||||
|
||||
landmarks = [LandmarkPoint(index=lm_idx, x=lm.x, y=lm.y, z=lm.z) for lm_idx, lm in enumerate(hand_landmarks_mp)]
|
||||
|
||||
world_landmarks = None
|
||||
if detection_result.hand_world_landmarks and hand_idx < len(detection_result.hand_world_landmarks):
|
||||
world_landmarks = [
|
||||
LandmarkPoint(index=lm_idx, x=lm.x, y=lm.y, z=lm.z)
|
||||
for lm_idx, lm in enumerate(detection_result.hand_world_landmarks[hand_idx])
|
||||
]
|
||||
|
||||
|
||||
handedness = None
|
||||
if detection_result.handedness and hand_idx < len(detection_result.handedness):
|
||||
handedness_cats = detection_result.handedness[hand_idx]
|
||||
if handedness_cats: handedness = handedness_cats[0].display_name
|
||||
|
||||
current_image_results.append(HandLandmarksResult(
|
||||
landmarks=landmarks,
|
||||
world_landmarks=world_landmarks,
|
||||
handedness=handedness
|
||||
))
|
||||
|
||||
if handedness_cats:
|
||||
handedness = handedness_cats[0].display_name
|
||||
|
||||
current_image_results.append(
|
||||
HandLandmarksResult(landmarks=landmarks, world_landmarks=world_landmarks, handedness=handedness)
|
||||
)
|
||||
|
||||
batch_results.append(current_image_results)
|
||||
|
||||
# self._detector_instance.close() # Close handled on re-init
|
||||
return batch_results
|
||||
return batch_results
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
"""Holistic landmark detection module using MediaPipe Legacy API."""
|
||||
|
||||
from .detector import HolisticLandmarkDetector
|
||||
from .detector import HolisticLandmarkDetector
|
||||
|
||||
@@ -4,26 +4,29 @@ This implementation specifically handles the MediaPipe Holistic solution
|
||||
which is currently only available through the legacy API, not the Task API.
|
||||
"""
|
||||
|
||||
from typing import Any, List, Optional, Tuple
|
||||
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
import numpy as np
|
||||
import torch
|
||||
import cv2
|
||||
from typing import List, Tuple, Optional, Dict, Any
|
||||
|
||||
from ...utils.timing import TimestampProvider
|
||||
|
||||
# Import needed types
|
||||
from ..types import LandmarkPoint, HolisticLandmarksResult
|
||||
from ...utils.timing import TimestampProvider
|
||||
from ..types import HolisticLandmarksResult, LandmarkPoint
|
||||
|
||||
|
||||
class HolisticLandmarkDetector:
|
||||
"""Detects holistic landmarks combining face, pose and hand tracking.
|
||||
|
||||
|
||||
This detector uses the legacy MediaPipe Solutions API rather than the Task API
|
||||
since Holistic is not yet available in the Task API format.
|
||||
"""
|
||||
|
||||
|
||||
def __init__(self, model_path: Optional[str] = None):
|
||||
"""Initialize the detector.
|
||||
|
||||
|
||||
Args:
|
||||
model_path: Unused parameter to maintain API compatibility with other detectors.
|
||||
The holistic solution doesn't use a model path as it uses the built-in models.
|
||||
@@ -33,15 +36,17 @@ class HolisticLandmarkDetector:
|
||||
self._current_options = None
|
||||
self._timestamp_provider = TimestampProvider()
|
||||
|
||||
def _initialize_detector(self,
|
||||
min_detection_confidence: float,
|
||||
min_tracking_confidence: float,
|
||||
model_complexity: int,
|
||||
enable_segmentation: bool,
|
||||
refine_face_landmarks: bool,
|
||||
static_image_mode: bool) -> mp.solutions.holistic.Holistic:
|
||||
def _initialize_detector(
|
||||
self,
|
||||
min_detection_confidence: float,
|
||||
min_tracking_confidence: float,
|
||||
model_complexity: int,
|
||||
enable_segmentation: bool,
|
||||
refine_face_landmarks: bool,
|
||||
static_image_mode: bool,
|
||||
) -> mp.solutions.holistic.Holistic:
|
||||
"""Initializes the Holistic detector with the appropriate parameters.
|
||||
|
||||
|
||||
Args:
|
||||
min_detection_confidence: Minimum confidence for detection to be considered
|
||||
min_tracking_confidence: Minimum confidence to maintain tracking
|
||||
@@ -49,20 +54,20 @@ class HolisticLandmarkDetector:
|
||||
enable_segmentation: Whether to generate segmentation masks
|
||||
refine_face_landmarks: Whether to use refined face landmarks (more details around eyes/lips)
|
||||
static_image_mode: If True, treats input as independent images (slower but more accurate)
|
||||
|
||||
|
||||
Returns:
|
||||
Initialized Holistic detector instance
|
||||
"""
|
||||
# Store options as a dictionary for parameter comparison
|
||||
self._current_options = {
|
||||
'min_detection_confidence': min_detection_confidence,
|
||||
'min_tracking_confidence': min_tracking_confidence,
|
||||
'model_complexity': model_complexity,
|
||||
'enable_segmentation': enable_segmentation,
|
||||
'refine_face_landmarks': refine_face_landmarks,
|
||||
'static_image_mode': static_image_mode
|
||||
"min_detection_confidence": min_detection_confidence,
|
||||
"min_tracking_confidence": min_tracking_confidence,
|
||||
"model_complexity": model_complexity,
|
||||
"enable_segmentation": enable_segmentation,
|
||||
"refine_face_landmarks": refine_face_landmarks,
|
||||
"static_image_mode": static_image_mode,
|
||||
}
|
||||
|
||||
|
||||
# Create the holistic solution instance with specified options
|
||||
return mp.solutions.holistic.Holistic(
|
||||
static_image_mode=static_image_mode,
|
||||
@@ -70,18 +75,21 @@ class HolisticLandmarkDetector:
|
||||
enable_segmentation=enable_segmentation,
|
||||
refine_face_landmarks=refine_face_landmarks,
|
||||
min_detection_confidence=min_detection_confidence,
|
||||
min_tracking_confidence=min_tracking_confidence
|
||||
min_tracking_confidence=min_tracking_confidence,
|
||||
)
|
||||
|
||||
def detect(self, image: torch.Tensor,
|
||||
min_detection_confidence: float = 0.5,
|
||||
min_tracking_confidence: float = 0.5,
|
||||
model_complexity: int = 1,
|
||||
enable_segmentation: bool = False,
|
||||
refine_face_landmarks: bool = False,
|
||||
static_image_mode: bool = False) -> Tuple[List[HolisticLandmarksResult], Optional[List[torch.Tensor]]]:
|
||||
def detect(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
min_detection_confidence: float = 0.5,
|
||||
min_tracking_confidence: float = 0.5,
|
||||
model_complexity: int = 1,
|
||||
enable_segmentation: bool = False,
|
||||
refine_face_landmarks: bool = False,
|
||||
static_image_mode: bool = False,
|
||||
) -> Tuple[List[HolisticLandmarksResult], Optional[List[torch.Tensor]]]:
|
||||
"""Detects holistic landmarks in the input image tensor.
|
||||
|
||||
|
||||
Args:
|
||||
image: Input image tensor in BHWC format
|
||||
min_detection_confidence: Minimum confidence for detection (0.0-1.0)
|
||||
@@ -90,7 +98,7 @@ class HolisticLandmarkDetector:
|
||||
enable_segmentation: Whether to output segmentation masks
|
||||
refine_face_landmarks: Whether to use refined face landmarks
|
||||
static_image_mode: If True, treats input as independent images
|
||||
|
||||
|
||||
Returns:
|
||||
Tuple containing:
|
||||
- holistic_landmarks_batch: List of HolisticLandmarksResult objects for each image
|
||||
@@ -101,17 +109,17 @@ class HolisticLandmarkDetector:
|
||||
|
||||
# Create a parameters tuple for comparison
|
||||
new_options = {
|
||||
'min_detection_confidence': min_detection_confidence,
|
||||
'min_tracking_confidence': min_tracking_confidence,
|
||||
'model_complexity': model_complexity,
|
||||
'enable_segmentation': enable_segmentation,
|
||||
'refine_face_landmarks': refine_face_landmarks,
|
||||
'static_image_mode': static_image_mode
|
||||
"min_detection_confidence": min_detection_confidence,
|
||||
"min_tracking_confidence": min_tracking_confidence,
|
||||
"model_complexity": model_complexity,
|
||||
"enable_segmentation": enable_segmentation,
|
||||
"refine_face_landmarks": refine_face_landmarks,
|
||||
"static_image_mode": static_image_mode,
|
||||
}
|
||||
|
||||
# Initialize or update detector if needed
|
||||
if self._detector_instance is None or self._current_options != new_options:
|
||||
if self._detector_instance and hasattr(self._detector_instance, 'close'):
|
||||
if self._detector_instance and hasattr(self._detector_instance, "close"):
|
||||
self._detector_instance.close()
|
||||
self._detector_instance = self._initialize_detector(
|
||||
min_detection_confidence,
|
||||
@@ -119,42 +127,40 @@ class HolisticLandmarkDetector:
|
||||
model_complexity,
|
||||
enable_segmentation,
|
||||
refine_face_landmarks,
|
||||
static_image_mode
|
||||
static_image_mode,
|
||||
)
|
||||
|
||||
batch_size = image.shape[0]
|
||||
batch_holistic_landmarks = []
|
||||
batch_segmentation_masks = [] if enable_segmentation else None
|
||||
|
||||
|
||||
for i in range(batch_size):
|
||||
img_tensor = image[i] # HWC
|
||||
# Convert to numpy with proper scaling and datatype
|
||||
np_image = (img_tensor.cpu().numpy() * 255).astype(np.uint8)
|
||||
|
||||
|
||||
# Process the image with MediaPipe Holistic
|
||||
holistic_results = self._detector_instance.process(
|
||||
cv2.cvtColor(np_image, cv2.COLOR_BGR2RGB)
|
||||
)
|
||||
|
||||
holistic_results = self._detector_instance.process(cv2.cvtColor(np_image, cv2.COLOR_BGR2RGB))
|
||||
|
||||
# Convert results to our standardized format
|
||||
holistic_result = self._convert_to_landmark_result(holistic_results)
|
||||
batch_holistic_landmarks.append(holistic_result)
|
||||
|
||||
|
||||
# Handle segmentation mask if requested
|
||||
if enable_segmentation and holistic_results.segmentation_mask is not None:
|
||||
mask_np = holistic_results.segmentation_mask
|
||||
mask_tensor = torch.from_numpy(mask_np).float() # HW
|
||||
if batch_segmentation_masks is not None:
|
||||
batch_segmentation_masks.append(mask_tensor)
|
||||
|
||||
|
||||
return batch_holistic_landmarks, batch_segmentation_masks
|
||||
|
||||
def _convert_to_landmark_result(self, holistic_results: Any) -> HolisticLandmarksResult:
|
||||
"""Converts MediaPipe Holistic results to standardized HolisticLandmarksResult.
|
||||
|
||||
|
||||
Args:
|
||||
holistic_results: Raw results from mp.solutions.holistic
|
||||
|
||||
|
||||
Returns:
|
||||
Standardized HolisticLandmarksResult containing all landmarks
|
||||
"""
|
||||
@@ -167,75 +173,53 @@ class HolisticLandmarkDetector:
|
||||
x=landmark.x,
|
||||
y=landmark.y,
|
||||
z=landmark.z,
|
||||
visibility=getattr(landmark, 'visibility', None)
|
||||
visibility=getattr(landmark, "visibility", None),
|
||||
)
|
||||
for idx, landmark in enumerate(holistic_results.face_landmarks.landmark)
|
||||
]
|
||||
|
||||
|
||||
# Convert pose landmarks
|
||||
pose_landmarks = None
|
||||
if holistic_results.pose_landmarks:
|
||||
pose_landmarks = [
|
||||
LandmarkPoint(
|
||||
index=idx,
|
||||
x=landmark.x,
|
||||
y=landmark.y,
|
||||
z=landmark.z,
|
||||
visibility=landmark.visibility
|
||||
)
|
||||
LandmarkPoint(index=idx, x=landmark.x, y=landmark.y, z=landmark.z, visibility=landmark.visibility)
|
||||
for idx, landmark in enumerate(holistic_results.pose_landmarks.landmark)
|
||||
]
|
||||
|
||||
|
||||
# Convert pose world landmarks
|
||||
pose_world_landmarks = None
|
||||
if holistic_results.pose_world_landmarks:
|
||||
pose_world_landmarks = [
|
||||
LandmarkPoint(
|
||||
index=idx,
|
||||
x=landmark.x,
|
||||
y=landmark.y,
|
||||
z=landmark.z,
|
||||
visibility=landmark.visibility
|
||||
)
|
||||
LandmarkPoint(index=idx, x=landmark.x, y=landmark.y, z=landmark.z, visibility=landmark.visibility)
|
||||
for idx, landmark in enumerate(holistic_results.pose_world_landmarks.landmark)
|
||||
]
|
||||
|
||||
|
||||
# Convert left hand landmarks
|
||||
left_hand_landmarks = None
|
||||
if holistic_results.left_hand_landmarks:
|
||||
left_hand_landmarks = [
|
||||
LandmarkPoint(
|
||||
index=idx,
|
||||
x=landmark.x,
|
||||
y=landmark.y,
|
||||
z=landmark.z
|
||||
)
|
||||
LandmarkPoint(index=idx, x=landmark.x, y=landmark.y, z=landmark.z)
|
||||
for idx, landmark in enumerate(holistic_results.left_hand_landmarks.landmark)
|
||||
]
|
||||
|
||||
|
||||
# Convert right hand landmarks
|
||||
right_hand_landmarks = None
|
||||
if holistic_results.right_hand_landmarks:
|
||||
right_hand_landmarks = [
|
||||
LandmarkPoint(
|
||||
index=idx,
|
||||
x=landmark.x,
|
||||
y=landmark.y,
|
||||
z=landmark.z
|
||||
)
|
||||
LandmarkPoint(index=idx, x=landmark.x, y=landmark.y, z=landmark.z)
|
||||
for idx, landmark in enumerate(holistic_results.right_hand_landmarks.landmark)
|
||||
]
|
||||
|
||||
|
||||
return HolisticLandmarksResult(
|
||||
face_landmarks=face_landmarks,
|
||||
pose_landmarks=pose_landmarks,
|
||||
pose_world_landmarks=pose_world_landmarks,
|
||||
left_hand_landmarks=left_hand_landmarks,
|
||||
right_hand_landmarks=right_hand_landmarks
|
||||
right_hand_landmarks=right_hand_landmarks,
|
||||
)
|
||||
|
||||
|
||||
def close(self):
|
||||
"""Releases resources used by the detector."""
|
||||
if self._detector_instance and hasattr(self._detector_instance, 'close'):
|
||||
if self._detector_instance and hasattr(self._detector_instance, "close"):
|
||||
self._detector_instance.close()
|
||||
self._detector_instance = None
|
||||
self._detector_instance = None
|
||||
|
||||
@@ -1 +1 @@
|
||||
"""Image Classification implementation using MediaPipe."""
|
||||
"""Image Classification implementation using MediaPipe."""
|
||||
|
||||
@@ -1 +1 @@
|
||||
"""Image Embedding implementation using MediaPipe."""
|
||||
"""Image Embedding implementation using MediaPipe."""
|
||||
|
||||
@@ -1,15 +1,19 @@
|
||||
from typing import List
|
||||
|
||||
import mediapipe as mp
|
||||
import numpy as np
|
||||
import torch
|
||||
from mediapipe.tasks import python
|
||||
from mediapipe.tasks.python import vision
|
||||
from mediapipe.tasks.python.core.base_options import BaseOptions
|
||||
from typing import List, Optional
|
||||
# Import new types
|
||||
from ..types import ImageEmbedderResult
|
||||
|
||||
# Import TimestampProvider from the new location
|
||||
from ...utils.timing import TimestampProvider
|
||||
|
||||
# Import new types
|
||||
from ..types import ImageEmbedderResult
|
||||
|
||||
|
||||
class ImageEmbedder:
|
||||
"""Generates image embeddings using MediaPipe ImageEmbedder."""
|
||||
|
||||
@@ -19,45 +23,50 @@ class ImageEmbedder:
|
||||
self.model_path = model_path
|
||||
self._detector_instance = None
|
||||
self._current_options = None
|
||||
self._timestamp_provider = None # Added
|
||||
self._timestamp_provider = None # Added
|
||||
|
||||
def _initialize_detector(self, l2_normalize: bool, quantize: bool,
|
||||
running_mode: str, delegate: str): # Added running_mode
|
||||
mode_enum = vision.RunningMode.IMAGE if running_mode == "image" else vision.RunningMode.VIDEO # Added
|
||||
delegate_enum = BaseOptions.Delegate.CPU if delegate.lower() == 'cpu' else BaseOptions.Delegate.GPU
|
||||
def _initialize_detector(self, l2_normalize: bool, quantize: bool, running_mode: str, delegate: str): # Added running_mode
|
||||
mode_enum = vision.RunningMode.IMAGE if running_mode == "image" else vision.RunningMode.VIDEO # Added
|
||||
delegate_enum = BaseOptions.Delegate.CPU if delegate.lower() == "cpu" else BaseOptions.Delegate.GPU
|
||||
base_options = python.BaseOptions(model_asset_path=self.model_path, delegate=delegate_enum)
|
||||
options = vision.ImageEmbedderOptions(
|
||||
base_options=base_options,
|
||||
running_mode=mode_enum, # Use enum mode
|
||||
running_mode=mode_enum, # Use enum mode
|
||||
l2_normalize=l2_normalize,
|
||||
quantize=quantize
|
||||
quantize=quantize,
|
||||
)
|
||||
self._current_options = options
|
||||
# Only create timestamp provider for VIDEO mode
|
||||
self._timestamp_provider = TimestampProvider() if mode_enum == vision.RunningMode.VIDEO else None
|
||||
self._timestamp_provider = TimestampProvider() if mode_enum == vision.RunningMode.VIDEO else None
|
||||
return vision.ImageEmbedder.create_from_options(options)
|
||||
|
||||
def embed(self, image: torch.Tensor, l2_normalize: bool = True, quantize: bool = False,
|
||||
running_mode: str = "video", delegate: str = 'cpu') -> List[List[ImageEmbedderResult]]: # Added running_mode
|
||||
def embed(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
l2_normalize: bool = True,
|
||||
quantize: bool = False,
|
||||
running_mode: str = "video",
|
||||
delegate: str = "cpu",
|
||||
) -> List[List[ImageEmbedderResult]]: # Added running_mode
|
||||
"""Generates embeddings for the input image tensor(s)."""
|
||||
if image.dim() != 4:
|
||||
raise ValueError("Input tensor must be in BHWC format.")
|
||||
|
||||
|
||||
# Add running_mode to tuple check
|
||||
new_options_tuple = (l2_normalize, quantize, running_mode, delegate.lower())
|
||||
current_options_tuple = None
|
||||
if self._current_options:
|
||||
current_options_tuple = (
|
||||
self._current_options.l2_normalize,
|
||||
self._current_options.quantize,
|
||||
"image" if self._current_options.running_mode == vision.RunningMode.IMAGE else "video", # Map enum back to string
|
||||
'cpu' if self._current_options.base_options.delegate == BaseOptions.Delegate.CPU else 'gpu'
|
||||
)
|
||||
current_options_tuple = (
|
||||
self._current_options.l2_normalize,
|
||||
self._current_options.quantize,
|
||||
"image" if self._current_options.running_mode == vision.RunningMode.IMAGE else "video", # Map enum back to string
|
||||
"cpu" if self._current_options.base_options.delegate == BaseOptions.Delegate.CPU else "gpu",
|
||||
)
|
||||
|
||||
if self._detector_instance is None or new_options_tuple != current_options_tuple:
|
||||
# ImageEmbedder doesn't seem to have close(), so just replace instance
|
||||
if self._detector_instance and hasattr(self._detector_instance, 'close'):
|
||||
self._detector_instance.close()
|
||||
if self._detector_instance and hasattr(self._detector_instance, "close"):
|
||||
self._detector_instance.close()
|
||||
self._detector_instance = self._initialize_detector(l2_normalize, quantize, running_mode, delegate)
|
||||
|
||||
batch_size = image.shape[0]
|
||||
@@ -67,7 +76,7 @@ class ImageEmbedder:
|
||||
embed_func = self._detector_instance.embed_for_video if is_video_mode else self._detector_instance.embed
|
||||
|
||||
for i in range(batch_size):
|
||||
img_tensor = image[i]
|
||||
img_tensor = image[i]
|
||||
np_image = (img_tensor.cpu().numpy() * 255).astype(np.uint8)
|
||||
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=np_image)
|
||||
|
||||
@@ -76,12 +85,17 @@ class ImageEmbedder:
|
||||
embedding_result = embed_func(mp_image, timestamp_ms)
|
||||
else:
|
||||
embedding_result = embed_func(mp_image)
|
||||
|
||||
|
||||
current_image_embeddings = []
|
||||
if embedding_result and embedding_result.embeddings: # Added check for embedding_result
|
||||
if embedding_result and embedding_result.embeddings: # Added check for embedding_result
|
||||
for emb in embedding_result.embeddings:
|
||||
result = ImageEmbedderResult(float_embedding=list(emb.embedding) if not quantize else None, quantized_embedding=bytes(emb.embedding) if quantize else None, head_index=emb.head_index, head_name=emb.head_name)
|
||||
result = ImageEmbedderResult(
|
||||
float_embedding=list(emb.embedding) if not quantize else None,
|
||||
quantized_embedding=bytes(emb.embedding) if quantize else None,
|
||||
head_index=emb.head_index,
|
||||
head_name=emb.head_name,
|
||||
)
|
||||
current_image_embeddings.append(result)
|
||||
batch_results.append(current_image_embeddings)
|
||||
|
||||
return batch_results
|
||||
|
||||
return batch_results
|
||||
|
||||
@@ -1 +1 @@
|
||||
"""Image Segmentation implementation using MediaPipe."""
|
||||
"""Image Segmentation implementation using MediaPipe."""
|
||||
|
||||
@@ -3,21 +3,24 @@
|
||||
This module contains the implementation of MediaPipe Image Segmentation functionality.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import platform # Needed for OS check
|
||||
|
||||
import mediapipe as mp
|
||||
import numpy as np
|
||||
import torch
|
||||
from mediapipe.tasks import python
|
||||
from mediapipe.tasks.python import vision
|
||||
from mediapipe.tasks.python.vision import ImageSegmenter
|
||||
import platform # Needed for OS check
|
||||
import logging
|
||||
|
||||
from ...utils.timing import TimestampProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ImageSegmenter:
|
||||
"""Performs image segmentation using MediaPipe ImageSegmenter.
|
||||
Adapted from ComfyUI-Stream-Pack implementation.
|
||||
Adapted from ComfyUI-Stream-Pack implementation.
|
||||
"""
|
||||
|
||||
def __init__(self, model_path: str):
|
||||
@@ -33,49 +36,47 @@ class ImageSegmenter:
|
||||
# Keep track of the initialized segmenter instance and its config
|
||||
self._detector_instance = None
|
||||
self.current_config = None
|
||||
self.delegate = None # Store delegate used
|
||||
self._timestamp_provider = None # Added
|
||||
self.delegate = None # Store delegate used
|
||||
self._timestamp_provider = None # Added
|
||||
|
||||
def _initialize_segmenter(
|
||||
self,
|
||||
output_confidence_masks: bool,
|
||||
output_category_mask: bool,
|
||||
running_mode: str, delegate_mode="cpu"):
|
||||
def _initialize_segmenter(self, output_confidence_masks: bool, output_category_mask: bool, running_mode: str, delegate_mode="cpu"):
|
||||
"""Initializes the ImageSegmenter or returns existing if config matches.
|
||||
Handles GPU delegate selection.
|
||||
Handles GPU delegate selection.
|
||||
"""
|
||||
# Return type removed from signature as it's handled internally
|
||||
new_config = (output_confidence_masks, output_category_mask, running_mode, delegate_mode)
|
||||
|
||||
|
||||
# Re-initialize if config changed or not initialized
|
||||
if self._detector_instance is None or self.current_config != new_config:
|
||||
if self._detector_instance:
|
||||
logger.info("Closing existing ImageSegmenter due to config change.")
|
||||
try:
|
||||
self._detector_instance.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing previous segmenter: {e}")
|
||||
self._detector_instance = None
|
||||
|
||||
logger.info(f"Initializing ImageSegmenter: model='{self.model_path}', conf_mask={output_confidence_masks}, cat_mask={output_category_mask}, mode={running_mode}, delegate={delegate_mode}")
|
||||
|
||||
logger.info("Closing existing ImageSegmenter due to config change.")
|
||||
try:
|
||||
self._detector_instance.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing previous segmenter: {e}")
|
||||
self._detector_instance = None
|
||||
|
||||
logger.info(
|
||||
f"Initializing ImageSegmenter: model='{self.model_path}', conf_mask={output_confidence_masks}, cat_mask={output_category_mask}, mode={running_mode}, delegate={delegate_mode}"
|
||||
)
|
||||
|
||||
# Configure BaseOptions with delegate
|
||||
use_gpu = (delegate_mode.lower() == "gpu")
|
||||
use_gpu = delegate_mode.lower() == "gpu"
|
||||
if use_gpu and platform.system().lower() == "windows":
|
||||
logger.warning("GPU delegate is not supported on Windows for Python MediaPipe Tasks. Falling back to CPU.")
|
||||
use_gpu = False
|
||||
delegate_mode = "cpu"
|
||||
|
||||
|
||||
delegate = python.BaseOptions.Delegate.GPU if use_gpu else python.BaseOptions.Delegate.CPU
|
||||
base_options = python.BaseOptions(model_asset_path=self.model_path, delegate=delegate)
|
||||
|
||||
|
||||
try:
|
||||
mode_enum = vision.RunningMode.IMAGE if running_mode == "image" else vision.RunningMode.VIDEO
|
||||
options = vision.ImageSegmenterOptions(
|
||||
base_options=base_options,
|
||||
running_mode=mode_enum,
|
||||
output_confidence_masks=output_confidence_masks,
|
||||
output_category_mask=output_category_mask
|
||||
output_category_mask=output_category_mask,
|
||||
)
|
||||
self._detector_instance = vision.ImageSegmenter.create_from_options(options)
|
||||
self.current_config = new_config
|
||||
@@ -89,15 +90,21 @@ class ImageSegmenter:
|
||||
else:
|
||||
# logger.debug("Reusing existing ImageSegmenter instance.") # Optional: for debugging
|
||||
pass
|
||||
|
||||
|
||||
# No explicit return needed, instance is stored in self._detector_instance
|
||||
|
||||
def segment(self, image: torch.Tensor, output_confidence_masks: bool = True, output_category_mask: bool = False,
|
||||
running_mode: str = "video", delegate_mode="cpu"):
|
||||
def segment(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
output_confidence_masks: bool = True,
|
||||
output_category_mask: bool = False,
|
||||
running_mode: str = "video",
|
||||
delegate_mode="cpu",
|
||||
):
|
||||
"""Segments the input image tensor.
|
||||
|
||||
Args:
|
||||
image: Input image tensor (BHWC, RGB, 0-1 float).
|
||||
image: Input image tensor (BHWC, RGB, 0-1 float).
|
||||
NOTE: This implementation currently only supports batch size 1 for simplicity.
|
||||
output_confidence_masks: Whether to output confidence masks (float, 0-1).
|
||||
output_category_mask: Whether to output a category mask (uint8).
|
||||
@@ -113,18 +120,18 @@ class ImageSegmenter:
|
||||
raise ValueError("Input tensor must be in BHWC format.")
|
||||
# if image.shape[0] != 1: # Temporarily removed batch size 1 restriction
|
||||
# logger.warning("ImageSegmenter currently processes batch > 1 sequentially. Performance may vary.")
|
||||
# raise ValueError("Input tensor must be in BHWC format with batch size 1.")
|
||||
# raise ValueError("Input tensor must be in BHWC format with batch size 1.")
|
||||
if not output_confidence_masks and not output_category_mask:
|
||||
raise ValueError("At least one output type (confidence or category mask) must be enabled.")
|
||||
raise ValueError("At least one output type (confidence or category mask) must be enabled.")
|
||||
|
||||
# Initialize (or get existing) segmenter instance
|
||||
# Initialization now happens within the function if needed
|
||||
self._initialize_segmenter(output_confidence_masks, output_category_mask, running_mode, delegate_mode)
|
||||
if not self._detector_instance:
|
||||
# Initialization failed, return None or raise error
|
||||
logger.error("Segmenter instance is not available after initialization attempt.")
|
||||
# Return structure matching expected output format (list per batch item)
|
||||
return [None] * image.shape[0], [None] * image.shape[0]
|
||||
# Initialization failed, return None or raise error
|
||||
logger.error("Segmenter instance is not available after initialization attempt.")
|
||||
# Return structure matching expected output format (list per batch item)
|
||||
return [None] * image.shape[0], [None] * image.shape[0]
|
||||
|
||||
batch_size = image.shape[0]
|
||||
batch_results_confidence = [] if output_confidence_masks else None
|
||||
@@ -135,7 +142,7 @@ class ImageSegmenter:
|
||||
segment_func = self._detector_instance.segment_for_video if is_video_mode else self._detector_instance.segment
|
||||
|
||||
for i in range(batch_size):
|
||||
img_tensor = image[i] # HWC
|
||||
img_tensor = image[i] # HWC
|
||||
# Convert image format
|
||||
np_image = (img_tensor.cpu().numpy() * 255).astype(np.uint8)
|
||||
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=np_image)
|
||||
@@ -149,11 +156,13 @@ class ImageSegmenter:
|
||||
else:
|
||||
segmentation_result = segment_func(mp_image)
|
||||
except Exception as e:
|
||||
logger.error(f"Error during segmentation for image {i}: {e}")
|
||||
# Append None results for this image
|
||||
if output_confidence_masks: batch_results_confidence.append(None)
|
||||
if output_category_mask: batch_results_category.append(None)
|
||||
continue # Skip to the next image
|
||||
logger.error(f"Error during segmentation for image {i}: {e}")
|
||||
# Append None results for this image
|
||||
if output_confidence_masks:
|
||||
batch_results_confidence.append(None)
|
||||
if output_category_mask:
|
||||
batch_results_category.append(None)
|
||||
continue # Skip to the next image
|
||||
|
||||
# Process results for the current image
|
||||
current_image_confidence_masks = None
|
||||
@@ -164,7 +173,7 @@ class ImageSegmenter:
|
||||
current_image_confidence_masks = []
|
||||
for mask in segmentation_result.confidence_masks:
|
||||
mask_np = mask.numpy_view()
|
||||
mask_tensor = torch.from_numpy(mask_np).float() # Keep as HW
|
||||
mask_tensor = torch.from_numpy(mask_np).float() # Keep as HW
|
||||
current_image_confidence_masks.append(mask_tensor)
|
||||
batch_results_confidence.append(current_image_confidence_masks)
|
||||
|
||||
@@ -172,23 +181,23 @@ class ImageSegmenter:
|
||||
if segmentation_result and segmentation_result.category_mask:
|
||||
mask_np = segmentation_result.category_mask.numpy_view()
|
||||
# Category mask is uint8, convert to long tensor
|
||||
current_image_category_mask = torch.from_numpy(mask_np).long() # Keep as HW
|
||||
current_image_category_mask = torch.from_numpy(mask_np).long() # Keep as HW
|
||||
batch_results_category.append(current_image_category_mask)
|
||||
|
||||
return batch_results_confidence, batch_results_category
|
||||
|
||||
def close(self):
|
||||
"""Closes the segmenter instance if it exists."""
|
||||
if self._detector_instance:
|
||||
"""Closes the segmenter instance if it exists."""
|
||||
if self._detector_instance:
|
||||
logger.info("Closing ImageSegmenter instance.")
|
||||
try:
|
||||
self._detector_instance.close()
|
||||
self._detector_instance.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing segmenter instance: {e}")
|
||||
logger.warning(f"Error closing segmenter instance: {e}")
|
||||
self._detector_instance = None
|
||||
self.current_config = None
|
||||
self.delegate = None
|
||||
|
||||
def __del__(self):
|
||||
# Ensure resources are released when the object is destroyed
|
||||
self.close()
|
||||
self.close()
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
|
||||
@@ -1,24 +1,21 @@
|
||||
import logging
|
||||
from typing import List, Optional
|
||||
|
||||
import mediapipe as mp
|
||||
import numpy as np
|
||||
import torch
|
||||
from mediapipe.tasks import python
|
||||
from mediapipe.tasks.python import vision
|
||||
from mediapipe.tasks.python.vision.interactive_segmenter import (
|
||||
InteractiveSegmenter,
|
||||
)
|
||||
from mediapipe.tasks.python.vision import interactive_segmenter
|
||||
from mediapipe.tasks.python.components.containers import keypoint
|
||||
from mediapipe.tasks.python.vision import interactive_segmenter
|
||||
|
||||
from typing import List, Optional
|
||||
from ..types import PointOfInterest
|
||||
from ...utils.timing import TimestampProvider
|
||||
from ..types import PointOfInterest
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
NormalizedKeypoint = keypoint.NormalizedKeypoint
|
||||
|
||||
|
||||
class InteractiveSegmenterProcessor:
|
||||
"""Performs interactive image segmentation using MediaPipe."""
|
||||
|
||||
@@ -33,16 +30,21 @@ class InteractiveSegmenterProcessor:
|
||||
def _initialize_processor(self, output_category_mask: bool, output_confidence_mask: bool):
|
||||
base_options = python.BaseOptions(model_asset_path=self.model_path)
|
||||
options = interactive_segmenter.InteractiveSegmenterOptions(
|
||||
base_options=base_options,
|
||||
output_category_mask=output_category_mask,
|
||||
output_confidence_masks=output_confidence_mask
|
||||
)
|
||||
base_options=base_options,
|
||||
output_category_mask=output_category_mask,
|
||||
output_confidence_masks=output_confidence_mask,
|
||||
)
|
||||
self.current_options = (output_category_mask, output_confidence_mask)
|
||||
self._timestamp_provider = TimestampProvider()
|
||||
return interactive_segmenter.InteractiveSegmenter.create_from_options(options)
|
||||
|
||||
def segment(self, image: torch.Tensor, points_of_interest: List[PointOfInterest],
|
||||
output_category_mask: bool = True, output_confidence_mask: bool = False) -> List[Optional[torch.Tensor]]:
|
||||
def segment(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
points_of_interest: List[PointOfInterest],
|
||||
output_category_mask: bool = True,
|
||||
output_confidence_mask: bool = False,
|
||||
) -> List[Optional[torch.Tensor]]:
|
||||
"""Performs interactive segmentation based on points of interest.
|
||||
Iterates through points, generates mask for each, and combines them.
|
||||
|
||||
@@ -59,19 +61,19 @@ class InteractiveSegmenterProcessor:
|
||||
if image.dim() != 4:
|
||||
raise ValueError("Input tensor must be in BHWC format.")
|
||||
if not points_of_interest:
|
||||
raise ValueError("At least one point_of_interest must be provided.")
|
||||
raise ValueError("At least one point_of_interest must be provided.")
|
||||
|
||||
batch_size = image.shape[0]
|
||||
batch_results_combined = []
|
||||
|
||||
|
||||
new_options = (output_category_mask, output_confidence_mask)
|
||||
if self.detector_instance is None or self.current_options != new_options:
|
||||
if self.detector_instance and hasattr(self.detector_instance, 'close'):
|
||||
self.detector_instance.close()
|
||||
self.detector_instance = self._initialize_processor(output_category_mask, output_confidence_mask)
|
||||
if self.detector_instance and hasattr(self.detector_instance, "close"):
|
||||
self.detector_instance.close()
|
||||
self.detector_instance = self._initialize_processor(output_category_mask, output_confidence_mask)
|
||||
|
||||
for i in range(batch_size):
|
||||
img_tensor = image[i] # HWC
|
||||
img_tensor = image[i] # HWC
|
||||
np_image = (img_tensor.cpu().numpy() * 255).astype(np.uint8)
|
||||
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=np_image)
|
||||
|
||||
@@ -80,12 +82,11 @@ class InteractiveSegmenterProcessor:
|
||||
for point in points_of_interest:
|
||||
roi_keypoint_single = NormalizedKeypoint(x=point.x, y=point.y, label=point.label)
|
||||
roi = interactive_segmenter.RegionOfInterest(
|
||||
format=interactive_segmenter.RegionOfInterest.Format.KEYPOINT,
|
||||
keypoint=roi_keypoint_single
|
||||
format=interactive_segmenter.RegionOfInterest.Format.KEYPOINT, keypoint=roi_keypoint_single
|
||||
)
|
||||
|
||||
|
||||
segmentation_result = self.detector_instance.segment(mp_image, roi)
|
||||
|
||||
|
||||
current_point_mask = None
|
||||
if output_category_mask and segmentation_result.category_mask:
|
||||
mask_np = segmentation_result.category_mask.numpy_view()
|
||||
@@ -94,7 +95,7 @@ class InteractiveSegmenterProcessor:
|
||||
if segmentation_result.confidence_masks:
|
||||
mask_np = segmentation_result.confidence_masks[0].numpy_view()
|
||||
current_point_mask = torch.from_numpy(mask_np).float()
|
||||
|
||||
|
||||
if current_point_mask is not None:
|
||||
individual_masks_for_image.append(current_point_mask)
|
||||
else:
|
||||
@@ -109,6 +110,6 @@ class InteractiveSegmenterProcessor:
|
||||
else:
|
||||
mask_stack = torch.stack(individual_masks_for_image, dim=0)
|
||||
combined_mask_for_image = torch.max(mask_stack, dim=0).values
|
||||
|
||||
|
||||
batch_results_combined.append(combined_mask_for_image)
|
||||
return batch_results_combined
|
||||
return batch_results_combined
|
||||
|
||||
@@ -44,7 +44,6 @@ FACE_LANDMARK_DESCRIPTIONS = {
|
||||
86: "Bottom lip bottom center",
|
||||
14: "Upper teeth",
|
||||
13: "Lower teeth",
|
||||
|
||||
# General regions (for tooltip summary)
|
||||
# 0-17: Jaw/face outline
|
||||
# 18-35: Eyebrows
|
||||
@@ -75,7 +74,7 @@ Notable:
|
||||
# Hand landmarks (21 points)
|
||||
HAND_LANDMARK_DESCRIPTIONS = {
|
||||
0: "Wrist",
|
||||
1: "Thumb CMC",
|
||||
1: "Thumb CMC",
|
||||
2: "Thumb MCP",
|
||||
3: "Thumb IP",
|
||||
4: "Thumb tip",
|
||||
@@ -94,7 +93,7 @@ HAND_LANDMARK_DESCRIPTIONS = {
|
||||
17: "Pinky MCP",
|
||||
18: "Pinky PIP",
|
||||
19: "Pinky DIP",
|
||||
20: "Pinky tip"
|
||||
20: "Pinky tip",
|
||||
}
|
||||
|
||||
# Tooltip for hand landmarks
|
||||
@@ -143,7 +142,7 @@ POSE_LANDMARK_DESCRIPTIONS = {
|
||||
29: "Left heel",
|
||||
30: "Right heel",
|
||||
31: "Left foot index",
|
||||
32: "Right foot index"
|
||||
32: "Right foot index",
|
||||
}
|
||||
|
||||
# Tooltip for pose landmarks
|
||||
@@ -158,4 +157,4 @@ POSE_LANDMARK_TOOLTIP = """MediaPipe Pose Landmarks (0-32):
|
||||
23-24: Hips
|
||||
25-26: Knees
|
||||
27-28: Ankles
|
||||
29-32: Feet"""
|
||||
29-32: Feet"""
|
||||
|
||||
@@ -3,18 +3,19 @@
|
||||
Handles model definition, download, and path retrieval.
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
import urllib.request
|
||||
import folder_paths # ComfyUI path helper
|
||||
from typing import Dict, Any, Optional, List
|
||||
import os
|
||||
import traceback
|
||||
import urllib.request
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import folder_paths # ComfyUI path helper
|
||||
|
||||
# Configure logging
|
||||
# logging.basicConfig(level=logging.INFO) # Avoid reconfiguring root logger
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# --- Model Definitions ---
|
||||
# --- Model Definitions ---
|
||||
# Central dictionary mapping task types and variants to model details
|
||||
# Structure: { task_type: { variant_name: { url: ..., filename: ... } } }
|
||||
# Task types should match the keys used elsewhere (e.g., detector class validation)
|
||||
@@ -27,99 +28,99 @@ MEDIAPIPE_MODELS = {
|
||||
"short_range": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/face_detector/blaze_face_short_range/float16/latest/blaze_face_short_range.tflite",
|
||||
"filename": "blaze_face_short_range.tflite",
|
||||
"description": "Optimized for detecting faces at close range, ideal for selfie-style images and near-field face detection."
|
||||
"description": "Optimized for detecting faces at close range, ideal for selfie-style images and near-field face detection.",
|
||||
},
|
||||
"full_range_sparse": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/face_detector/blaze_face_back_camera/float16/latest/blaze_face_back_camera.tflite",
|
||||
"filename": "blaze_face_full_range_sparse.tflite",
|
||||
"description": "Designed for detecting faces at various distances, suitable for group photos and wider-angle shots."
|
||||
}
|
||||
"description": "Designed for detecting faces at various distances, suitable for group photos and wider-angle shots.",
|
||||
},
|
||||
},
|
||||
"face_landmarker": {
|
||||
"default": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/latest/face_landmarker.task",
|
||||
"filename": "face_landmarker.task",
|
||||
"description": "Standard face landmark detection model that locates key facial points for tracking and analysis."
|
||||
"description": "Standard face landmark detection model that locates key facial points for tracking and analysis.",
|
||||
},
|
||||
"with_blendshapes": { # If a separate model exists
|
||||
"with_blendshapes": { # If a separate model exists
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker_v2_with_blendshapes/float16/latest/face_landmarker_v2_with_blendshapes.task",
|
||||
"filename": "face_landmarker_v2_with_blendshapes.task",
|
||||
"description": "Advanced face landmark model that includes blendshape coefficients for facial expression analysis."
|
||||
}
|
||||
"description": "Advanced face landmark model that includes blendshape coefficients for facial expression analysis.",
|
||||
},
|
||||
},
|
||||
"hand_landmarker": {
|
||||
"default": { # Often includes world landmarks
|
||||
"default": { # Often includes world landmarks
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/latest/hand_landmarker.task",
|
||||
"filename": "hand_landmarker.task",
|
||||
"description": "Detects hand landmarks for tracking finger positions and hand gestures in 2D and 3D space."
|
||||
"description": "Detects hand landmarks for tracking finger positions and hand gestures in 2D and 3D space.",
|
||||
}
|
||||
},
|
||||
"pose_landmarker": {
|
||||
"lite": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/latest/pose_landmarker_lite.task",
|
||||
"filename": "pose_landmarker_lite.task",
|
||||
"description": "Lightweight pose landmark model for faster inference with slightly reduced accuracy."
|
||||
"description": "Lightweight pose landmark model for faster inference with slightly reduced accuracy.",
|
||||
},
|
||||
"full": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_full/float16/latest/pose_landmarker_full.task",
|
||||
"filename": "pose_landmarker_full.task",
|
||||
"description": "Balance of performance and accuracy for detecting body pose landmarks in typical use cases."
|
||||
"description": "Balance of performance and accuracy for detecting body pose landmarks in typical use cases.",
|
||||
},
|
||||
"heavy": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_heavy/float16/latest/pose_landmarker_heavy.task",
|
||||
"filename": "pose_landmarker_heavy.task",
|
||||
"description": "Higher accuracy pose detection model for more precise landmark tracking at the cost of performance."
|
||||
}
|
||||
"description": "Higher accuracy pose detection model for more precise landmark tracking at the cost of performance.",
|
||||
},
|
||||
},
|
||||
"object_detector": {
|
||||
"efficientdet_lite0": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/object_detector/efficientdet_lite0/float16/latest/efficientdet_lite0.tflite",
|
||||
"filename": "efficientdet_lite0.tflite",
|
||||
"description": "Lightweight object detection model that balances speed and accuracy for real-time applications."
|
||||
"description": "Lightweight object detection model that balances speed and accuracy for real-time applications.",
|
||||
},
|
||||
"efficientdet_lite2": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/object_detector/efficientdet_lite2/float16/latest/efficientdet_lite2.tflite",
|
||||
"filename": "efficientdet_lite2.tflite",
|
||||
"description": "Medium-sized object detection model with improved accuracy over lite0 while maintaining reasonable performance."
|
||||
},
|
||||
"ssd_mobilenet_v2": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/object_detector/ssd_mobilenet_v2/float32/latest/ssd_mobilenet_v2.tflite",
|
||||
"filename": "ssd_mobilenet_v2.tflite",
|
||||
"description": "Single Shot Detector with MobileNetV2 backbone for efficient object detection tasks."
|
||||
}
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/object_detector/efficientdet_lite2/float16/latest/efficientdet_lite2.tflite",
|
||||
"filename": "efficientdet_lite2.tflite",
|
||||
"description": "Medium-sized object detection model with improved accuracy over lite0 while maintaining reasonable performance.",
|
||||
},
|
||||
"ssd_mobilenet_v2": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/object_detector/ssd_mobilenet_v2/float32/latest/ssd_mobilenet_v2.tflite",
|
||||
"filename": "ssd_mobilenet_v2.tflite",
|
||||
"description": "Single Shot Detector with MobileNetV2 backbone for efficient object detection tasks.",
|
||||
},
|
||||
},
|
||||
"image_segmenter": {
|
||||
"deeplab_v3": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/image_segmenter/deeplab_v3/float32/latest/deeplab_v3.tflite",
|
||||
"filename": "deeplab_v3.tflite",
|
||||
"description": "General-purpose semantic segmentation model that can segment images into different object categories."
|
||||
"description": "General-purpose semantic segmentation model that can segment images into different object categories.",
|
||||
},
|
||||
"selfie_segmenter_landscape": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/latest/selfie_segmenter_landscape.tflite",
|
||||
"filename": "selfie_segmenter_landscape.tflite",
|
||||
"description": "Optimized for landscape-oriented images to accurately separate people from backgrounds."
|
||||
"description": "Optimized for landscape-oriented images to accurately separate people from backgrounds.",
|
||||
},
|
||||
"selfie_segmenter_general": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite",
|
||||
"filename": "selfie_segmenter.tflite",
|
||||
"description": "General-purpose person segmentation model that works across different orientations and scenarios."
|
||||
"description": "General-purpose person segmentation model that works across different orientations and scenarios.",
|
||||
},
|
||||
"hair_segmenter": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/image_segmenter/hair_segmenter/float32/latest/hair_segmenter.tflite",
|
||||
"filename": "hair_segmenter.tflite",
|
||||
"description": "Specialized segmentation model for isolating hair in portrait images with high precision."
|
||||
"description": "Specialized segmentation model for isolating hair in portrait images with high precision.",
|
||||
},
|
||||
"selfie_multiclass": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_multiclass_256x256/float32/latest/selfie_multiclass_256x256.tflite",
|
||||
"filename": "selfie_multiclass_256x256.tflite",
|
||||
"description": "Multi-class segmentation model that can separate people into different body parts and clothing items."
|
||||
}
|
||||
"description": "Multi-class segmentation model that can separate people into different body parts and clothing items.",
|
||||
},
|
||||
},
|
||||
"gesture_recognizer": {
|
||||
"default": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/latest/gesture_recognizer.task",
|
||||
"filename": "gesture_recognizer.task",
|
||||
"description": "Detects and classifies hand gestures such as thumbs up, victory sign, and pointing gestures."
|
||||
"description": "Detects and classifies hand gestures such as thumbs up, victory sign, and pointing gestures.",
|
||||
}
|
||||
},
|
||||
# "holistic_landmarker": {
|
||||
@@ -134,74 +135,75 @@ MEDIAPIPE_MODELS = {
|
||||
# }
|
||||
# Add other tasks like image_classification etc. here
|
||||
"image_embedder": {
|
||||
"mobilenet_v3_small": {
|
||||
"mobilenet_v3_small": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/image_embedder/mobilenet_v3_small/float32/1/mobilenet_v3_small.tflite",
|
||||
"filename": "mobilenet_v3_small.tflite",
|
||||
"description": "Converts images into feature vectors (embeddings) for similarity comparison and classification tasks."
|
||||
"description": "Converts images into feature vectors (embeddings) for similarity comparison and classification tasks.",
|
||||
},
|
||||
"efficientnet_lite0": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/image_embedder/efficientnet_lite0/float32/latest/efficientnet_lite0.tflite",
|
||||
"filename": "efficientnet_lite0.tflite",
|
||||
"description": "EfficientNet Lite0-based embedder providing robust image embeddings with a balance of accuracy and efficiency."
|
||||
}
|
||||
"description": "EfficientNet Lite0-based embedder providing robust image embeddings with a balance of accuracy and efficiency.",
|
||||
},
|
||||
},
|
||||
# Placeholder for Interactive Segmenter - Typically uses a general segmentation model
|
||||
"interactive_segmenter": {
|
||||
"magic_touch": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/interactive_segmenter/magic_touch/float32/latest/magic_touch.tflite",
|
||||
"filename": "magic_touch.tflite",
|
||||
"description": "Interactive segmentation model that segments objects in an image based on a user-provided point of interest."
|
||||
"description": "Interactive segmentation model that segments objects in an image based on a user-provided point of interest.",
|
||||
},
|
||||
"default": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite",
|
||||
"filename": "selfie_segmenter_interactive.tflite",
|
||||
"description": "Enables user-guided segmentation by providing points or regions to indicate foreground objects."
|
||||
}
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite",
|
||||
"filename": "selfie_segmenter_interactive.tflite",
|
||||
"description": "Enables user-guided segmentation by providing points or regions to indicate foreground objects.",
|
||||
},
|
||||
},
|
||||
"face_stylizer": {
|
||||
"color_sketch": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/face_stylizer/blaze_face_stylizer/float32/latest/face_stylizer_color_sketch.task",
|
||||
"filename": "face_stylizer_color_sketch.task",
|
||||
"description": "Transforms faces into colorful sketch-like artistic renderings with clean lines and vibrant colors."
|
||||
"description": "Transforms faces into colorful sketch-like artistic renderings with clean lines and vibrant colors.",
|
||||
},
|
||||
"color_ink": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/face_stylizer/blaze_face_stylizer/float32/latest/face_stylizer_color_ink.task",
|
||||
"filename": "face_stylizer_color_ink.task",
|
||||
"description": "Applies a color ink effect to faces with flowing brush-like strokes and dynamic color blending."
|
||||
"description": "Applies a color ink effect to faces with flowing brush-like strokes and dynamic color blending.",
|
||||
},
|
||||
"oil_painting": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/face_stylizer/blaze_face_stylizer/float32/latest/face_stylizer_oil_painting.task",
|
||||
"filename": "face_stylizer_oil_painting.task",
|
||||
"description": "Renders faces in classic oil painting style with textured brush strokes and rich color tones."
|
||||
}
|
||||
"description": "Renders faces in classic oil painting style with textured brush strokes and rich color tones.",
|
||||
},
|
||||
},
|
||||
"image_classifier": {
|
||||
"efficientnet_lite0": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/image_classifier/efficientnet_lite0/float32/latest/efficientnet_lite0.tflite",
|
||||
"filename": "efficientnet_lite0.tflite",
|
||||
"description": "EfficientNet-Lite0 model for general-purpose image classification tasks with optimized performance."
|
||||
"description": "EfficientNet-Lite0 model for general-purpose image classification tasks with optimized performance.",
|
||||
},
|
||||
"efficientnet_lite2": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/image_classifier/efficientnet_lite2/float32/latest/efficientnet_lite2.tflite",
|
||||
"filename": "efficientnet_lite2.tflite",
|
||||
"description": "EfficientNet-Lite2 model offering higher accuracy for image classification tasks at the cost of increased computation."
|
||||
}
|
||||
"description": "EfficientNet-Lite2 model offering higher accuracy for image classification tasks at the cost of increased computation.",
|
||||
},
|
||||
},
|
||||
"face_mesh": {
|
||||
"default": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/face_mesh/face_mesh/float16/latest/face_mesh.tflite",
|
||||
"filename": "face_mesh.tflite",
|
||||
"description": "High-fidelity 3D face landmark model estimating 468 landmarks for detailed facial geometry analysis."
|
||||
"description": "High-fidelity 3D face landmark model estimating 468 landmarks for detailed facial geometry analysis.",
|
||||
},
|
||||
"attention": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/face_mesh/face_mesh_attention/float16/latest/face_mesh_attention.tflite",
|
||||
"filename": "face_mesh_attention.tflite",
|
||||
"description": "Enhanced face mesh model with attention mechanism for improved accuracy around eyes and lips."
|
||||
}
|
||||
}
|
||||
"description": "Enhanced face mesh model with attention mechanism for improved accuracy around eyes and lips.",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
# --- Utility Functions ---
|
||||
# --- Utility Functions ---
|
||||
|
||||
|
||||
def get_mediapipe_model_dir(task_type: str) -> str:
|
||||
"""Gets the dedicated directory for a specific MediaPipe task type within ComfyUI models."""
|
||||
@@ -213,7 +215,7 @@ def get_mediapipe_model_dir(task_type: str) -> str:
|
||||
base_path = mediapipe_paths[0]
|
||||
logger.debug(f"Using specific MediaPipe model path: {base_path}")
|
||||
else:
|
||||
logger.debug("No valid specific 'mediapipe' path found or defined.")
|
||||
logger.debug("No valid specific 'mediapipe' path found or defined.")
|
||||
except (KeyError, IndexError, TypeError) as e:
|
||||
logger.debug(f"Could not retrieve specific 'mediapipe' path ({e}). Falling back to default.")
|
||||
|
||||
@@ -221,11 +223,11 @@ def get_mediapipe_model_dir(task_type: str) -> str:
|
||||
if base_path is None:
|
||||
try:
|
||||
# Ensure models_dir exists
|
||||
if hasattr(folder_paths, 'models_dir') and os.path.isdir(folder_paths.models_dir):
|
||||
if hasattr(folder_paths, "models_dir") and os.path.isdir(folder_paths.models_dir):
|
||||
base_path = os.path.join(folder_paths.models_dir, "mediapipe")
|
||||
logger.debug(f"Using fallback MediaPipe model path: {base_path}")
|
||||
else:
|
||||
raise RuntimeError("ComfyUI models directory ('models_dir') not found or invalid.")
|
||||
raise RuntimeError("ComfyUI models directory ('models_dir') not found or invalid.")
|
||||
except Exception as e:
|
||||
logger.error(f"Error determining fallback MediaPipe directory: {e}")
|
||||
raise RuntimeError("Could not determine a valid directory for MediaPipe models.") from e
|
||||
@@ -243,17 +245,18 @@ def get_mediapipe_model_dir(task_type: str) -> str:
|
||||
|
||||
return task_dir
|
||||
|
||||
|
||||
def _download_url_to_file(url: str, save_path: str):
|
||||
"""Downloads a file with progress indication."""
|
||||
logger.info(f"Downloading model from {url} to {save_path}...")
|
||||
try:
|
||||
with urllib.request.urlopen(url) as response, open(save_path, 'wb') as out_file:
|
||||
total_size_header = response.getheader('Content-Length')
|
||||
with urllib.request.urlopen(url) as response, open(save_path, "wb") as out_file:
|
||||
total_size_header = response.getheader("Content-Length")
|
||||
total_size = int(total_size_header) if total_size_header else None
|
||||
bytes_downloaded = 0
|
||||
block_size = 8192
|
||||
last_print_bytes = 0
|
||||
print_interval = 1 * 1024 * 1024 # Print every 1MB
|
||||
print_interval = 1 * 1024 * 1024 # Print every 1MB
|
||||
|
||||
if total_size:
|
||||
logger.info(f" File size: {total_size / (1024*1024):.2f} MB")
|
||||
@@ -274,10 +277,10 @@ def _download_url_to_file(url: str, save_path: str):
|
||||
progress = (bytes_downloaded / total_size) * 100
|
||||
progress_str += f" / {total_size / (1024*1024):.2f} MB ({progress:.1f}%)"
|
||||
# Use print with carriage return for inline update
|
||||
print(progress_str, end=' \r')
|
||||
print(progress_str, end=" \r")
|
||||
last_print_bytes = bytes_downloaded
|
||||
|
||||
print() # Newline after download finishes
|
||||
print() # Newline after download finishes
|
||||
logger.info(f"Download complete: {save_path}")
|
||||
return True
|
||||
except Exception as e:
|
||||
@@ -292,6 +295,7 @@ def _download_url_to_file(url: str, save_path: str):
|
||||
logger.error(f"Error removing incomplete download {save_path}: {rm_err}")
|
||||
return False
|
||||
|
||||
|
||||
def get_model_path(task_type: str, model_variant: str) -> Optional[str]:
|
||||
"""Gets the full path to the model file, downloading if necessary."""
|
||||
if task_type not in MEDIAPIPE_MODELS:
|
||||
@@ -307,32 +311,35 @@ def get_model_path(task_type: str, model_variant: str) -> Optional[str]:
|
||||
model_details = MEDIAPIPE_MODELS[task_type][model_variant]
|
||||
model_filename = model_details["filename"]
|
||||
model_url = model_details["url"]
|
||||
|
||||
|
||||
# Get the specific directory for this task
|
||||
try:
|
||||
model_dir = get_mediapipe_model_dir(task_type)
|
||||
except Exception as e:
|
||||
logger.error(f"Cannot determine model directory for {task_type}: {e}")
|
||||
return None
|
||||
|
||||
logger.error(f"Cannot determine model directory for {task_type}: {e}")
|
||||
return None
|
||||
|
||||
model_path = os.path.join(model_dir, model_filename)
|
||||
|
||||
# Check and download
|
||||
if not os.path.exists(model_path):
|
||||
logger.info(f"Model not found locally at {model_path}. Attempting download.")
|
||||
if not _download_url_to_file(model_url, model_path):
|
||||
return None # Download failed
|
||||
return None # Download failed
|
||||
else:
|
||||
logger.debug(f"Found model locally: {model_path}")
|
||||
|
||||
return model_path
|
||||
|
||||
# --- Public Functions for Node Wrapper ---
|
||||
|
||||
# --- Public Functions for Node Wrapper ---
|
||||
|
||||
|
||||
def get_available_tasks() -> List[str]:
|
||||
"""Get a list of available MediaPipe Vision task types from the definition."""
|
||||
return list(MEDIAPIPE_MODELS.keys())
|
||||
|
||||
|
||||
def get_available_models(task_type: str) -> List[str]:
|
||||
"""Get a list of available model variants for a specific task."""
|
||||
if task_type in MEDIAPIPE_MODELS:
|
||||
@@ -341,50 +348,51 @@ def get_available_models(task_type: str) -> List[str]:
|
||||
logger.warning(f"Task type '{task_type}' not found in model definitions.")
|
||||
return []
|
||||
|
||||
|
||||
def get_model_description(task_type: str, model_variant: str) -> str:
|
||||
"""Get the description for a specific model variant.
|
||||
|
||||
|
||||
Args:
|
||||
task_type: The MediaPipe task type
|
||||
model_variant: The specific model variant
|
||||
|
||||
|
||||
Returns:
|
||||
The description string if found, or a default message if not available
|
||||
"""
|
||||
if task_type in MEDIAPIPE_MODELS and model_variant in MEDIAPIPE_MODELS[task_type]:
|
||||
return MEDIAPIPE_MODELS[task_type][model_variant].get(
|
||||
"description", "No description available"
|
||||
)
|
||||
return MEDIAPIPE_MODELS[task_type][model_variant].get("description", "No description available")
|
||||
return "Model description not available"
|
||||
|
||||
# --- Model Loader Class (Simplified) ---
|
||||
|
||||
# --- Model Loader Class (Simplified) ---
|
||||
|
||||
|
||||
class MediaPipeModelLoader:
|
||||
"""Helper class to manage loading state and info.
|
||||
Used temporarily by the loader node to get model info.
|
||||
Used temporarily by the loader node to get model info.
|
||||
"""
|
||||
|
||||
|
||||
def __init__(self):
|
||||
self.model_path = None
|
||||
self.task_type = None
|
||||
self.model_variant = None
|
||||
self.model_loaded = False
|
||||
self.model_info = {}
|
||||
|
||||
|
||||
def load_model(self, task_type: str, model_variant: str) -> bool:
|
||||
"""Sets model info and triggers path retrieval (incl. download)."""
|
||||
logger.debug(f"MediaPipeModelLoader: Loading {task_type}/{model_variant}")
|
||||
self.task_type = task_type
|
||||
self.model_variant = model_variant
|
||||
self.model_path = get_model_path(task_type, model_variant)
|
||||
|
||||
|
||||
if self.model_path and os.path.exists(self.model_path):
|
||||
self.model_loaded = True
|
||||
self.model_info = {
|
||||
'task_type': self.task_type,
|
||||
'model_variant': self.model_variant,
|
||||
'model_path': self.model_path,
|
||||
'description': get_model_description(task_type, model_variant)
|
||||
"task_type": self.task_type,
|
||||
"model_variant": self.model_variant,
|
||||
"model_path": self.model_path,
|
||||
"description": get_model_description(task_type, model_variant),
|
||||
}
|
||||
logger.debug(f"Model info set: {self.model_info}")
|
||||
return True
|
||||
@@ -393,23 +401,24 @@ class MediaPipeModelLoader:
|
||||
self.model_loaded = False
|
||||
self.model_info = {}
|
||||
return False
|
||||
|
||||
|
||||
def get_model_info(self) -> Dict[str, Any]:
|
||||
"""Get information about the loaded model."""
|
||||
# Ensure info is consistent with loaded state
|
||||
if self.model_loaded:
|
||||
# Add the description to the model info
|
||||
if self.task_type and self.model_variant:
|
||||
description = get_model_description(self.task_type, self.model_variant)
|
||||
self.model_info['description'] = description
|
||||
return self.model_info
|
||||
# Add the description to the model info
|
||||
if self.task_type and self.model_variant:
|
||||
description = get_model_description(self.task_type, self.model_variant)
|
||||
self.model_info["description"] = description
|
||||
return self.model_info
|
||||
else:
|
||||
# Return minimal info or empty dict if not loaded
|
||||
return {
|
||||
'task_type': self.task_type,
|
||||
'model_variant': self.model_variant,
|
||||
'model_path': None # Indicate path is not valid/loaded
|
||||
}
|
||||
# Return minimal info or empty dict if not loaded
|
||||
return {
|
||||
"task_type": self.task_type,
|
||||
"model_variant": self.model_variant,
|
||||
"model_path": None, # Indicate path is not valid/loaded
|
||||
}
|
||||
|
||||
|
||||
# Removed original get_model_path, download_model etc. from global scope
|
||||
# They are now integrated into the new get_model_path function.
|
||||
# They are now integrated into the new get_model_path function.
|
||||
|
||||
@@ -1 +1 @@
|
||||
"""Object Detection implementation using MediaPipe."""
|
||||
"""Object Detection implementation using MediaPipe."""
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
from typing import List
|
||||
|
||||
import mediapipe as mp
|
||||
import numpy as np
|
||||
import torch
|
||||
from mediapipe.tasks import python
|
||||
from mediapipe.tasks.python import vision
|
||||
from mediapipe.tasks.python.core.base_options import BaseOptions
|
||||
from typing import List
|
||||
# Import new types
|
||||
from ..types import ObjectDetectionResult, ObjectDetectionCategory, BoundingBox
|
||||
|
||||
from ...utils.timing import TimestampProvider
|
||||
|
||||
# Import new types
|
||||
from ..types import BoundingBox, ObjectDetectionCategory, ObjectDetectionResult
|
||||
|
||||
|
||||
class ObjectDetector:
|
||||
"""Detects objects using MediaPipe ObjectDetector."""
|
||||
|
||||
@@ -24,27 +28,32 @@ class ObjectDetector:
|
||||
self.model_path = model_path
|
||||
self._detector_instance = None
|
||||
self._current_options = None
|
||||
self._timestamp_provider = None # Added
|
||||
self._timestamp_provider = None # Added
|
||||
|
||||
def _initialize_detector(self, score_threshold: float, max_results: int,
|
||||
running_mode: str, delegate: str): # Added running_mode
|
||||
def _initialize_detector(self, score_threshold: float, max_results: int, running_mode: str, delegate: str): # Added running_mode
|
||||
"""Initializes the ObjectDetector."""
|
||||
mode_enum = vision.RunningMode.IMAGE if running_mode == "image" else vision.RunningMode.VIDEO # Added
|
||||
delegate_enum = BaseOptions.Delegate.CPU if delegate.lower() == 'cpu' else BaseOptions.Delegate.GPU
|
||||
mode_enum = vision.RunningMode.IMAGE if running_mode == "image" else vision.RunningMode.VIDEO # Added
|
||||
delegate_enum = BaseOptions.Delegate.CPU if delegate.lower() == "cpu" else BaseOptions.Delegate.GPU
|
||||
base_options = python.BaseOptions(model_asset_path=self.model_path, delegate=delegate_enum)
|
||||
options = vision.ObjectDetectorOptions(
|
||||
base_options=base_options,
|
||||
running_mode=mode_enum, # Use enum mode
|
||||
running_mode=mode_enum, # Use enum mode
|
||||
score_threshold=score_threshold,
|
||||
max_results=max_results
|
||||
max_results=max_results,
|
||||
)
|
||||
self._current_options = options
|
||||
# Only create timestamp provider for VIDEO mode
|
||||
self._timestamp_provider = TimestampProvider() if mode_enum == vision.RunningMode.VIDEO else None
|
||||
self._timestamp_provider = TimestampProvider() if mode_enum == vision.RunningMode.VIDEO else None
|
||||
return vision.ObjectDetector.create_from_options(options)
|
||||
|
||||
def detect(self, image: torch.Tensor, score_threshold: float = 0.5, max_results: int = 5,
|
||||
running_mode: str = "video", delegate: str = 'cpu') -> List[List[ObjectDetectionResult]]: # Added running_mode
|
||||
def detect(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
score_threshold: float = 0.5,
|
||||
max_results: int = 5,
|
||||
running_mode: str = "video",
|
||||
delegate: str = "cpu",
|
||||
) -> List[List[ObjectDetectionResult]]: # Added running_mode
|
||||
"""Detects objects in the input image tensor.
|
||||
|
||||
Returns:
|
||||
@@ -57,16 +66,16 @@ class ObjectDetector:
|
||||
new_options_tuple = (score_threshold, max_results, running_mode, delegate.lower())
|
||||
current_options_tuple = None
|
||||
if self._current_options:
|
||||
current_options_tuple = (
|
||||
self._current_options.score_threshold,
|
||||
self._current_options.max_results,
|
||||
"image" if self._current_options.running_mode == vision.RunningMode.IMAGE else "video", # Map enum back to string
|
||||
'cpu' if self._current_options.base_options.delegate == BaseOptions.Delegate.CPU else 'gpu'
|
||||
)
|
||||
current_options_tuple = (
|
||||
self._current_options.score_threshold,
|
||||
self._current_options.max_results,
|
||||
"image" if self._current_options.running_mode == vision.RunningMode.IMAGE else "video", # Map enum back to string
|
||||
"cpu" if self._current_options.base_options.delegate == BaseOptions.Delegate.CPU else "gpu",
|
||||
)
|
||||
|
||||
if self._detector_instance is None or new_options_tuple != current_options_tuple:
|
||||
if self._detector_instance and hasattr(self._detector_instance, 'close'):
|
||||
self._detector_instance.close()
|
||||
if self._detector_instance and hasattr(self._detector_instance, "close"):
|
||||
self._detector_instance.close()
|
||||
self._detector_instance = self._initialize_detector(score_threshold, max_results, running_mode, delegate)
|
||||
|
||||
batch_size = image.shape[0]
|
||||
@@ -74,9 +83,9 @@ class ObjectDetector:
|
||||
# Determine the correct detection function based on mode
|
||||
is_video_mode = self._current_options.running_mode == vision.RunningMode.VIDEO
|
||||
detect_func = self._detector_instance.detect_for_video if is_video_mode else self._detector_instance.detect
|
||||
|
||||
|
||||
for i in range(batch_size):
|
||||
img_tensor = image[i] # HWC
|
||||
img_tensor = image[i] # HWC
|
||||
np_image = (img_tensor.cpu().numpy() * 255).astype(np.uint8)
|
||||
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=np_image)
|
||||
|
||||
@@ -85,36 +94,28 @@ class ObjectDetector:
|
||||
detection_result = detect_func(mp_image, timestamp_ms)
|
||||
else:
|
||||
detection_result = detect_func(mp_image)
|
||||
|
||||
|
||||
current_image_detections = []
|
||||
if detection_result and detection_result.detections: # Added check for detection_result
|
||||
if detection_result and detection_result.detections: # Added check for detection_result
|
||||
for detection in detection_result.detections:
|
||||
bbox_mp = detection.bounding_box
|
||||
categories_mp = detection.categories
|
||||
|
||||
bbox = BoundingBox(
|
||||
origin_x=bbox_mp.origin_x,
|
||||
origin_y=bbox_mp.origin_y,
|
||||
width=bbox_mp.width,
|
||||
height=bbox_mp.height
|
||||
)
|
||||
|
||||
|
||||
bbox = BoundingBox(origin_x=bbox_mp.origin_x, origin_y=bbox_mp.origin_y, width=bbox_mp.width, height=bbox_mp.height)
|
||||
|
||||
categories = [
|
||||
ObjectDetectionCategory(
|
||||
index=cat.index,
|
||||
index=cat.index,
|
||||
score=cat.score,
|
||||
display_name=cat.display_name,
|
||||
category_name=cat.category_name
|
||||
display_name=cat.display_name,
|
||||
category_name=cat.category_name,
|
||||
)
|
||||
for cat in categories_mp
|
||||
]
|
||||
|
||||
current_image_detections.append(ObjectDetectionResult(
|
||||
bounding_box=bbox,
|
||||
categories=categories
|
||||
))
|
||||
|
||||
|
||||
current_image_detections.append(ObjectDetectionResult(bounding_box=bbox, categories=categories))
|
||||
|
||||
batch_results.append(current_image_detections)
|
||||
|
||||
# self._detector_instance.close() # Close handled on re-init
|
||||
return batch_results
|
||||
return batch_results
|
||||
|
||||
@@ -1 +1 @@
|
||||
"""Pose Landmark Detection implementation using MediaPipe."""
|
||||
"""Pose Landmark Detection implementation using MediaPipe."""
|
||||
|
||||
@@ -1,17 +1,21 @@
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import mediapipe as mp
|
||||
import numpy as np
|
||||
import torch
|
||||
from mediapipe.tasks import python
|
||||
from mediapipe.tasks.python import vision
|
||||
from mediapipe.tasks.python.core.base_options import BaseOptions
|
||||
from typing import List, Tuple, Optional
|
||||
|
||||
from ...utils.timing import TimestampProvider
|
||||
|
||||
# Import new types
|
||||
from ..types import LandmarkPoint, PoseLandmarksResult
|
||||
from ...utils.timing import TimestampProvider
|
||||
|
||||
|
||||
class PoseLandmarkDetector:
|
||||
"""Detects pose landmarks in an image using MediaPipe PoseLandmarker."""
|
||||
|
||||
|
||||
def __init__(self, model_path: str):
|
||||
"""Initialize the detector with the model path.
|
||||
|
||||
@@ -20,38 +24,51 @@ class PoseLandmarkDetector:
|
||||
"""
|
||||
if not model_path:
|
||||
raise ValueError("A valid model_path must be provided.")
|
||||
|
||||
|
||||
self.model_path = model_path
|
||||
self._detector_instance = None
|
||||
self._current_options = None
|
||||
self._timestamp_provider = None # Added
|
||||
self._timestamp_provider = None # Added
|
||||
|
||||
def _initialize_detector(self, num_poses: int, min_detection_confidence: float,
|
||||
min_presence_confidence: float, min_tracking_confidence: float,
|
||||
output_segmentation_masks: bool,
|
||||
running_mode: str, delegate: str) -> vision.PoseLandmarker: # Added running_mode
|
||||
def _initialize_detector(
|
||||
self,
|
||||
num_poses: int,
|
||||
min_detection_confidence: float,
|
||||
min_presence_confidence: float,
|
||||
min_tracking_confidence: float,
|
||||
output_segmentation_masks: bool,
|
||||
running_mode: str,
|
||||
delegate: str,
|
||||
) -> vision.PoseLandmarker: # Added running_mode
|
||||
"""Initializes the PoseLandmarker detector."""
|
||||
mode_enum = vision.RunningMode.IMAGE if running_mode == "image" else vision.RunningMode.VIDEO # Added
|
||||
delegate_enum = BaseOptions.Delegate.CPU if delegate.lower() == 'cpu' else BaseOptions.Delegate.GPU
|
||||
mode_enum = vision.RunningMode.IMAGE if running_mode == "image" else vision.RunningMode.VIDEO # Added
|
||||
delegate_enum = BaseOptions.Delegate.CPU if delegate.lower() == "cpu" else BaseOptions.Delegate.GPU
|
||||
base_options = python.BaseOptions(model_asset_path=self.model_path, delegate=delegate_enum)
|
||||
options = vision.PoseLandmarkerOptions(
|
||||
base_options=base_options,
|
||||
running_mode=mode_enum, # Use enum mode
|
||||
running_mode=mode_enum, # Use enum mode
|
||||
num_poses=num_poses,
|
||||
min_pose_detection_confidence=min_detection_confidence,
|
||||
min_pose_presence_confidence=min_presence_confidence,
|
||||
min_pose_presence_confidence=min_presence_confidence,
|
||||
min_tracking_confidence=min_tracking_confidence,
|
||||
output_segmentation_masks=output_segmentation_masks
|
||||
output_segmentation_masks=output_segmentation_masks,
|
||||
)
|
||||
self._current_options = options
|
||||
# Only create timestamp provider for VIDEO mode
|
||||
self._timestamp_provider = TimestampProvider() if mode_enum == vision.RunningMode.VIDEO else None
|
||||
self._timestamp_provider = TimestampProvider() if mode_enum == vision.RunningMode.VIDEO else None
|
||||
return vision.PoseLandmarker.create_from_options(options)
|
||||
|
||||
def detect(self, image: torch.Tensor, num_poses: int = 1, min_detection_confidence: float = 0.5,
|
||||
min_presence_confidence: float = 0.5, min_tracking_confidence: float = 0.5,
|
||||
output_segmentation_masks: bool = False,
|
||||
running_mode: str = "video", delegate: str = 'cpu') -> Tuple[List[List[PoseLandmarksResult]], Optional[List[List[torch.Tensor]]]]: # Added running_mode
|
||||
def detect(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
num_poses: int = 1,
|
||||
min_detection_confidence: float = 0.5,
|
||||
min_presence_confidence: float = 0.5,
|
||||
min_tracking_confidence: float = 0.5,
|
||||
output_segmentation_masks: bool = False,
|
||||
running_mode: str = "video",
|
||||
delegate: str = "cpu",
|
||||
) -> Tuple[List[List[PoseLandmarksResult]], Optional[List[List[torch.Tensor]]]]: # Added running_mode
|
||||
"""Detects pose landmarks in the input image tensor.
|
||||
|
||||
Returns:
|
||||
@@ -63,38 +80,51 @@ class PoseLandmarkDetector:
|
||||
raise ValueError("Input tensor must be in BHWC format.")
|
||||
|
||||
# Add running_mode to tuple check
|
||||
new_options_tuple = (num_poses, min_detection_confidence, min_presence_confidence,
|
||||
min_tracking_confidence, output_segmentation_masks, running_mode, delegate.lower())
|
||||
new_options_tuple = (
|
||||
num_poses,
|
||||
min_detection_confidence,
|
||||
min_presence_confidence,
|
||||
min_tracking_confidence,
|
||||
output_segmentation_masks,
|
||||
running_mode,
|
||||
delegate.lower(),
|
||||
)
|
||||
current_options_tuple = None
|
||||
if self._current_options:
|
||||
current_options_tuple = (
|
||||
self._current_options.num_poses,
|
||||
self._current_options.min_pose_detection_confidence,
|
||||
self._current_options.min_pose_presence_confidence,
|
||||
self._current_options.min_tracking_confidence,
|
||||
self._current_options.output_segmentation_masks,
|
||||
"image" if self._current_options.running_mode == vision.RunningMode.IMAGE else "video", # Map enum back to string
|
||||
'cpu' if self._current_options.base_options.delegate == BaseOptions.Delegate.CPU else 'gpu'
|
||||
)
|
||||
current_options_tuple = (
|
||||
self._current_options.num_poses,
|
||||
self._current_options.min_pose_detection_confidence,
|
||||
self._current_options.min_pose_presence_confidence,
|
||||
self._current_options.min_tracking_confidence,
|
||||
self._current_options.output_segmentation_masks,
|
||||
"image" if self._current_options.running_mode == vision.RunningMode.IMAGE else "video", # Map enum back to string
|
||||
"cpu" if self._current_options.base_options.delegate == BaseOptions.Delegate.CPU else "gpu",
|
||||
)
|
||||
|
||||
if self._detector_instance is None or new_options_tuple != current_options_tuple:
|
||||
if self._detector_instance and hasattr(self._detector_instance, 'close'):
|
||||
self._detector_instance.close()
|
||||
if self._detector_instance and hasattr(self._detector_instance, "close"):
|
||||
self._detector_instance.close()
|
||||
# Pass new params to initializer
|
||||
self._detector_instance = self._initialize_detector(num_poses, min_detection_confidence,
|
||||
min_presence_confidence, min_tracking_confidence,
|
||||
output_segmentation_masks, running_mode, delegate) # Pass running_mode
|
||||
self._detector_instance = self._initialize_detector(
|
||||
num_poses,
|
||||
min_detection_confidence,
|
||||
min_presence_confidence,
|
||||
min_tracking_confidence,
|
||||
output_segmentation_masks,
|
||||
running_mode,
|
||||
delegate,
|
||||
) # Pass running_mode
|
||||
|
||||
batch_size = image.shape[0]
|
||||
batch_results_landmarks = []
|
||||
batch_results_masks = [] if output_segmentation_masks else None
|
||||
|
||||
|
||||
# Determine the correct detection function based on mode
|
||||
is_video_mode = self._current_options.running_mode == vision.RunningMode.VIDEO
|
||||
detect_func = self._detector_instance.detect_for_video if is_video_mode else self._detector_instance.detect
|
||||
|
||||
|
||||
for i in range(batch_size):
|
||||
img_tensor = image[i] # HWC
|
||||
img_tensor = image[i] # HWC
|
||||
np_image = (img_tensor.cpu().numpy() * 255).astype(np.uint8)
|
||||
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=np_image)
|
||||
|
||||
@@ -107,42 +137,49 @@ class PoseLandmarkDetector:
|
||||
current_image_landmarks = []
|
||||
current_image_masks = [] if output_segmentation_masks else None
|
||||
|
||||
if detection_result and detection_result.pose_landmarks: # Added check for detection_result
|
||||
if detection_result and detection_result.pose_landmarks: # Added check for detection_result
|
||||
for pose_idx, pose_landmarks_mp in enumerate(detection_result.pose_landmarks):
|
||||
landmarks = [
|
||||
LandmarkPoint(index=lm_idx, x=lm.x, y=lm.y, z=lm.z,
|
||||
visibility=getattr(lm, 'visibility', None), # Use getattr for safety
|
||||
presence=getattr(lm, 'presence', None))
|
||||
LandmarkPoint(
|
||||
index=lm_idx,
|
||||
x=lm.x,
|
||||
y=lm.y,
|
||||
z=lm.z,
|
||||
visibility=getattr(lm, "visibility", None), # Use getattr for safety
|
||||
presence=getattr(lm, "presence", None),
|
||||
)
|
||||
for lm_idx, lm in enumerate(pose_landmarks_mp)
|
||||
]
|
||||
|
||||
|
||||
world_landmarks = None
|
||||
if detection_result.pose_world_landmarks and pose_idx < len(detection_result.pose_world_landmarks):
|
||||
world_landmarks = [
|
||||
LandmarkPoint(index=lm_idx, x=lm.x, y=lm.y, z=lm.z,
|
||||
visibility=getattr(lm, 'visibility', None),
|
||||
presence=getattr(lm, 'presence', None))
|
||||
for lm_idx, lm in enumerate(detection_result.pose_world_landmarks[pose_idx])
|
||||
]
|
||||
|
||||
current_image_landmarks.append(PoseLandmarksResult(
|
||||
landmarks=landmarks,
|
||||
world_landmarks=world_landmarks
|
||||
))
|
||||
world_landmarks = [
|
||||
LandmarkPoint(
|
||||
index=lm_idx,
|
||||
x=lm.x,
|
||||
y=lm.y,
|
||||
z=lm.z,
|
||||
visibility=getattr(lm, "visibility", None),
|
||||
presence=getattr(lm, "presence", None),
|
||||
)
|
||||
for lm_idx, lm in enumerate(detection_result.pose_world_landmarks[pose_idx])
|
||||
]
|
||||
|
||||
current_image_landmarks.append(PoseLandmarksResult(landmarks=landmarks, world_landmarks=world_landmarks))
|
||||
|
||||
# Handle segmentation masks separately as they are not indexed per pose in the result
|
||||
if output_segmentation_masks and detection_result.segmentation_masks:
|
||||
# MediaPipe returns one mask per image if output_segmentation_masks is True
|
||||
# We store it as a list associated with the image, not per pose
|
||||
# Assuming only one mask is relevant per image, might need adjustment if multiple masks are returned
|
||||
mask_mp = detection_result.segmentation_masks[0] # Get the first mask
|
||||
mask_np = mask_mp.numpy_view()
|
||||
mask_tensor = torch.from_numpy(mask_np).float() # HW
|
||||
current_image_masks = [mask_tensor] # Store as a list for consistency with return type
|
||||
|
||||
# MediaPipe returns one mask per image if output_segmentation_masks is True
|
||||
# We store it as a list associated with the image, not per pose
|
||||
# Assuming only one mask is relevant per image, might need adjustment if multiple masks are returned
|
||||
mask_mp = detection_result.segmentation_masks[0] # Get the first mask
|
||||
mask_np = mask_mp.numpy_view()
|
||||
mask_tensor = torch.from_numpy(mask_np).float() # HW
|
||||
current_image_masks = [mask_tensor] # Store as a list for consistency with return type
|
||||
|
||||
batch_results_landmarks.append(current_image_landmarks)
|
||||
if output_segmentation_masks:
|
||||
batch_results_masks.append(current_image_masks)
|
||||
|
||||
# self._detector_instance.close() # Close handled on re-init
|
||||
return batch_results_landmarks, batch_results_masks
|
||||
return batch_results_landmarks, batch_results_masks
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
from typing import List, Optional, NamedTuple, Tuple
|
||||
import numpy as np # Added numpy for matrix type
|
||||
from typing import List, NamedTuple, Optional
|
||||
|
||||
import numpy as np # Added numpy for matrix type
|
||||
|
||||
|
||||
# Define a structure for a single landmark point
|
||||
class LandmarkPoint(NamedTuple):
|
||||
@@ -7,93 +9,109 @@ class LandmarkPoint(NamedTuple):
|
||||
x: float
|
||||
y: float
|
||||
z: float
|
||||
visibility: Optional[float] = None # Available in Pose landmarks
|
||||
presence: Optional[float] = None # Available in Pose landmarks
|
||||
visibility: Optional[float] = None # Available in Pose landmarks
|
||||
presence: Optional[float] = None # Available in Pose landmarks
|
||||
|
||||
|
||||
# Define container structures for the output of each landmark task
|
||||
# These will be returned by the respective detector nodes
|
||||
|
||||
|
||||
class FaceLandmarksResult(NamedTuple):
|
||||
landmarks: List[LandmarkPoint]
|
||||
# Add other face-specific results if needed later (e.g., bounding box)
|
||||
|
||||
|
||||
class HandLandmarksResult(NamedTuple):
|
||||
landmarks: List[LandmarkPoint]
|
||||
world_landmarks: Optional[List[LandmarkPoint]] # Keep world landmarks optional for now
|
||||
handedness: Optional[str] # 'Left', 'Right'
|
||||
world_landmarks: Optional[List[LandmarkPoint]] # Keep world landmarks optional for now
|
||||
handedness: Optional[str] # 'Left', 'Right'
|
||||
|
||||
|
||||
class PoseLandmarksResult(NamedTuple):
|
||||
landmarks: List[LandmarkPoint]
|
||||
world_landmarks: Optional[List[LandmarkPoint]]
|
||||
# Segmentation mask info could potentially go here if tightly coupled
|
||||
|
||||
|
||||
# We might add more types later for detections, segmentations etc.
|
||||
# For now, focus is on landmarks.
|
||||
|
||||
# Define the list types that will be used for batching
|
||||
# These will be the types used in node RETURN_TYPES/INPUT_TYPES strings
|
||||
FACE_LANDMARKS_LIST = List[List[FaceLandmarksResult]] # Batch[FacesPerImage[Result]]
|
||||
HAND_LANDMARKS_LIST = List[List[HandLandmarksResult]] # Batch[HandsPerImage[Result]]
|
||||
POSE_LANDMARKS_LIST = List[List[PoseLandmarksResult]] # Batch[PosesPerImage[Result]]
|
||||
FACE_LANDMARKS_LIST = List[List[FaceLandmarksResult]] # Batch[FacesPerImage[Result]]
|
||||
HAND_LANDMARKS_LIST = List[List[HandLandmarksResult]] # Batch[HandsPerImage[Result]]
|
||||
POSE_LANDMARKS_LIST = List[List[PoseLandmarksResult]] # Batch[PosesPerImage[Result]]
|
||||
|
||||
# --- Blendshape Result ---
|
||||
|
||||
# --- Blendshape Result ---
|
||||
class Blendshape(NamedTuple):
|
||||
index: Optional[int]
|
||||
score: float
|
||||
category_name: str # e.g., mouthSmileLeft, eyeBlinkRight
|
||||
category_name: str # e.g., mouthSmileLeft, eyeBlinkRight
|
||||
display_name: Optional[str]
|
||||
|
||||
# --- Common Types ---
|
||||
|
||||
# --- Common Types ---
|
||||
class BoundingBox(NamedTuple):
|
||||
origin_x: int
|
||||
origin_y: int
|
||||
width: int
|
||||
height: int
|
||||
|
||||
# --- Face Detection Result ---
|
||||
|
||||
# --- Face Detection Result ---
|
||||
class FaceKeypoint(NamedTuple):
|
||||
label: Optional[str] # e.g., 'nose_tip', 'left_eye'
|
||||
label: Optional[str] # e.g., 'nose_tip', 'left_eye'
|
||||
x: float
|
||||
y: float
|
||||
|
||||
|
||||
class FaceDetectionResult(NamedTuple):
|
||||
bounding_box: BoundingBox
|
||||
keypoints: Optional[List[FaceKeypoint]]
|
||||
score: Optional[float] # Confidence score for the detection
|
||||
score: Optional[float] # Confidence score for the detection
|
||||
|
||||
# --- Object Detection Result ---
|
||||
|
||||
# --- Object Detection Result ---
|
||||
class ObjectDetectionCategory(NamedTuple):
|
||||
index: Optional[int]
|
||||
score: float
|
||||
display_name: Optional[str]
|
||||
category_name: Optional[str]
|
||||
|
||||
|
||||
class ObjectDetectionResult(NamedTuple):
|
||||
bounding_box: BoundingBox
|
||||
categories: List[ObjectDetectionCategory]
|
||||
|
||||
# --- Batch List Types for Nodes ---
|
||||
|
||||
# --- Batch List Types for Nodes ---
|
||||
# Define the list types that will be used for batching
|
||||
# These will be the types used in node RETURN_TYPES/INPUT_TYPES strings
|
||||
FACE_LANDMARKS = List[List[FaceLandmarksResult]] # Batch[FacesPerImage[Result]]
|
||||
HAND_LANDMARKS = List[List[HandLandmarksResult]] # Batch[HandsPerImage[Result]]
|
||||
POSE_LANDMARKS = List[List[PoseLandmarksResult]] # Batch[PosesPerImage[Result]]
|
||||
OBJECT_DETECTIONS = List[List[ObjectDetectionResult]] # Batch[DetectionsPerImage[Result]]
|
||||
FACE_DETECTIONS = List[List[FaceDetectionResult]] # Batch[DetectionsPerImage[Result]]
|
||||
FACE_LANDMARKS = List[List[FaceLandmarksResult]] # Batch[FacesPerImage[Result]]
|
||||
HAND_LANDMARKS = List[List[HandLandmarksResult]] # Batch[HandsPerImage[Result]]
|
||||
POSE_LANDMARKS = List[List[PoseLandmarksResult]] # Batch[PosesPerImage[Result]]
|
||||
OBJECT_DETECTIONS = List[List[ObjectDetectionResult]] # Batch[DetectionsPerImage[Result]]
|
||||
FACE_DETECTIONS = List[List[FaceDetectionResult]] # Batch[DetectionsPerImage[Result]]
|
||||
|
||||
# --- Gesture Recognition Result ---
|
||||
|
||||
# --- Gesture Recognition Result ---
|
||||
class GestureCategory(NamedTuple):
|
||||
index: Optional[int]
|
||||
score: float
|
||||
display_name: Optional[str]
|
||||
category_name: Optional[str]
|
||||
|
||||
|
||||
class GestureRecognitionResult(NamedTuple):
|
||||
gestures: List[GestureCategory] # Top gesture(s) recognized for a hand
|
||||
handedness: Optional[str] # Which hand this gesture belongs to
|
||||
gestures: List[GestureCategory] # Top gesture(s) recognized for a hand
|
||||
handedness: Optional[str] # Which hand this gesture belongs to
|
||||
|
||||
|
||||
# Define Batch List Type
|
||||
GESTURE_RECOGNITIONS = List[List[GestureRecognitionResult]] # Batch[Hands[Gestures]]
|
||||
GESTURE_RECOGNITIONS = List[List[GestureRecognitionResult]] # Batch[Hands[Gestures]]
|
||||
|
||||
|
||||
# --- Image Embedding Result ---
|
||||
class EmbeddingEntry(NamedTuple):
|
||||
@@ -102,49 +120,55 @@ class EmbeddingEntry(NamedTuple):
|
||||
display_name: Optional[str]
|
||||
category_name: Optional[str]
|
||||
|
||||
|
||||
class ImageEmbedderResult(NamedTuple):
|
||||
float_embedding: Optional[List[float]]
|
||||
quantized_embedding: Optional[bytes]
|
||||
head_index: int
|
||||
head_name: Optional[str]
|
||||
|
||||
|
||||
# Define Batch List Type
|
||||
IMAGE_EMBEDDINGS = List[List[ImageEmbedderResult]] # Batch[EmbeddingsPerHead[Result]]
|
||||
IMAGE_EMBEDDINGS = List[List[ImageEmbedderResult]] # Batch[EmbeddingsPerHead[Result]]
|
||||
|
||||
|
||||
# --- Interactive Segmentation Input/Result ---
|
||||
class PointOfInterest(NamedTuple):
|
||||
x: float
|
||||
y: float
|
||||
label: Optional[str] = None # Optional label for the point
|
||||
label: Optional[str] = None # Optional label for the point
|
||||
|
||||
|
||||
# Result is typically a mask, potentially multiple if categories are involved
|
||||
# Using existing MASK type might suffice, or define a specific one if needed.
|
||||
|
||||
# --- Holistic Landmark Result ---
|
||||
|
||||
# --- Holistic Landmark Result ---
|
||||
class HolisticLandmarksResult(NamedTuple):
|
||||
face_landmarks: Optional[List[LandmarkPoint]] # Only one face per person
|
||||
face_landmarks: Optional[List[LandmarkPoint]] # Only one face per person
|
||||
pose_landmarks: Optional[List[LandmarkPoint]]
|
||||
pose_world_landmarks: Optional[List[LandmarkPoint]]
|
||||
left_hand_landmarks: Optional[List[LandmarkPoint]]
|
||||
right_hand_landmarks: Optional[List[LandmarkPoint]]
|
||||
# Pose segmentation mask could also be included here
|
||||
|
||||
|
||||
# Define Batch List Type
|
||||
HOLISTIC_LANDMARKS = List[HolisticLandmarksResult] # Batch[ResultPerPerson]
|
||||
HOLISTIC_LANDMARKS = List[HolisticLandmarksResult] # Batch[ResultPerPerson]
|
||||
|
||||
# =======================================
|
||||
# --- Batch List Types for Node I/O ---
|
||||
# --- Batch List Types for Node I/O ---
|
||||
# =======================================
|
||||
FACE_LANDMARKS = List[List[FaceLandmarksResult]] # Batch[FacesPerImage[Result]]
|
||||
HAND_LANDMARKS = List[List[HandLandmarksResult]] # Batch[HandsPerImage[Result]]
|
||||
POSE_LANDMARKS = List[List[PoseLandmarksResult]] # Batch[PosesPerImage[Result]]
|
||||
OBJECT_DETECTIONS = List[List[ObjectDetectionResult]] # Batch[DetectionsPerImage[Result]]
|
||||
FACE_DETECTIONS = List[List[FaceDetectionResult]] # Batch[DetectionsPerImage[Result]]
|
||||
GESTURE_RECOGNITIONS = List[List[GestureRecognitionResult]] # Batch[Hands[Gestures]]
|
||||
IMAGE_EMBEDDINGS = List[List[ImageEmbedderResult]] # Batch[EmbeddingsPerHead[Result]]
|
||||
HOLISTIC_LANDMARKS = List[HolisticLandmarksResult] # Batch[ResultPerPerson]
|
||||
FACE_LANDMARKS = List[List[FaceLandmarksResult]] # Batch[FacesPerImage[Result]]
|
||||
HAND_LANDMARKS = List[List[HandLandmarksResult]] # Batch[HandsPerImage[Result]]
|
||||
POSE_LANDMARKS = List[List[PoseLandmarksResult]] # Batch[PosesPerImage[Result]]
|
||||
OBJECT_DETECTIONS = List[List[ObjectDetectionResult]] # Batch[DetectionsPerImage[Result]]
|
||||
FACE_DETECTIONS = List[List[FaceDetectionResult]] # Batch[DetectionsPerImage[Result]]
|
||||
GESTURE_RECOGNITIONS = List[List[GestureRecognitionResult]] # Batch[Hands[Gestures]]
|
||||
IMAGE_EMBEDDINGS = List[List[ImageEmbedderResult]] # Batch[EmbeddingsPerHead[Result]]
|
||||
HOLISTIC_LANDMARKS = List[HolisticLandmarksResult] # Batch[ResultPerPerson]
|
||||
|
||||
# New Types - Defined AFTER Blendshape class
|
||||
# Updated names to match FaceLandmarker output convention
|
||||
BLENDSHAPES_LIST = List[List[List[Blendshape]]] # Batch[FacesPerImage[BlendshapeList[Blendshape]]]
|
||||
TRANSFORM_MATRIX_LIST = List[List[np.ndarray]] # Batch[FacesPerImage[Matrix]]
|
||||
BLENDSHAPES_LIST = List[List[List[Blendshape]]] # Batch[FacesPerImage[BlendshapeList[Blendshape]]]
|
||||
TRANSFORM_MATRIX_LIST = List[List[np.ndarray]] # Batch[FacesPerImage[Matrix]]
|
||||
|
||||
@@ -1,19 +1,13 @@
|
||||
"""Utility functions for MediaPipe Vision."""
|
||||
|
||||
# DEPRECATED: These imports are maintained only for backward compatibility.
|
||||
# For new code, please import directly from src.utils
|
||||
# For new code, please import directly from src.utils
|
||||
|
||||
from ...utils.math import scale_value
|
||||
from ...utils.image import convert_to_cv2, convert_to_tensor
|
||||
from ...utils.math import scale_value
|
||||
from .delta_utils import calculate_euclidean_delta
|
||||
|
||||
# Keep only the functions specific to mediapipe_vision
|
||||
from .download_utils import download_model
|
||||
from .delta_utils import calculate_euclidean_delta
|
||||
|
||||
__all__ = [
|
||||
"convert_to_cv2",
|
||||
"convert_to_tensor",
|
||||
"calculate_euclidean_delta",
|
||||
"scale_value",
|
||||
"download_model"
|
||||
]
|
||||
__all__ = ["convert_to_cv2", "convert_to_tensor", "calculate_euclidean_delta", "scale_value", "download_model"]
|
||||
|
||||
@@ -1,29 +1,67 @@
|
||||
# src/utils/misc_utils.py
|
||||
import mediapipe as mp
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_blendshape_categories() -> List[str]:
|
||||
"""Returns the list of standard blendshape category names from MediaPipe."""
|
||||
# Define blendshape categories directly
|
||||
# Define blendshape categories directly
|
||||
# This is based on the MediaPipe standard blendshapes
|
||||
standard_blendshapes = [
|
||||
"browDownLeft", "browDownRight", "browInnerUp", "browOuterUpLeft",
|
||||
"browOuterUpRight", "cheekPuff", "cheekSquintLeft", "cheekSquintRight",
|
||||
"eyeBlinkLeft", "eyeBlinkRight", "eyeLookDownLeft", "eyeLookDownRight",
|
||||
"eyeLookInLeft", "eyeLookInRight", "eyeLookOutLeft", "eyeLookOutRight",
|
||||
"eyeLookUpLeft", "eyeLookUpRight", "eyeSquintLeft", "eyeSquintRight",
|
||||
"eyeWideLeft", "eyeWideRight", "jawForward", "jawLeft", "jawOpen",
|
||||
"jawRight", "mouthClose", "mouthDimpleLeft", "mouthDimpleRight",
|
||||
"mouthFrownLeft", "mouthFrownRight", "mouthFunnel", "mouthLeft",
|
||||
"mouthLowerDownLeft", "mouthLowerDownRight", "mouthPressLeft",
|
||||
"mouthPressRight", "mouthPucker", "mouthRight", "mouthRollLower",
|
||||
"mouthRollUpper", "mouthShrugLower", "mouthShrugUpper", "mouthSmileLeft",
|
||||
"mouthSmileRight", "mouthStretchLeft", "mouthStretchRight", "mouthUpperUpLeft",
|
||||
"mouthUpperUpRight", "noseSneerLeft", "noseSneerRight"
|
||||
"browDownLeft",
|
||||
"browDownRight",
|
||||
"browInnerUp",
|
||||
"browOuterUpLeft",
|
||||
"browOuterUpRight",
|
||||
"cheekPuff",
|
||||
"cheekSquintLeft",
|
||||
"cheekSquintRight",
|
||||
"eyeBlinkLeft",
|
||||
"eyeBlinkRight",
|
||||
"eyeLookDownLeft",
|
||||
"eyeLookDownRight",
|
||||
"eyeLookInLeft",
|
||||
"eyeLookInRight",
|
||||
"eyeLookOutLeft",
|
||||
"eyeLookOutRight",
|
||||
"eyeLookUpLeft",
|
||||
"eyeLookUpRight",
|
||||
"eyeSquintLeft",
|
||||
"eyeSquintRight",
|
||||
"eyeWideLeft",
|
||||
"eyeWideRight",
|
||||
"jawForward",
|
||||
"jawLeft",
|
||||
"jawOpen",
|
||||
"jawRight",
|
||||
"mouthClose",
|
||||
"mouthDimpleLeft",
|
||||
"mouthDimpleRight",
|
||||
"mouthFrownLeft",
|
||||
"mouthFrownRight",
|
||||
"mouthFunnel",
|
||||
"mouthLeft",
|
||||
"mouthLowerDownLeft",
|
||||
"mouthLowerDownRight",
|
||||
"mouthPressLeft",
|
||||
"mouthPressRight",
|
||||
"mouthPucker",
|
||||
"mouthRight",
|
||||
"mouthRollLower",
|
||||
"mouthRollUpper",
|
||||
"mouthShrugLower",
|
||||
"mouthShrugUpper",
|
||||
"mouthSmileLeft",
|
||||
"mouthSmileRight",
|
||||
"mouthStretchLeft",
|
||||
"mouthStretchRight",
|
||||
"mouthUpperUpLeft",
|
||||
"mouthUpperUpRight",
|
||||
"noseSneerLeft",
|
||||
"noseSneerRight",
|
||||
]
|
||||
|
||||
|
||||
logger.info("Using predefined list of standard blendshape categories.")
|
||||
return standard_blendshapes
|
||||
return standard_blendshapes
|
||||
|
||||
@@ -8,19 +8,21 @@
|
||||
import math
|
||||
from typing import Optional
|
||||
|
||||
from ...utils.math import scale_value
|
||||
from ..types import LandmarkPoint # Assuming LandmarkPoint is in src/types.py
|
||||
#TODO: refactor to common after unified coordinate system
|
||||
from ..types import LandmarkPoint # Assuming LandmarkPoint is in src/types.py
|
||||
|
||||
|
||||
# TODO: refactor to common after unified coordinate system
|
||||
def calculate_euclidean_delta(p1: Optional[LandmarkPoint], p2: Optional[LandmarkPoint]) -> Optional[float]:
|
||||
"""Calculates the Euclidean distance between two LandmarkPoint objects."""
|
||||
if p1 is None or p2 is None:
|
||||
return None # Cannot calculate delta if one point is missing
|
||||
|
||||
return None # Cannot calculate delta if one point is missing
|
||||
|
||||
delta_x = p1.x - p2.x
|
||||
delta_y = p1.y - p2.y
|
||||
delta_z = p1.z - p2.z
|
||||
|
||||
|
||||
distance = math.sqrt(delta_x**2 + delta_y**2 + delta_z**2)
|
||||
return distance
|
||||
|
||||
# scale_value is now imported from utils.math
|
||||
|
||||
# scale_value is now imported from utils.math
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
"""Utilities for downloading MediaPipe models."""
|
||||
|
||||
import os
|
||||
import requests
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
import requests
|
||||
from tqdm import tqdm
|
||||
from typing import Dict, Optional, Any, Tuple
|
||||
|
||||
from .model_utils import MEDIAPIPE_MODELS, get_model_path
|
||||
|
||||
@@ -15,11 +16,11 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
def download_model(task_type: str, model_variant: str) -> Optional[str]:
|
||||
"""Download a MediaPipe model for a specific task and variant.
|
||||
|
||||
|
||||
Args:
|
||||
task_type: The task category (e.g., 'face_mesh', 'pose_detection')
|
||||
model_variant: The specific model variant (e.g., 'standard', 'lite')
|
||||
|
||||
|
||||
Returns:
|
||||
Path to the downloaded model file, or None if download failed
|
||||
"""
|
||||
@@ -27,85 +28,85 @@ def download_model(task_type: str, model_variant: str) -> Optional[str]:
|
||||
if task_type not in MEDIAPIPE_MODELS:
|
||||
logger.error(f"Task '{task_type}' not found in available models.")
|
||||
return None
|
||||
|
||||
|
||||
if model_variant not in MEDIAPIPE_MODELS[task_type]:
|
||||
logger.error(f"Model variant '{model_variant}' not found for task '{task_type}'.")
|
||||
return None
|
||||
|
||||
|
||||
# Get model details
|
||||
model_info = MEDIAPIPE_MODELS[task_type][model_variant]
|
||||
url = model_info["url"]
|
||||
filename = model_info["filename"]
|
||||
|
||||
|
||||
# Get the full path for the model
|
||||
output_path = get_model_path(task_type, filename)
|
||||
|
||||
|
||||
# If model already exists, return the path
|
||||
if os.path.exists(output_path):
|
||||
logger.info(f"Using existing model: {output_path}")
|
||||
return output_path
|
||||
|
||||
|
||||
# Download the model
|
||||
logger.info(f"Downloading {task_type}/{model_variant} model from {url}...")
|
||||
try:
|
||||
# Ensure parent directory exists
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
|
||||
|
||||
# Stream the download with progress bar
|
||||
response = requests.get(url, stream=True)
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
# Get content length for progress bar if available
|
||||
total_size = int(response.headers.get('content-length', 0))
|
||||
total_size = int(response.headers.get("content-length", 0))
|
||||
block_size = 8192 # 8 KB
|
||||
|
||||
with open(output_path, 'wb') as f:
|
||||
with tqdm(total=total_size, unit='B', unit_scale=True, desc=f"{task_type}/{model_variant}") as pbar:
|
||||
|
||||
with open(output_path, "wb") as f:
|
||||
with tqdm(total=total_size, unit="B", unit_scale=True, desc=f"{task_type}/{model_variant}") as pbar:
|
||||
for chunk in response.iter_content(chunk_size=block_size):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
pbar.update(len(chunk))
|
||||
|
||||
|
||||
logger.info(f"Successfully downloaded {task_type}/{model_variant} to {output_path}")
|
||||
return output_path
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error downloading {task_type}/{model_variant}: {str(e)}")
|
||||
|
||||
|
||||
# Clean up partial download if it exists
|
||||
if os.path.exists(output_path):
|
||||
try:
|
||||
os.remove(output_path)
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_model_options(task_type: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""Get available model options for a specific task or all tasks.
|
||||
|
||||
|
||||
Args:
|
||||
task_type: The task category to get options for, or None for all tasks
|
||||
|
||||
|
||||
Returns:
|
||||
Dictionary of available models and variants
|
||||
"""
|
||||
if task_type is None:
|
||||
# Return all tasks and their variants
|
||||
return {task: list(variants.keys()) for task, variants in MEDIAPIPE_MODELS.items()}
|
||||
|
||||
|
||||
# Return variants for specific task
|
||||
if task_type in MEDIAPIPE_MODELS:
|
||||
return {task_type: list(MEDIAPIPE_MODELS[task_type].keys())}
|
||||
|
||||
|
||||
# Task not found
|
||||
return {}
|
||||
|
||||
|
||||
def get_all_task_types() -> list:
|
||||
"""Get a list of all available task types.
|
||||
|
||||
|
||||
Returns:
|
||||
List of task type strings
|
||||
"""
|
||||
@@ -114,10 +115,10 @@ def get_all_task_types() -> list:
|
||||
|
||||
def get_model_variants(task_type: str) -> list:
|
||||
"""Get a list of all available model variants for a specific task.
|
||||
|
||||
|
||||
Args:
|
||||
task_type: The task category
|
||||
|
||||
|
||||
Returns:
|
||||
List of model variant strings, or empty list if task not found
|
||||
"""
|
||||
@@ -128,11 +129,11 @@ def get_model_variants(task_type: str) -> list:
|
||||
|
||||
def get_model_info(task_type: str, model_variant: str) -> Tuple[Optional[str], Optional[Dict[str, Any]]]:
|
||||
"""Get information about a specific model.
|
||||
|
||||
|
||||
Args:
|
||||
task_type: The task category
|
||||
model_variant: The specific model variant
|
||||
|
||||
|
||||
Returns:
|
||||
Tuple containing:
|
||||
- Path to the model file (downloaded if needed)
|
||||
@@ -141,16 +142,16 @@ def get_model_info(task_type: str, model_variant: str) -> Tuple[Optional[str], O
|
||||
# Check if the task and variant exist
|
||||
if task_type not in MEDIAPIPE_MODELS or model_variant not in MEDIAPIPE_MODELS[task_type]:
|
||||
return None, None
|
||||
|
||||
|
||||
# Get model details
|
||||
model_info = MEDIAPIPE_MODELS[task_type][model_variant].copy()
|
||||
|
||||
|
||||
# Download the model if needed and get the path
|
||||
model_path = download_model(task_type, model_variant)
|
||||
if not model_path:
|
||||
return None, None
|
||||
|
||||
|
||||
# Add the model path to the info
|
||||
model_info["model_path"] = model_path
|
||||
|
||||
return model_path, model_info
|
||||
|
||||
return model_path, model_info
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
"""Utilities for managing MediaPipe Vision models."""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from typing import Dict, Any, Optional
|
||||
import os
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
@@ -11,6 +10,7 @@ logger = logging.getLogger(__name__)
|
||||
# Import ComfyUI's folder_paths to use the models directory
|
||||
try:
|
||||
import folder_paths
|
||||
|
||||
HAS_COMFY_PATHS = True
|
||||
except ImportError:
|
||||
logger.warning("ComfyUI folder_paths module not available. Using local paths.")
|
||||
@@ -19,9 +19,9 @@ except ImportError:
|
||||
|
||||
def get_mediapipe_models_directory() -> str:
|
||||
"""Get the directory where MediaPipe models are stored.
|
||||
|
||||
|
||||
Uses ComfyUI's models directory structure when available.
|
||||
|
||||
|
||||
Returns:
|
||||
Path to the MediaPipe models directory
|
||||
"""
|
||||
@@ -32,7 +32,7 @@ def get_mediapipe_models_directory() -> str:
|
||||
# Fallback to local path
|
||||
base_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
mediapipe_dir = os.path.join(base_path, "models")
|
||||
|
||||
|
||||
# Create the directory if it doesn't exist
|
||||
os.makedirs(mediapipe_dir, exist_ok=True)
|
||||
return mediapipe_dir
|
||||
@@ -40,18 +40,18 @@ def get_mediapipe_models_directory() -> str:
|
||||
|
||||
def get_model_path(model_type: str, model_name: str) -> str:
|
||||
"""Get the path for a specific MediaPipe Vision model.
|
||||
|
||||
|
||||
Args:
|
||||
model_type: Type/category of model (e.g., 'face_mesh', 'pose')
|
||||
model_name: Name of the specific model file
|
||||
|
||||
|
||||
Returns:
|
||||
Full path to the model file
|
||||
"""
|
||||
models_dir = get_mediapipe_models_directory()
|
||||
model_type_dir = os.path.join(models_dir, model_type)
|
||||
os.makedirs(model_type_dir, exist_ok=True)
|
||||
|
||||
|
||||
return os.path.join(model_type_dir, model_name)
|
||||
|
||||
|
||||
@@ -61,88 +61,88 @@ MEDIAPIPE_MODELS = {
|
||||
"face_detection": {
|
||||
"short_range": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/face_detector/blaze_face_short_range/float16/1/blaze_face_short_range.task",
|
||||
"filename": "blaze_face_short_range.task"
|
||||
"filename": "blaze_face_short_range.task",
|
||||
},
|
||||
"full_range": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/face_detector/blaze_face/float16/1/blaze_face.task",
|
||||
"filename": "blaze_face.task"
|
||||
}
|
||||
"filename": "blaze_face.task",
|
||||
},
|
||||
},
|
||||
"face_mesh": {
|
||||
"standard": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task",
|
||||
"filename": "face_landmarker.task",
|
||||
"description": "Detects face landmarks and facial features (468 points)."
|
||||
"description": "Detects face landmarks and facial features (468 points).",
|
||||
},
|
||||
"attention": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/2/face_landmarker.task",
|
||||
"filename": "face_landmarker_v2.task",
|
||||
"description": "Improved version with better accuracy for face landmarks."
|
||||
}
|
||||
"description": "Improved version with better accuracy for face landmarks.",
|
||||
},
|
||||
},
|
||||
"hand_tracking": {
|
||||
"standard": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task",
|
||||
"filename": "hand_landmarker.task",
|
||||
"description": "Detects hand landmarks (21 points per hand)."
|
||||
"description": "Detects hand landmarks (21 points per hand).",
|
||||
}
|
||||
},
|
||||
"pose_detection": {
|
||||
"lite": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task",
|
||||
"filename": "pose_landmarker_lite.task",
|
||||
"description": "Lightweight pose detection model (33 points)."
|
||||
"description": "Lightweight pose detection model (33 points).",
|
||||
},
|
||||
"full": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_full/float16/1/pose_landmarker_full.task",
|
||||
"filename": "pose_landmarker_full.task",
|
||||
"description": "Full pose detection model with high accuracy."
|
||||
"description": "Full pose detection model with high accuracy.",
|
||||
},
|
||||
"heavy": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_heavy/float16/1/pose_landmarker_heavy.task",
|
||||
"filename": "pose_landmarker_heavy.task",
|
||||
"description": "Most accurate pose detection model."
|
||||
}
|
||||
"description": "Most accurate pose detection model.",
|
||||
},
|
||||
},
|
||||
"image_segmentation": {
|
||||
"selfie": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/1/selfie_segmenter.task",
|
||||
"filename": "selfie_segmenter.task",
|
||||
"description": "Segments people from background."
|
||||
"description": "Segments people from background.",
|
||||
},
|
||||
"selfie_landscape": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/1/selfie_segmenter_landscape.task",
|
||||
"filename": "selfie_segmenter_landscape.task",
|
||||
"description": "Optimized for landscape orientation."
|
||||
"description": "Optimized for landscape orientation.",
|
||||
},
|
||||
"multiclass": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_multiclass_256x256/float32/1/selfie_multiclass_256x256.task",
|
||||
"filename": "selfie_multiclass_256x256.task",
|
||||
"description": "Segments multiple body parts (face, hair, clothes, etc.)."
|
||||
}
|
||||
"description": "Segments multiple body parts (face, hair, clothes, etc.).",
|
||||
},
|
||||
},
|
||||
"object_detection": {
|
||||
"efficientdet_lite0": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/object_detector/efficientdet_lite0/float16/1/efficientdet_lite0.task",
|
||||
"filename": "efficientdet_lite0.task",
|
||||
"description": "General object detection (80 classes)."
|
||||
"description": "General object detection (80 classes).",
|
||||
},
|
||||
"efficientdet_lite2": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/object_detector/efficientdet_lite2/float16/1/efficientdet_lite2.task",
|
||||
"filename": "efficientdet_lite2.task",
|
||||
"description": "Higher accuracy object detection."
|
||||
}
|
||||
"description": "Higher accuracy object detection.",
|
||||
},
|
||||
},
|
||||
"image_classification": {
|
||||
"efficientnet_lite0": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/image_classifier/efficientnet_lite0/float32/1/efficientnet_lite0.task",
|
||||
"filename": "efficientnet_lite0.task",
|
||||
"description": "General image classification (1000 classes)."
|
||||
"description": "General image classification (1000 classes).",
|
||||
},
|
||||
"efficientnet_lite4": {
|
||||
"url": "https://storage.googleapis.com/mediapipe-models/image_classifier/efficientnet_lite4/float32/2/efficientnet_lite4.task",
|
||||
"filename": "efficientnet_lite4.task",
|
||||
"description": "Higher accuracy image classification."
|
||||
}
|
||||
}
|
||||
}
|
||||
"description": "Higher accuracy image classification.",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1 +1 @@
|
||||
# This file intentionally left blank
|
||||
# This file intentionally left blank
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user