diff --git a/README.md b/README.md index 34dba58..9e21e95 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,7 @@ ComfyUI-SAM3/ - **SAM3**: Meta AI Research (https://github.com/facebookresearch/sam3) - **ComfyUI Integration**: ComfyUI-SAM3 +- **Interactive Points Editor**: Adapted from [ComfyUI-KJNodes](https://github.com/kijai/ComfyUI-KJNodes) by kijai (Apache 2.0 License). The SAM3PointsEditor node is based on the PointsEditor implementation from KJNodes, simplified for SAM3-specific point-based segmentation. ## License diff --git a/__init__.py b/__init__.py index 8e99352..77f541f 100644 --- a/__init__.py +++ b/__init__.py @@ -14,6 +14,14 @@ Helper Nodes (Visual Prompt Creation): - SAM3CombineBoxes: Combine multiple box prompts (up to 5) - SAM3CombinePoints: Combine multiple point prompts (up to 10) +Interactive Features: +- Right-click any node with IMAGE/MASK output → "Open in SAM3 Detector" +- Interactive point-and-click segmentation (left-click=positive, right-click=negative) + +Video Tracking: +- SAM3VideoModelLoader, SAM3InitVideoSession, SAM3AddVideoPrompt +- SAM3PropagateVideo, SAM3VideoOutput, SAM3CloseVideoSession + Workflow Example: [SAM3CreateBox] → [SAM3CombineBoxes] → [SAM3Segmentation] ← [LoadImage] [SAM3CreatePoint] → [SAM3CombinePoints] ↗ @@ -22,20 +30,24 @@ All geometric refinement uses SAM3's grounding model approach. No JSON typing required - pure visual node-based workflow! Author: ComfyUI-SAM3 -Version: 2.0.0 +Version: 2.1.0 License: MIT """ from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS -# Web directory for custom UI (optional, not needed for basic functionality) -WEB_DIRECTORY = None +# Web directory for custom UI (interactive SAM3 detector) +WEB_DIRECTORY = "./web" + +# Import server to register API endpoints +from . import sam3_server # Version info -__version__ = "2.0.0" +__version__ = "2.1.0" # Export for ComfyUI __all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS', 'WEB_DIRECTORY'] print(f"[SAM3] ComfyUI-SAM3 v{__version__} loaded successfully") print(f"[SAM3] Available nodes: {', '.join(NODE_CLASS_MAPPINGS.keys())}") +print(f"[SAM3] Interactive SAM3 Detector: Right-click any IMAGE/MASK node → 'Open in SAM3 Detector'") diff --git a/assets/bedroom.mp4 b/assets/bedroom.mp4 new file mode 100644 index 0000000..50a9349 Binary files /dev/null and b/assets/bedroom.mp4 differ diff --git a/nodes/__init__.py b/nodes/__init__.py index d2b9206..0b95122 100644 --- a/nodes/__init__.py +++ b/nodes/__init__.py @@ -6,14 +6,22 @@ from .load_model import NODE_CLASS_MAPPINGS as LOAD_MODEL_MAPPINGS from .load_model import NODE_DISPLAY_NAME_MAPPINGS as LOAD_MODEL_DISPLAY_MAPPINGS from .segmentation import NODE_CLASS_MAPPINGS as SEGMENTATION_MAPPINGS from .segmentation import NODE_DISPLAY_NAME_MAPPINGS as SEGMENTATION_DISPLAY_MAPPINGS +from .sam3_video_nodes import NODE_CLASS_MAPPINGS as VIDEO_MAPPINGS +from .sam3_video_nodes import NODE_DISPLAY_NAME_MAPPINGS as VIDEO_DISPLAY_MAPPINGS +from .sam3_interactive import NODE_CLASS_MAPPINGS as INTERACTIVE_MAPPINGS +from .sam3_interactive import NODE_DISPLAY_NAME_MAPPINGS as INTERACTIVE_DISPLAY_MAPPINGS # Combine all node mappings NODE_CLASS_MAPPINGS = {} NODE_CLASS_MAPPINGS.update(LOAD_MODEL_MAPPINGS) NODE_CLASS_MAPPINGS.update(SEGMENTATION_MAPPINGS) +NODE_CLASS_MAPPINGS.update(VIDEO_MAPPINGS) +NODE_CLASS_MAPPINGS.update(INTERACTIVE_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS = {} NODE_DISPLAY_NAME_MAPPINGS.update(LOAD_MODEL_DISPLAY_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(SEGMENTATION_DISPLAY_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(VIDEO_DISPLAY_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(INTERACTIVE_DISPLAY_MAPPINGS) __all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS'] diff --git a/nodes/sam3_interactive.py b/nodes/sam3_interactive.py new file mode 100644 index 0000000..b248daf --- /dev/null +++ b/nodes/sam3_interactive.py @@ -0,0 +1,134 @@ +""" +SAM3 Point Collector - Interactive Point Selection + +Point editor widget adapted from ComfyUI-KJNodes +Original: https://github.com/kijai/ComfyUI-KJNodes +Author: kijai +License: Apache 2.0 + +Modifications for SAM3: +- Removed bounding box functionality +- Simplified to positive/negative points only +- Outputs point arrays for use with SAM3Segmentation node +""" + +import torch +import numpy as np +import json +import io +import base64 +from PIL import Image + + +class SAM3PointCollector: + """ + Interactive Point Collector for SAM3 + + Displays image canvas in the node where users can click to add: + - Positive points (Left-click) - green circles + - Negative points (Shift+Left-click or Right-click) - red circles + + Outputs point arrays to feed into SAM3Segmentation node. + """ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "image": ("IMAGE",), + "points_store": ("STRING", {"multiline": False, "default": "{}"}), + "coordinates": ("STRING", {"multiline": False, "default": "[]"}), + "neg_coordinates": ("STRING", {"multiline": False, "default": "[]"}), + }, + } + + RETURN_TYPES = ("SAM3_POINTS_PROMPT", "SAM3_POINTS_PROMPT") + RETURN_NAMES = ("positive_points", "negative_points") + FUNCTION = "collect_points" + CATEGORY = "SAM3" + OUTPUT_NODE = True # Makes node executable even without outputs connected + + def collect_points(self, image, points_store, coordinates, neg_coordinates): + """ + Collect points from interactive canvas + + Args: + image: ComfyUI image tensor [B, H, W, C] + points_store: Combined JSON storage (hidden widget) + coordinates: Positive points JSON (hidden widget) + neg_coordinates: Negative points JSON (hidden widget) + + Returns: + Tuple of (positive_points, negative_points) as separate SAM3_POINTS_PROMPT outputs + """ + # Parse coordinates from JSON + try: + pos_coords = json.loads(coordinates) if coordinates and coordinates.strip() else [] + neg_coords = json.loads(neg_coordinates) if neg_coordinates and neg_coordinates.strip() else [] + except json.JSONDecodeError: + pos_coords = [] + neg_coords = [] + + print(f"[SAM3 Point Collector] Collected {len(pos_coords)} positive, {len(neg_coords)} negative points") + + # Get image dimensions for normalization + img_height, img_width = image.shape[1], image.shape[2] + print(f"[SAM3 Point Collector] Image dimensions: {img_width}x{img_height}") + + # Convert to SAM3 point format - separate positive and negative outputs + # SAM3 expects normalized coordinates (0-1), so divide by image dimensions + positive_points = {"points": [], "labels": []} + negative_points = {"points": [], "labels": []} + + # Add positive points (label = 1) - normalize to 0-1 + for p in pos_coords: + normalized_x = p['x'] / img_width + normalized_y = p['y'] / img_height + positive_points["points"].append([normalized_x, normalized_y]) + positive_points["labels"].append(1) + print(f"[SAM3 Point Collector] Positive point: ({p['x']:.1f}, {p['y']:.1f}) -> ({normalized_x:.3f}, {normalized_y:.3f})") + + # Add negative points (label = 0) - normalize to 0-1 + for n in neg_coords: + normalized_x = n['x'] / img_width + normalized_y = n['y'] / img_height + negative_points["points"].append([normalized_x, normalized_y]) + negative_points["labels"].append(0) + print(f"[SAM3 Point Collector] Negative point: ({n['x']:.1f}, {n['y']:.1f}) -> ({normalized_x:.3f}, {normalized_y:.3f})") + + print(f"[SAM3 Point Collector] Output: {len(positive_points['points'])} positive, {len(negative_points['points'])} negative") + + # Send image back to widget as base64 + img_base64 = self.tensor_to_base64(image) + + return { + "ui": {"bg_image": [img_base64]}, + "result": (positive_points, negative_points) + } + + def tensor_to_base64(self, tensor): + """Convert ComfyUI image tensor to base64 string for JavaScript widget""" + # Convert from [B, H, W, C] to PIL Image + # Take first image if batch + img_array = tensor[0].cpu().numpy() + # Convert from 0-1 float to 0-255 uint8 + img_array = (img_array * 255).astype(np.uint8) + pil_img = Image.fromarray(img_array) + + # Convert to base64 + buffered = io.BytesIO() + pil_img.save(buffered, format="JPEG", quality=75) + img_bytes = buffered.getvalue() + img_base64 = base64.b64encode(img_bytes).decode('utf-8') + + return img_base64 + + +# Node mappings for ComfyUI registration +NODE_CLASS_MAPPINGS = { + "SAM3PointCollector": SAM3PointCollector, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "SAM3PointCollector": "SAM3 Point Collector", +} diff --git a/nodes/sam3_video_nodes.py b/nodes/sam3_video_nodes.py new file mode 100644 index 0000000..b8823bf --- /dev/null +++ b/nodes/sam3_video_nodes.py @@ -0,0 +1,423 @@ +""" +SAM3 Video Tracking Nodes for ComfyUI + +These nodes provide video object tracking and segmentation capabilities using SAM3. +They use the full sam3 library (not the vendored image-only version). +""" + +import sys +from pathlib import Path +import torch +import numpy as np +import folder_paths + +# Add the full sam3 library to the path +sam3_root = Path(__file__).parent.parent.parent.parent.parent / "sam3" +if str(sam3_root) not in sys.path: + sys.path.insert(0, str(sam3_root)) + +from sam3.model_builder import build_sam3_video_predictor + + +class SAM3VideoModelLoader: + """Load SAM3 model for video tracking""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "checkpoint_path": ("STRING", { + "default": "", + "multiline": False + }), + "bpe_path": ("STRING", { + "default": "", + "multiline": False + }), + "hf_token": ("STRING", { + "default": "", + "multiline": False + }), + }, + "optional": { + "use_multi_gpu": ("BOOLEAN", {"default": False}), + "gpu_ids": ("STRING", { + "default": "0", + "multiline": False + }), + } + } + + RETURN_TYPES = ("SAM3_VIDEO_MODEL",) + RETURN_NAMES = ("video_model",) + FUNCTION = "load_model" + CATEGORY = "SAM3/video" + + def load_model(self, checkpoint_path, bpe_path, hf_token, use_multi_gpu=False, gpu_ids="0"): + """Load the SAM3 video model""" + import os + if hf_token: + os.environ["HF_TOKEN"] = hf_token + + # Parse GPU IDs if using multi-GPU + gpus_to_use = None + if use_multi_gpu: + gpus_to_use = [int(x.strip()) for x in gpu_ids.split(",")] + + print(f"[SAM3 Video] Loading video model from {checkpoint_path}") + + # Build the video predictor + predictor = build_sam3_video_predictor( + checkpoint_path=checkpoint_path if checkpoint_path else None, + bpe_path=bpe_path if bpe_path else None, + gpus_to_use=gpus_to_use, + ) + + print(f"[SAM3 Video] Model loaded successfully") + + return (predictor,) + + +class SAM3InitVideoSession: + """Initialize a video tracking session""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "video_model": ("SAM3_VIDEO_MODEL",), + "video_frames": ("IMAGE",), # ComfyUI video frames [B, H, W, C] + }, + "optional": { + "session_id": ("STRING", { + "default": "", + "multiline": False + }), + } + } + + RETURN_TYPES = ("SAM3_VIDEO_SESSION", "STRING") + RETURN_NAMES = ("session", "session_id") + FUNCTION = "init_session" + CATEGORY = "SAM3/video" + + def init_session(self, video_model, video_frames, session_id=""): + """Initialize a tracking session with video frames""" + # Convert ComfyUI frames to temporary directory + import tempfile + import os + from PIL import Image + + # Create a temporary directory for frames + temp_dir = tempfile.mkdtemp(prefix="sam3_video_") + + # Save frames as JPEG + num_frames = video_frames.shape[0] + for i in range(num_frames): + frame = video_frames[i].cpu().numpy() + # Convert from [H, W, C] float32 0-1 to uint8 0-255 + frame = (frame * 255).astype(np.uint8) + img = Image.fromarray(frame) + img.save(os.path.join(temp_dir, f"{i:05d}.jpg")) + + print(f"[SAM3 Video] Saved {num_frames} frames to {temp_dir}") + + # Start the session + response = video_model.start_session( + resource_path=temp_dir, + session_id=session_id if session_id else None + ) + + actual_session_id = response["session_id"] + + session_data = { + "model": video_model, + "session_id": actual_session_id, + "temp_dir": temp_dir, + "num_frames": num_frames, + "height": video_frames.shape[1], + "width": video_frames.shape[2], + } + + print(f"[SAM3 Video] Initialized session {actual_session_id}") + + return (session_data, actual_session_id) + + +class SAM3AddVideoPrompt: + """Add a text, box, or point prompt to a specific video frame""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "session": ("SAM3_VIDEO_SESSION",), + "frame_index": ("INT", { + "default": 0, + "min": 0, + "max": 10000, + "step": 1 + }), + }, + "optional": { + "text_prompt": ("STRING", { + "default": "", + "multiline": False + }), + "obj_id": ("INT", { + "default": 1, + "min": 1, + "max": 100, + "step": 1 + }), + "boxes": ("SAM3_BOXES_PROMPT",), + "points": ("SAM3_POINTS_PROMPT",), + } + } + + RETURN_TYPES = ("SAM3_VIDEO_SESSION",) + RETURN_NAMES = ("session",) + FUNCTION = "add_prompt" + CATEGORY = "SAM3/video" + + def add_prompt(self, session, frame_index, text_prompt="", obj_id=1, boxes=None, points=None): + """Add a prompt on a specific frame""" + video_model = session["model"] + session_id = session["session_id"] + + # Prepare prompt parameters + bounding_boxes = None + bounding_box_labels = None + point_coords = None + point_labels = None + + if boxes is not None: + bounding_boxes = boxes["boxes"] + bounding_box_labels = boxes.get("labels", None) + + if points is not None: + point_coords = points["points"] + point_labels = points["labels"] + + print(f"[SAM3 Video] Adding prompt on frame {frame_index}: text='{text_prompt}', obj_id={obj_id}") + + # Add the prompt + response = video_model.add_prompt( + session_id=session_id, + frame_idx=frame_index, + text=text_prompt if text_prompt else None, + points=point_coords, + point_labels=point_labels, + bounding_boxes=bounding_boxes, + bounding_box_labels=bounding_box_labels, + obj_id=obj_id, + ) + + print(f"[SAM3 Video] Prompt added successfully") + + return (session,) + + +class SAM3PropagateVideo: + """Propagate masks through the entire video""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "session": ("SAM3_VIDEO_SESSION",), + }, + "optional": { + "propagation_direction": (["both", "forward", "backward"], {"default": "both"}), + "start_frame_index": ("INT", { + "default": 0, + "min": 0, + "max": 10000, + "step": 1 + }), + "max_frames": ("INT", { + "default": -1, + "min": -1, + "max": 10000, + "step": 1 + }), + } + } + + RETURN_TYPES = ("SAM3_VIDEO_MASKS",) + RETURN_NAMES = ("video_masks",) + FUNCTION = "propagate" + CATEGORY = "SAM3/video" + + def propagate(self, session, propagation_direction="both", start_frame_index=0, max_frames=-1): + """Propagate the prompts through all video frames""" + video_model = session["model"] + session_id = session["session_id"] + num_frames = session["num_frames"] + + max_frame_num_to_track = max_frames if max_frames > 0 else num_frames + + print(f"[SAM3 Video] Propagating in {propagation_direction} direction from frame {start_frame_index}") + + # Collect all frames' masks + all_masks = {} + all_obj_ids = None + + request = { + "type": "propagate_in_video", + "session_id": session_id, + "propagation_direction": propagation_direction, + "start_frame_index": start_frame_index, + "max_frame_num_to_track": max_frame_num_to_track, + } + + for response in video_model.handle_stream_request(request): + frame_idx = response["frame_index"] + outputs = response["outputs"] + + all_masks[frame_idx] = outputs + if all_obj_ids is None: + all_obj_ids = outputs.get("obj_ids", []) + + print(f"[SAM3 Video] Propagation complete. Tracked {len(all_masks)} frames, {len(all_obj_ids)} objects") + + video_masks = { + "session": session, + "masks": all_masks, + "obj_ids": all_obj_ids, + "num_frames": num_frames, + } + + return (video_masks,) + + +class SAM3VideoOutput: + """Output video masks as ComfyUI tensors""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "video_masks": ("SAM3_VIDEO_MASKS",), + }, + "optional": { + "obj_id_filter": ("INT", { + "default": -1, + "min": -1, + "max": 100, + "step": 1 + }), + } + } + + RETURN_TYPES = ("MASK",) + RETURN_NAMES = ("masks",) + FUNCTION = "output_masks" + CATEGORY = "SAM3/video" + + def output_masks(self, video_masks, obj_id_filter=-1): + """Convert video masks to ComfyUI mask format""" + masks_dict = video_masks["masks"] + obj_ids = video_masks["obj_ids"] + num_frames = video_masks["num_frames"] + session = video_masks["session"] + height = session["height"] + width = session["width"] + + # Create output tensor [N, H, W] where N is number of frames + output_masks = torch.zeros((num_frames, height, width), dtype=torch.float32) + + for frame_idx in range(num_frames): + if frame_idx in masks_dict: + frame_output = masks_dict[frame_idx] + + # Get masks for this frame + if "video_res_masks" in frame_output: + frame_masks = frame_output["video_res_masks"] + elif "pred_masks" in frame_output: + # Resize if needed + frame_masks = frame_output["pred_masks"] + if frame_masks.shape[-2:] != (height, width): + frame_masks = torch.nn.functional.interpolate( + frame_masks, + size=(height, width), + mode="bilinear", + align_corners=False, + ) + else: + continue + + # Filter by object ID if specified + if obj_id_filter > 0: + try: + obj_idx = obj_ids.index(obj_id_filter) + mask = frame_masks[obj_idx, 0] > 0.0 + except (ValueError, IndexError): + mask = torch.zeros((height, width), dtype=torch.bool) + else: + # Combine all object masks + mask = (frame_masks[:, 0] > 0.0).any(dim=0) + + output_masks[frame_idx] = mask.float().cpu() + + print(f"[SAM3 Video] Output {num_frames} mask frames") + + return (output_masks,) + + +class SAM3CloseVideoSession: + """Close a video tracking session and cleanup resources""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "session": ("SAM3_VIDEO_SESSION",), + }, + } + + RETURN_TYPES = () + OUTPUT_NODE = True + FUNCTION = "close_session" + CATEGORY = "SAM3/video" + + def close_session(self, session): + """Close the session and cleanup temporary files""" + import shutil + + video_model = session["model"] + session_id = session["session_id"] + temp_dir = session["temp_dir"] + + # Close the session + video_model.close_session(session_id) + + # Clean up temporary directory + if temp_dir and Path(temp_dir).exists(): + shutil.rmtree(temp_dir) + print(f"[SAM3 Video] Cleaned up temp directory: {temp_dir}") + + print(f"[SAM3 Video] Closed session {session_id}") + + return () + + +# Node class mappings +NODE_CLASS_MAPPINGS = { + "SAM3VideoModelLoader": SAM3VideoModelLoader, + "SAM3InitVideoSession": SAM3InitVideoSession, + "SAM3AddVideoPrompt": SAM3AddVideoPrompt, + "SAM3PropagateVideo": SAM3PropagateVideo, + "SAM3VideoOutput": SAM3VideoOutput, + "SAM3CloseVideoSession": SAM3CloseVideoSession, +} + +# Node display name mappings +NODE_DISPLAY_NAME_MAPPINGS = { + "SAM3VideoModelLoader": "SAM3 Load Video Model", + "SAM3InitVideoSession": "SAM3 Init Video Session", + "SAM3AddVideoPrompt": "SAM3 Add Video Prompt", + "SAM3PropagateVideo": "SAM3 Propagate Video", + "SAM3VideoOutput": "SAM3 Video Output", + "SAM3CloseVideoSession": "SAM3 Close Video Session", +} diff --git a/nodes/segmentation.py b/nodes/segmentation.py index e47b620..757813b 100644 --- a/nodes/segmentation.py +++ b/nodes/segmentation.py @@ -41,8 +41,10 @@ class SAM3Segmentation: "multiline": False, "placeholder": "e.g., 'cat', 'person in red', 'car'" }), - "boxes_prompt": ("SAM3_BOXES_PROMPT",), - "points_prompt": ("SAM3_POINTS_PROMPT",), + "positive_boxes": ("SAM3_BOXES_PROMPT",), + "negative_boxes": ("SAM3_BOXES_PROMPT",), + "positive_points": ("SAM3_POINTS_PROMPT",), + "negative_points": ("SAM3_POINTS_PROMPT",), "mask_prompt": ("MASK",), "max_detections": ("INT", { "default": -1, @@ -60,7 +62,8 @@ class SAM3Segmentation: CATEGORY = "SAM3" def segment(self, sam3_model, image, confidence_threshold=0.2, text_prompt="", - boxes_prompt=None, points_prompt=None, mask_prompt=None, max_detections=-1): + positive_boxes=None, negative_boxes=None, positive_points=None, negative_points=None, + mask_prompt=None, max_detections=-1): """ Perform SAM3 segmentation with optional geometric prompts @@ -69,8 +72,10 @@ class SAM3Segmentation: image: ComfyUI image tensor [B, H, W, C] confidence_threshold: Minimum confidence score for detections text_prompt: Optional text description of objects to segment - boxes_prompt: Optional box prompts from SAM3CombineBoxes - points_prompt: Optional point prompts from SAM3CombinePoints + positive_boxes: Optional positive box prompts + negative_boxes: Optional negative box prompts + positive_points: Optional positive point prompts + negative_points: Optional negative point prompts mask_prompt: Optional mask prompt max_detections: Maximum number of detections to return @@ -84,10 +89,14 @@ class SAM3Segmentation: print(f"[SAM3] Running segmentation") if text_prompt: print(f"[SAM3] Text prompt: '{text_prompt}'") - if boxes_prompt: - print(f"[SAM3] Boxes: {len(boxes_prompt['boxes'])}") - if points_prompt: - print(f"[SAM3] Points: {len(points_prompt['points'])}") + if positive_boxes: + print(f"[SAM3] Positive boxes: {len(positive_boxes['boxes'])}") + if negative_boxes: + print(f"[SAM3] Negative boxes: {len(negative_boxes['boxes'])}") + if positive_points: + print(f"[SAM3] Positive points: {len(positive_points['points'])}") + if negative_points: + print(f"[SAM3] Negative points: {len(negative_points['points'])}") if mask_prompt is not None: print(f"[SAM3] Mask prompt provided") print(f"[SAM3] Confidence threshold: {confidence_threshold}") @@ -107,20 +116,42 @@ class SAM3Segmentation: print(f"[SAM3] Adding text prompt...") state = processor.set_text_prompt(text_prompt.strip(), state) - # Add geometric prompts - if boxes_prompt is not None and len(boxes_prompt['boxes']) > 0: - print(f"[SAM3] Adding {len(boxes_prompt['boxes'])} box prompts...") + # Add geometric prompts - combine positive and negative + all_boxes = [] + all_box_labels = [] + + if positive_boxes is not None and len(positive_boxes['boxes']) > 0: + all_boxes.extend(positive_boxes['boxes']) + all_box_labels.extend(positive_boxes['labels']) + + if negative_boxes is not None and len(negative_boxes['boxes']) > 0: + all_boxes.extend(negative_boxes['boxes']) + all_box_labels.extend(negative_boxes['labels']) + + if len(all_boxes) > 0: + print(f"[SAM3] Adding {len(all_boxes)} box prompts...") state = processor.add_multiple_box_prompts( - boxes_prompt['boxes'], - boxes_prompt['labels'], + all_boxes, + all_box_labels, state ) - if points_prompt is not None and len(points_prompt['points']) > 0: - print(f"[SAM3] Adding {len(points_prompt['points'])} point prompts...") + all_points = [] + all_point_labels = [] + + if positive_points is not None and len(positive_points['points']) > 0: + all_points.extend(positive_points['points']) + all_point_labels.extend(positive_points['labels']) + + if negative_points is not None and len(negative_points['points']) > 0: + all_points.extend(negative_points['points']) + all_point_labels.extend(negative_points['labels']) + + if len(all_points) > 0: + print(f"[SAM3] Adding {len(all_points)} point prompts...") state = processor.add_point_prompt( - points_prompt['points'], - points_prompt['labels'], + all_points, + all_point_labels, state ) diff --git a/prestartup_script.py b/prestartup_script.py index 1e34119..77fdfe7 100644 --- a/prestartup_script.py +++ b/prestartup_script.py @@ -71,13 +71,29 @@ def main(): else: print(f"[SAM3] No assets folder found") - # Copy workflows to user workflows folder + # Copy workflows to user workflows folder with SAM3- prefix workflows_src = script_dir / "workflows" workflows_dst = comfy_root / "user" / "default" / "workflows" if workflows_src.exists(): print(f"[SAM3] Copying workflows to {workflows_dst}...") - copied, skipped = copy_files_skip_existing(str(workflows_src), str(workflows_dst)) + os.makedirs(str(workflows_dst), exist_ok=True) + copied = 0 + skipped = 0 + + for item in os.listdir(str(workflows_src)): + src_path = workflows_src / item + if src_path.is_file(): + # Add SAM3- prefix to workflow filenames + prefixed_name = f"SAM3-{item}" if not item.startswith("SAM3-") else item + dst_path = workflows_dst / prefixed_name + + if dst_path.exists(): + skipped += 1 + else: + shutil.copy2(str(src_path), str(dst_path)) + copied += 1 + print(f"[SAM3] Workflows: {copied} copied, {skipped} skipped") else: print(f"[SAM3] No workflows folder found") diff --git a/requirements.txt b/requirements.txt index 87c2fde..c53bcb0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,12 @@ regex iopath>=0.1.10 huggingface_hub +# Video support dependencies +decord>=0.6.0 +opencv-python>=4.8.0 +psutil>=5.9.0 +pycocotools>=2.0.6 + # Standard dependencies numpy>=1.26 Pillow diff --git a/sam3_lib/sam3_video_predictor.py b/sam3_lib/sam3_video_predictor.py new file mode 100644 index 0000000..7355dc7 --- /dev/null +++ b/sam3_lib/sam3_video_predictor.py @@ -0,0 +1,521 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved + +import datetime +import gc +import multiprocessing as mp +import os +import queue +import socket +import sys +import time +import uuid +from contextlib import closing +from typing import List, Optional + +import psutil +import torch + +# Adapted imports for vendored version +import logging +logger = logging.getLogger(__name__) + + +class Sam3VideoPredictor: + # a global dictionary that holds all inference states for this model (key is session_id) + _ALL_INFERENCE_STATES = {} + + def __init__( + self, + checkpoint_path=None, + bpe_path=None, + has_presence_token=True, + geo_encoder_use_img_cross_attn=True, + strict_state_dict_loading=True, + async_loading_frames=False, + video_loader_type="cv2", + apply_temporal_disambiguation: bool = True, + ): + self.async_loading_frames = async_loading_frames + self.video_loader_type = video_loader_type + from .model_builder import build_sam3_video_model + + self.model = ( + build_sam3_video_model( + checkpoint_path=checkpoint_path, + bpe_path=bpe_path, + has_presence_token=has_presence_token, + geo_encoder_use_img_cross_attn=geo_encoder_use_img_cross_attn, + strict_state_dict_loading=strict_state_dict_loading, + apply_temporal_disambiguation=apply_temporal_disambiguation, + ) + .cuda() + .eval() + ) + + @torch.inference_mode() + def handle_request(self, request): + """Dispatch a request based on its type.""" + request_type = request["type"] + if request_type == "start_session": + return self.start_session( + resource_path=request["resource_path"], + session_id=request.get("session_id", None), + ) + elif request_type == "add_prompt": + return self.add_prompt( + session_id=request["session_id"], + frame_idx=request["frame_index"], + text=request.get("text", None), + points=request.get("points", None), + point_labels=request.get("point_labels", None), + bounding_boxes=request.get("bounding_boxes", None), + bounding_box_labels=request.get("bounding_box_labels", None), + obj_id=request.get("obj_id", None), + ) + elif request_type == "remove_object": + return self.remove_object( + session_id=request["session_id"], + obj_id=request["obj_id"], + is_user_action=request.get("is_user_action", True), + ) + elif request_type == "reset_session": + return self.reset_session(session_id=request["session_id"]) + elif request_type == "close_session": + return self.close_session(session_id=request["session_id"]) + else: + raise RuntimeError(f"invalid request type: {request_type}") + + @torch.inference_mode() + def handle_stream_request(self, request): + """Dispatch a stream request based on its type.""" + request_type = request["type"] + if request_type == "propagate_in_video": + yield from self.propagate_in_video( + session_id=request["session_id"], + propagation_direction=request.get("propagation_direction", "both"), + start_frame_idx=request.get("start_frame_index", None), + max_frame_num_to_track=request.get("max_frame_num_to_track", None), + ) + else: + raise RuntimeError(f"invalid request type: {request_type}") + + def start_session(self, resource_path, session_id=None): + """ + Start a new inference session on an image or a video. Here `resource_path` + can be either a path to an image file (for image inference) or an MP4 file + or directory with JPEG video frames (for video inference). + + If `session_id` is defined, it will be used as identifier for the + session. If it is not defined, the start_session function will create + a session id and return it. + """ + # get an initial inference_state from the model + inference_state = self.model.init_state( + resource_path=resource_path, + async_loading_frames=self.async_loading_frames, + video_loader_type=self.video_loader_type, + ) + if not session_id: + session_id = str(uuid.uuid4()) + self._ALL_INFERENCE_STATES[session_id] = { + "state": inference_state, + "session_id": session_id, + "start_time": time.time(), + } + logger.debug( + f"started new session {session_id}; {self._get_session_stats()}; " + f"{self._get_torch_and_gpu_properties()}" + ) + return {"session_id": session_id} + + def add_prompt( + self, + session_id: str, + frame_idx: int, + text: Optional[str] = None, + points: Optional[List[List[float]]] = None, + point_labels: Optional[List[int]] = None, + bounding_boxes: Optional[List[List[float]]] = None, + bounding_box_labels: Optional[List[int]] = None, + obj_id: Optional[int] = None, + ): + """Add text, box and/or point prompt on a specific video frame.""" + logger.debug( + f"add prompt on frame {frame_idx} in session {session_id}: " + f"{text=}, {points=}, {point_labels=}, " + f"{bounding_boxes=}, {bounding_box_labels=}" + ) + session = self._get_session(session_id) + inference_state = session["state"] + + frame_idx, outputs = self.model.add_prompt( + inference_state=inference_state, + frame_idx=frame_idx, + text_str=text, + points=points, + point_labels=point_labels, + boxes_xywh=bounding_boxes, + box_labels=bounding_box_labels, + obj_id=obj_id, + ) + return {"frame_index": frame_idx, "outputs": outputs} + + def remove_object( + self, + session_id: str, + obj_id: int, + is_user_action: bool = True, + ): + """Remove an object from tracking.""" + logger.debug( + f"remove object {obj_id} in session {session_id}: " f"{is_user_action=}" + ) + session = self._get_session(session_id) + inference_state = session["state"] + + self.model.remove_object( + inference_state=inference_state, + obj_id=obj_id, + is_user_action=is_user_action, + ) + return {"is_success": True} + + def propagate_in_video( + self, + session_id, + propagation_direction, + start_frame_idx, + max_frame_num_to_track, + ): + """Propagate the added prompts to get grounding results on all video frames.""" + logger.debug( + f"propagate in video in session {session_id}: " + f"{propagation_direction=}, {start_frame_idx=}, {max_frame_num_to_track=}" + ) + try: + session = self._get_session(session_id) + inference_state = session["state"] + if propagation_direction not in ["both", "forward", "backward"]: + raise ValueError( + f"invalid propagation direction: {propagation_direction}" + ) + + # First doing the forward propagation + if propagation_direction in ["both", "forward"]: + for frame_idx, outputs in self.model.propagate_in_video( + inference_state=inference_state, + start_frame_idx=start_frame_idx, + max_frame_num_to_track=max_frame_num_to_track, + reverse=False, + ): + yield {"frame_index": frame_idx, "outputs": outputs} + # Then doing the backward propagation (reverse in time) + if propagation_direction in ["both", "backward"]: + for frame_idx, outputs in self.model.propagate_in_video( + inference_state=inference_state, + start_frame_idx=start_frame_idx, + max_frame_num_to_track=max_frame_num_to_track, + reverse=True, + ): + yield {"frame_index": frame_idx, "outputs": outputs} + finally: + # Log upon completion (so that e.g. we can see if two propagations happen in parallel). + # Using `finally` here to log even when the tracking is aborted with GeneratorExit. + logger.debug( + f"propagation ended in session {session_id}; {self._get_session_stats()}" + ) + + def reset_session(self, session_id): + """Reset the session to its initial state (as when it's initial opened).""" + logger.debug(f"reset session {session_id}") + session = self._get_session(session_id) + inference_state = session["state"] + self.model.reset_state(inference_state) + return {"is_success": True} + + def close_session(self, session_id): + """ + Close a session. This method is idempotent and can be called multiple + times on the same "session_id". + """ + session = self._ALL_INFERENCE_STATES.pop(session_id, None) + if session is None: + logger.warning( + f"cannot close session {session_id} as it does not exist (it might have expired); " + f"{self._get_session_stats()}" + ) + else: + del session + gc.collect() + logger.info(f"removed session {session_id}; {self._get_session_stats()}") + return {"is_success": True} + + def _get_session(self, session_id): + session = self._ALL_INFERENCE_STATES.get(session_id, None) + if session is None: + raise RuntimeError( + f"Cannot find session {session_id}; it might have expired" + ) + return session + + def _get_session_stats(self): + """Get a statistics string for live sessions and their GPU usage.""" + # print both the session ids and their video frame numbers + live_session_strs = [ + f"'{session_id}' ({session['state']['num_frames']} frames)" + for session_id, session in self._ALL_INFERENCE_STATES.items() + ] + session_stats_str = ( + f"live sessions: [{', '.join(live_session_strs)}], GPU memory: " + f"{torch.cuda.memory_allocated() // 1024**2} MiB used and " + f"{torch.cuda.memory_reserved() // 1024**2} MiB reserved" + f" (max over time: {torch.cuda.max_memory_allocated() // 1024**2} MiB used " + f"and {torch.cuda.max_memory_reserved() // 1024**2} MiB reserved)" + ) + return session_stats_str + + def _get_torch_and_gpu_properties(self): + """Get a string for PyTorch and GPU properties (for logging and debugging).""" + torch_and_gpu_str = ( + f"torch: {torch.__version__} with CUDA arch {torch.cuda.get_arch_list()}, " + f"GPU device: {torch.cuda.get_device_properties(torch.cuda.current_device())}" + ) + return torch_and_gpu_str + + def shutdown(self): + """Shutdown the predictor and clear all sessions.""" + self._ALL_INFERENCE_STATES.clear() + + +class Sam3VideoPredictorMultiGPU(Sam3VideoPredictor): + def __init__(self, *model_args, gpus_to_use=None, **model_kwargs): + if gpus_to_use is None: + # if not specified, use only the current GPU by default + gpus_to_use = [torch.cuda.current_device()] + + IS_MAIN_PROCESS = os.getenv("IS_MAIN_PROCESS", "1") == "1" + if IS_MAIN_PROCESS: + gpus_to_use = sorted(set(gpus_to_use)) + logger.info(f"using the following GPU IDs: {gpus_to_use}") + assert len(gpus_to_use) > 0 and all(isinstance(i, int) for i in gpus_to_use) + assert all(0 <= i < torch.cuda.device_count() for i in gpus_to_use) + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = f"{self._find_free_port()}" + os.environ["RANK"] = "0" + os.environ["WORLD_SIZE"] = f"{len(gpus_to_use)}" + + self.gpus_to_use = gpus_to_use + self.rank = int(os.environ["RANK"]) + self.world_size = int(os.environ["WORLD_SIZE"]) + self.rank_str = f"rank={self.rank} with world_size={self.world_size}" + self.device = torch.device(f"cuda:{self.gpus_to_use[self.rank]}") + torch.cuda.set_device(self.device) + self.has_shutdown = False + if self.rank == 0: + logger.info("\n\n\n\t*** START loading model on all ranks ***\n\n") + + logger.info(f"loading model on {self.rank_str} -- this could take a while ...") + super().__init__(*model_args, **model_kwargs) + logger.info(f"loading model on {self.rank_str} -- DONE locally") + + if self.world_size > 1 and self.rank == 0: + # start the worker processes *after* the model is loaded in the main process + # so that the main process can run torch.compile and fill the cache first + self._start_worker_processes(*model_args, **model_kwargs) + for rank in range(1, self.world_size): + self.command_queues[rank].put(("start_nccl_process_group", None)) + self._start_nccl_process_group() + + if self.rank == 0: + logger.info("\n\n\n\t*** DONE loading model on all ranks ***\n\n") + + @torch.inference_mode() + def handle_request(self, request): + """Dispatch a request based on its type.""" + if self.has_shutdown: + raise RuntimeError( + "cannot handle request after the predictor has shutdown; please create a new predictor" + ) + + # when starting a session, we need to create a session id before dispatching + # the request to the workers + if request["type"] == "start_session" and request.get("session_id") is None: + request["session_id"] = str(uuid.uuid4()) + # dispatch the request to all worker processes + if self.world_size > 1 and self.rank == 0: + for rank in range(1, self.world_size): + self.command_queues[rank].put((request, False)) + + response = super().handle_request(request) + + if self.world_size > 1: + torch.distributed.barrier() # wait for all ranks to finish + return response + + @torch.inference_mode() + def handle_stream_request(self, request): + """Dispatch a stream request based on its type.""" + if self.has_shutdown: + raise RuntimeError( + "cannot handle request after the predictor has shutdown; please create a new predictor" + ) + + # dispatch the request to all worker processes + if self.world_size > 1 and self.rank == 0: + for rank in range(1, self.world_size): + self.command_queues[rank].put((request, True)) + + yield from super().handle_stream_request(request) + + if self.world_size > 1: + torch.distributed.barrier() # wait for all ranks to finish + + def _start_worker_processes(self, *model_args, **model_kwargs): + """Start worker processes for handling model inference.""" + world_size = self.world_size + logger.info(f"spawning {world_size - 1} worker processes") + # Use "spawn" (instead of "fork") for different PyTorch or CUDA context + mp_ctx = mp.get_context("spawn") + self.command_queues = {rank: mp_ctx.Queue() for rank in range(1, world_size)} + self.result_queues = {rank: mp_ctx.Queue() for rank in range(1, world_size)} + parent_pid = os.getpid() + for rank in range(1, world_size): + # set the environment variables for each worker process + os.environ["IS_MAIN_PROCESS"] = "0" # mark this as a worker process + os.environ["RANK"] = f"{rank}" + worker_process = mp_ctx.Process( + target=Sam3VideoPredictorMultiGPU._worker_process_command_loop, + args=( + rank, + world_size, + self.command_queues[rank], + self.result_queues[rank], + model_args, + model_kwargs, + self.gpus_to_use, + parent_pid, + ), + daemon=True, + ) + worker_process.start() + # revert the environment variables for the main process + os.environ["IS_MAIN_PROCESS"] = "1" + os.environ["RANK"] = "0" + # wait for all the worker processes to load the model and collect their PIDs + self.worker_pids = {} + for rank in range(1, self.world_size): + # a large timeout to cover potentially long model loading time due to compilation + _, worker_pid = self.result_queues[rank].get(timeout=7200) + self.worker_pids[rank] = worker_pid + logger.info(f"spawned {world_size - 1} worker processes") + + def _start_nccl_process_group(self): + rank = int(os.environ["RANK"]) + world_size = int(os.environ["WORLD_SIZE"]) + if world_size == 1: + return + + logger.debug(f"starting NCCL process group on {rank=} with {world_size=}") + assert not torch.distributed.is_initialized() + # use the "env://" init method with environment variables set in start_worker_processes + # a short 3-min timeout to quickly detect any synchronization failures + timeout_sec = int(os.getenv("SAM3_COLLECTIVE_OP_TIMEOUT_SEC", "180")) + timeout = datetime.timedelta(seconds=timeout_sec) + torch.distributed.init_process_group( + backend="nccl", + init_method="env://", + timeout=timeout, + device_id=self.device, + ) + # warm-up the NCCL process group by running a dummy all-reduce + tensor = torch.ones(1024, 1024).cuda() + torch.distributed.all_reduce(tensor) + logger.debug(f"started NCCL process group on {rank=} with {world_size=}") + + def _find_free_port(self) -> int: + """ + Find a free port (a random free port from 1024 to 65535 will be selected) + https://stackoverflow.com/questions/1365265/on-localhost-how-do-i-pick-a-free-port-number) + """ + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + s.bind(("", 0)) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return s.getsockname()[1] + + @staticmethod + def _worker_process_command_loop( + rank, + world_size, + command_queue, + result_queue, + model_args, + model_kwargs, + gpus_to_use, + parent_pid, + ): + """ + The command loop for each worker process. It listens to commands from the main process + and executes them using the model. + """ + logger.info(f"starting worker process {rank=} with {world_size=}") + # verify that the environment variables are set correctly + assert int(os.environ["IS_MAIN_PROCESS"]) == 0 + assert int(os.environ["RANK"]) == rank + assert int(os.environ["WORLD_SIZE"]) == world_size + # load the model in this worker process + predictor = Sam3VideoPredictorMultiGPU( + *model_args, gpus_to_use=gpus_to_use, **model_kwargs + ) + logger.info(f"started worker {rank=} with {world_size=}") + # return the worker process id to the main process for bookkeeping + worker_pid = os.getpid() + result_queue.put(("load_model", worker_pid)) + + # wait for the command to start the NCCL process group + request_type, _ = command_queue.get(timeout=7200) + assert request_type == "start_nccl_process_group" + predictor._start_nccl_process_group() + + # keep listening to commands from the main process + while True: + try: + request, is_stream_request = command_queue.get(timeout=5.0) + if request == "shutdown": + logger.info(f"worker {rank=} shutting down") + torch.distributed.destroy_process_group() + result_queue.put(("shutdown", True)) # acknowledge the shutdown + sys.exit(0) + + logger.debug(f"worker {rank=} received request {request['type']=}") + if is_stream_request: + for _ in predictor.handle_stream_request(request): + pass # handle stream requests in a generator fashion + else: + predictor.handle_request(request) + except queue.Empty: + # Usually Python's multiprocessing module will shutdown all the daemon worker + # processes when the main process exits gracefully. However, the user may kill + # the main process using SIGKILL and thereby leaving no chance for the main process + # to clean up its daemon child processes. So here we manually check whether the + # parent process still exists (every 5 sec as in `command_queue.get` timeout). + if not psutil.pid_exists(parent_pid): + logger.info( + f"stopping worker {rank=} as its parent process has exited" + ) + sys.exit(1) + except Exception as e: + logger.error(f"worker {rank=} exception: {e}", exc_info=True) + + def shutdown(self): + """Shutdown all worker processes.""" + if self.rank == 0 and self.world_size > 1: + logger.info(f"shutting down {self.world_size - 1} worker processes") + for rank in range(1, self.world_size): + self.command_queues[rank].put(("shutdown", False)) + torch.distributed.destroy_process_group() + for rank in range(1, self.world_size): + self.result_queues[rank].get() # wait for the worker to acknowledge + logger.info(f"shut down {self.world_size - 1} worker processes") + self.has_shutdown = True + + super().shutdown() diff --git a/sam3_server.py b/sam3_server.py new file mode 100644 index 0000000..e76672d --- /dev/null +++ b/sam3_server.py @@ -0,0 +1,258 @@ +""" +SAM3 Server - API endpoints for interactive point selection + +Provides endpoints for the SAM3 interactive editor: +- /sam3/prepare: Load model and image +- /sam3/detect: Run SAM3 with point prompts +""" + +import io +import json +import os +import threading +import numpy as np +import torch +from PIL import Image +from aiohttp import web +from server import PromptServer +import folder_paths + +# Global state for SAM3 predictor +sam3_predictor = None +sam3_lock = threading.Condition() + + +def load_image_from_path(filename, file_type, subfolder=""): + """Load image from ComfyUI storage""" + # Get the appropriate directory based on type + if file_type == "input": + input_dir = folder_paths.get_input_directory() + elif file_type == "temp": + input_dir = folder_paths.get_temp_directory() + elif file_type == "output": + input_dir = folder_paths.get_output_directory() + else: + input_dir = folder_paths.get_input_directory() + + # Build full path + if subfolder: + full_path = os.path.join(input_dir, subfolder, filename) + else: + full_path = os.path.join(input_dir, filename) + + print(f"[SAM3 Server] Loading image from: {full_path}") + + # Load image + image = Image.open(full_path) + if image.mode != 'RGB': + image = image.convert('RGB') + + return image + + +@PromptServer.instance.routes.post("/sam3/prepare") +async def sam3_prepare(request): + """ + Load SAM3 model and prepare image for interactive editing + + Request JSON: + { + "sam_model_name": "auto", + "filename": "image.png", + "type": "temp", + "subfolder": "" + } + """ + global sam3_predictor + + try: + data = await request.json() + filename = data.get("filename", "") + file_type = data.get("type", "temp") + subfolder = data.get("subfolder", "") + + print(f"[SAM3 Server] Preparing SAM3 for image: {filename}") + + # Load image + image = load_image_from_path(filename, file_type, subfolder) + + # Load SAM3 model if not already loaded + # For now, we'll load the model on-demand when /sam3/detect is called + # This is because the model is managed by LoadSAM3Model node + + with sam3_lock: + sam3_predictor = { + "image": image, + "filename": filename, + "type": file_type, + "subfolder": subfolder, + } + print(f"[SAM3 Server] Image prepared: {image.size}") + + return web.Response(status=200) + + except Exception as e: + print(f"[SAM3 Server] Error in /sam3/prepare: {e}") + import traceback + traceback.print_exc() + return web.Response(status=500, text=str(e)) + + +@PromptServer.instance.routes.post("/sam3/detect") +async def sam3_detect(request): + """ + Run SAM3 detection with point prompts + + Request JSON: + { + "positive_points": [[x1, y1], [x2, y2], ...], + "negative_points": [[x3, y3], ...], + "threshold": 0.7 + } + + Returns: PNG image of the mask + """ + global sam3_predictor + + try: + data = await request.json() + positive_points = data.get("positive_points", []) + negative_points = data.get("negative_points", []) + threshold = data.get("threshold", 0.7) + + print(f"[SAM3 Server] Detecting with {len(positive_points)} positive and {len(negative_points)} negative points") + + if sam3_predictor is None: + return web.Response(status=400, text="SAM3 not prepared. Call /sam3/prepare first.") + + # Get the stored image + image = sam3_predictor.get("image") + if image is None: + return web.Response(status=400, text="No image loaded") + + # Load SAM3 model on-demand + # We need to import here to avoid circular imports + from .sam3_lib.model_builder import build_sam3_image_model + from .sam3_lib.model.sam3_image_processor import SAM3ImageProcessor + + # Build model (cached globally if possible) + if "model" not in sam3_predictor: + print(f"[SAM3 Server] Loading SAM3 model...") + model = build_sam3_image_model() + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = model.to(device).eval() + + processor = SAM3ImageProcessor(model, device=device) + + sam3_predictor["model"] = model + sam3_predictor["processor"] = processor + sam3_predictor["device"] = device + print(f"[SAM3 Server] SAM3 model loaded on {device}") + + processor = sam3_predictor["processor"] + device = sam3_predictor["device"] + + # Set image and extract features + print(f"[SAM3 Server] Setting image...") + state = processor.set_image(image) + + # Prepare points and labels + all_points = [] + all_labels = [] + + # Add positive points (label=1) + for point in positive_points: + all_points.append(point) + all_labels.append(1) + + # Add negative points (label=0) + for point in negative_points: + all_points.append(point) + all_labels.append(0) + + print(f"[SAM3 Server] Total points: {len(all_points)}") + + # Run detection with points if we have any + if len(all_points) > 0: + # Convert to tensors + points_tensor = torch.tensor(all_points, dtype=torch.float32).unsqueeze(0) # [1, N, 2] + labels_tensor = torch.tensor(all_labels, dtype=torch.int32).unsqueeze(0) # [1, N] + + print(f"[SAM3 Server] Running point prompt detection...") + state = processor.add_point_prompt(points_tensor, labels_tensor, state) + + # Get masks from state + masks = state.get("masks", None) + scores = state.get("scores", None) + + if masks is None or len(masks) == 0: + print(f"[SAM3 Server] No masks generated") + # Return empty mask + mask_array = np.zeros((image.height, image.width), dtype=np.uint8) + else: + print(f"[SAM3 Server] Got {len(masks)} masks, scores: {scores}") + + # Filter by threshold if scores available + if scores is not None: + valid_indices = scores > threshold + if valid_indices.sum() > 0: + masks = masks[valid_indices] + scores = scores[valid_indices] + print(f"[SAM3 Server] Filtered to {len(masks)} masks above threshold {threshold}") + else: + # Take best mask if none above threshold + best_idx = torch.argmax(scores) + masks = masks[best_idx:best_idx+1] + scores = scores[best_idx:best_idx+1] + print(f"[SAM3 Server] No masks above threshold, using best mask with score {scores[0]:.3f}") + + # Combine multiple masks using bitwise OR + mask_array = np.zeros((image.height, image.width), dtype=np.uint8) + for mask in masks: + # Convert tensor to numpy + if isinstance(mask, torch.Tensor): + mask_np = mask.cpu().numpy() + else: + mask_np = mask + + # Ensure correct shape + if mask_np.ndim == 3: + mask_np = mask_np[0] # Remove batch dimension + + # Resize if needed + if mask_np.shape != (image.height, image.width): + from PIL import Image as PILImage + mask_pil = PILImage.fromarray((mask_np > 0).astype(np.uint8) * 255) + mask_pil = mask_pil.resize((image.width, image.height), PILImage.NEAREST) + mask_np = np.array(mask_pil) > 0 + + # Combine with OR + mask_array = np.logical_or(mask_array, mask_np > 0).astype(np.uint8) * 255 + + else: + print(f"[SAM3 Server] No points provided, returning empty mask") + mask_array = np.zeros((image.height, image.width), dtype=np.uint8) + + # Convert mask to PIL Image + mask_image = Image.fromarray(mask_array, mode='L') + + # Convert to PNG bytes + img_buffer = io.BytesIO() + mask_image.save(img_buffer, format='PNG') + img_buffer.seek(0) + + print(f"[SAM3 Server] Returning mask PNG") + + return web.Response( + body=img_buffer.getvalue(), + headers={'Content-Type': 'image/png'} + ) + + except Exception as e: + print(f"[SAM3 Server] Error in /sam3/detect: {e}") + import traceback + traceback.print_exc() + return web.Response(status=500, text=str(e)) + + +print("[SAM3 Server] Registered routes: /sam3/prepare, /sam3/detect") diff --git a/web/sam3-editor.js b/web/sam3-editor.js new file mode 100644 index 0000000..67e542e --- /dev/null +++ b/web/sam3-editor.js @@ -0,0 +1,640 @@ +import { app } from "../../scripts/app.js"; +import { api } from "../../scripts/api.js"; +import { ComfyDialog, $el } from "../../scripts/ui.js"; +import { ComfyApp } from "../../scripts/app.js"; +import { ClipspaceDialog } from "../../extensions/core/clipspace.js"; + +function addMenuHandler(nodeType, cb) { + const getOpts = nodeType.prototype.getExtraMenuOptions; + nodeType.prototype.getExtraMenuOptions = function () { + const r = getOpts.apply(this, arguments); + cb.apply(this, arguments); + return r; + }; +} + +// Helper function to convert a data URL to a Blob object +function dataURLToBlob(dataURL) { + const parts = dataURL.split(';base64,'); + const contentType = parts[0].split(':')[1]; + const byteString = atob(parts[1]); + const arrayBuffer = new ArrayBuffer(byteString.length); + const uint8Array = new Uint8Array(arrayBuffer); + for (let i = 0; i < byteString.length; i++) { + uint8Array[i] = byteString.charCodeAt(i); + } + return new Blob([arrayBuffer], { type: contentType }); +} + +function loadedImageToBlob(image) { + const canvas = document.createElement('canvas'); + + canvas.width = image.width; + canvas.height = image.height; + + const ctx = canvas.getContext('2d'); + + ctx.drawImage(image, 0, 0); + + const dataURL = canvas.toDataURL('image/png', 1); + const blob = dataURLToBlob(dataURL); + + return blob; +} + +async function uploadMask(filepath, formData) { + await api.fetchApi('/upload/mask', { + method: 'POST', + body: formData + }).then(response => {}).catch(error => { + console.error('Error:', error); + }); + + ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']] = new Image(); + ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src = `view?filename=${filepath.filename}&type=${filepath.type}`; + + if(ComfyApp.clipspace.images) + ComfyApp.clipspace.images[ComfyApp.clipspace['selectedIndex']] = filepath; + + ClipspaceDialog.invalidatePreview(); +} + +class SAM3EditorDialog extends ComfyDialog { + static instance = null; + + static getInstance() { + if(!SAM3EditorDialog.instance) { + SAM3EditorDialog.instance = new SAM3EditorDialog(); + } + + return SAM3EditorDialog.instance; + } + + constructor() { + super(); + this.element = $el("div.comfy-modal", { parent: document.body }, + [ $el("div.comfy-modal-content", + [...this.createButtons()]), + ]); + } + + createButtons() { + return []; + } + + createButton(name, callback) { + var button = document.createElement("button"); + button.innerText = name; + button.addEventListener("click", callback); + return button; + } + + createLeftButton(name, callback) { + var button = this.createButton(name, callback); + button.style.cssFloat = "left"; + button.style.marginRight = "4px"; + return button; + } + + createRightButton(name, callback) { + var button = this.createButton(name, callback); + button.style.cssFloat = "right"; + button.style.marginLeft = "4px"; + return button; + } + + createLeftSlider(self, name, callback) { + const divElement = document.createElement('div'); + divElement.id = "sam-confidence-slider"; + divElement.style.cssFloat = "left"; + divElement.style.fontFamily = "sans-serif"; + divElement.style.marginRight = "4px"; + divElement.style.color = "var(--input-text)"; + divElement.style.backgroundColor = "var(--comfy-input-bg)"; + divElement.style.borderRadius = "8px"; + divElement.style.borderColor = "var(--border-color)"; + divElement.style.borderStyle = "solid"; + divElement.style.fontSize = "15px"; + divElement.style.height = "21px"; + divElement.style.padding = "1px 6px"; + divElement.style.display = "flex"; + divElement.style.position = "relative"; + divElement.style.top = "2px"; + self.confidence_slider_input = document.createElement('input'); + self.confidence_slider_input.setAttribute('type', 'range'); + self.confidence_slider_input.setAttribute('min', '0'); + self.confidence_slider_input.setAttribute('max', '100'); + self.confidence_slider_input.setAttribute('value', '70'); + const labelElement = document.createElement("label"); + labelElement.textContent = name; + + divElement.appendChild(labelElement); + divElement.appendChild(self.confidence_slider_input); + + self.confidence_slider_input.addEventListener("change", callback); + + return divElement; + } + + async detect_and_invalidate_mask_canvas(self) { + const mask_img = await self.detect(self); + + const canvas = self.maskCtx.canvas; + const ctx = self.maskCtx; + + ctx.clearRect(0, 0, canvas.width, canvas.height); + + await new Promise((resolve, reject) => { + self.mask_image = new Image(); + self.mask_image.onload = function() { + ctx.drawImage(self.mask_image, 0, 0, canvas.width, canvas.height); + resolve(); + }; + self.mask_image.onerror = reject; + self.mask_image.src = mask_img.src; + }); + } + + setlayout(imgCanvas, maskCanvas, pointsCanvas) { + const self = this; + + // If it is specified as relative, using it only as a hidden placeholder for padding is recommended + // to prevent anomalies where it exceeds a certain size and goes outside of the window. + var placeholder = document.createElement("div"); + placeholder.style.position = "relative"; + placeholder.style.height = "50px"; + + var bottom_panel = document.createElement("div"); + bottom_panel.style.position = "absolute"; + bottom_panel.style.bottom = "0px"; + bottom_panel.style.left = "20px"; + bottom_panel.style.right = "20px"; + bottom_panel.style.height = "50px"; + + var brush = document.createElement("div"); + brush.id = "sam-brush"; + brush.style.backgroundColor = "blue"; + brush.style.outline = "2px solid pink"; + brush.style.borderRadius = "50%"; + brush.style.MozBorderRadius = "50%"; + brush.style.WebkitBorderRadius = "50%"; + brush.style.position = "absolute"; + brush.style.zIndex = 100; + brush.style.pointerEvents = "none"; + this.brush = brush; + this.element.appendChild(imgCanvas); + this.element.appendChild(maskCanvas); + this.element.appendChild(pointsCanvas); + this.element.appendChild(placeholder); // must below z-index than bottom_panel to avoid covering button + this.element.appendChild(bottom_panel); + document.body.appendChild(brush); + this.brush_size = 5; + + var confidence_slider = this.createLeftSlider(self, "Confidence", (event) => { + self.confidence = event.target.value; + }); + + var clearButton = this.createLeftButton("Clear", () => { + self.maskCtx.clearRect(0, 0, self.maskCanvas.width, self.maskCanvas.height); + self.pointsCtx.clearRect(0, 0, self.pointsCanvas.width, self.pointsCanvas.height); + + self.prompt_points = []; + + self.invalidatePointsCanvas(self); + }); + + var detectButton = this.createLeftButton("Detect", () => self.detect_and_invalidate_mask_canvas(self)); + + var cancelButton = this.createRightButton("Cancel", () => { + document.removeEventListener("mouseup", SAM3EditorDialog.handleMouseUp); + document.removeEventListener("keydown", SAM3EditorDialog.handleKeyDown); + self.close(); + }); + + self.saveButton = this.createRightButton("Save", () => { + document.removeEventListener("mouseup", SAM3EditorDialog.handleMouseUp); + document.removeEventListener("keydown", SAM3EditorDialog.handleKeyDown); + self.save(self); + }); + + var undoButton = this.createLeftButton("Undo", () => { + if(self.prompt_points.length > 0) { + self.prompt_points.pop(); + self.pointsCtx.clearRect(0, 0, self.pointsCanvas.width, self.pointsCanvas.height); + self.invalidatePointsCanvas(self); + } + }); + + bottom_panel.appendChild(clearButton); + bottom_panel.appendChild(detectButton); + bottom_panel.appendChild(self.saveButton); + bottom_panel.appendChild(cancelButton); + bottom_panel.appendChild(confidence_slider); + bottom_panel.appendChild(undoButton); + + imgCanvas.style.position = "relative"; + imgCanvas.style.top = "200"; + imgCanvas.style.left = "0"; + + maskCanvas.style.position = "absolute"; + maskCanvas.style.opacity = 0.5; + pointsCanvas.style.position = "absolute"; + } + + show() { + this.mask_image = null; + self.prompt_points = []; + + this.message_box = $el("p", ["Please wait a moment while the SAM3 model and the image are being loaded."]); + this.element.appendChild(this.message_box); + + if(self.imgCtx) { + self.imgCtx.clearRect(0, 0, self.imageCanvas.width, self.imageCanvas.height); + } + + const target_image_path = ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src; + this.load_sam(target_image_path); + + if(!this.is_layout_created) { + // layout + const imgCanvas = document.createElement('canvas'); + const maskCanvas = document.createElement('canvas'); + const pointsCanvas = document.createElement('canvas'); + + imgCanvas.id = "imageCanvas"; + maskCanvas.id = "samEditorMaskCanvas"; + pointsCanvas.id = "pointsCanvas"; + + this.setlayout(imgCanvas, maskCanvas, pointsCanvas); + + // prepare content + this.imgCanvas = imgCanvas; + this.maskCanvas = maskCanvas; + this.pointsCanvas = pointsCanvas; + this.maskCtx = maskCanvas.getContext('2d'); + this.pointsCtx = pointsCanvas.getContext('2d'); + + this.is_layout_created = true; + + // replacement of onClose hook since close is not real close + const self = this; + const observer = new MutationObserver(function(mutations) { + mutations.forEach(function(mutation) { + if (mutation.type === 'attributes' && mutation.attributeName === 'style') { + if(self.last_display_style && self.last_display_style != 'none' && self.element.style.display == 'none') { + ComfyApp.onClipspaceEditorClosed(); + } + + self.last_display_style = self.element.style.display; + } + }); + }); + + const config = { attributes: true }; + observer.observe(this.element, config); + } + + this.setImages(target_image_path, this.imgCanvas, this.pointsCanvas); + + if(ComfyApp.clipspace_return_node) { + this.saveButton.innerText = "Save to node"; + } + else { + this.saveButton.innerText = "Save"; + } + this.saveButton.disabled = true; + + this.element.style.display = "block"; + this.element.style.zIndex = 8888; // NOTE: alert dialog must be high priority. + } + + updateBrushPreview(self, event) { + event.preventDefault(); + + const centerX = event.pageX; + const centerY = event.pageY; + + const brush = self.brush; + + brush.style.width = self.brush_size * 2 + "px"; + brush.style.height = self.brush_size * 2 + "px"; + brush.style.left = (centerX - self.brush_size) + "px"; + brush.style.top = (centerY - self.brush_size) + "px"; + } + + setImages(target_image_path, imgCanvas, pointsCanvas) { + const imgCtx = imgCanvas.getContext('2d'); + const maskCtx = this.maskCtx; + const maskCanvas = this.maskCanvas; + + const self = this; + + // image load + const orig_image = new Image(); + window.addEventListener("resize", () => { + // repositioning + imgCanvas.width = window.innerWidth - 250; + imgCanvas.height = window.innerHeight - 200; + + // redraw image + let drawWidth = orig_image.width; + let drawHeight = orig_image.height; + + if (orig_image.width > imgCanvas.width) { + drawWidth = imgCanvas.width; + drawHeight = (drawWidth / orig_image.width) * orig_image.height; + } + + if (drawHeight > imgCanvas.height) { + drawHeight = imgCanvas.height; + drawWidth = (drawHeight / orig_image.height) * orig_image.width; + } + + imgCtx.drawImage(orig_image, 0, 0, drawWidth, drawHeight); + + // update mask + let w = (drawWidth * imgCanvas.clientWidth/imgCanvas.width) + "px"; + let h = (drawHeight * imgCanvas.clientHeight/imgCanvas.height) + "px"; + + pointsCanvas.width = drawWidth * imgCanvas.clientWidth/imgCanvas.width; + pointsCanvas.height = drawHeight * imgCanvas.clientHeight/imgCanvas.height; + pointsCanvas.style.top = imgCanvas.offsetTop + "px"; + pointsCanvas.style.left = imgCanvas.offsetLeft + "px"; + + maskCanvas.width = pointsCanvas.width; + maskCanvas.height = pointsCanvas.height; + maskCanvas.style.top = imgCanvas.offsetTop + "px"; + maskCanvas.style.left = imgCanvas.offsetLeft + "px"; + + self.invalidateMaskCanvas(self); + self.invalidatePointsCanvas(self); + }); + + // original image load + orig_image.onload = () => self.onLoaded(self); + const rgb_url = new URL(target_image_path); + rgb_url.searchParams.delete('channel'); + rgb_url.searchParams.set('channel', 'rgb'); + orig_image.src = rgb_url; + self.image = orig_image; + } + + onLoaded(self) { + if(self.message_box) { + self.element.removeChild(self.message_box); + self.message_box = null; + } + + window.dispatchEvent(new Event('resize')); + + self.setEventHandler(pointsCanvas); + self.saveButton.disabled = false; + } + + setEventHandler(targetCanvas) { + targetCanvas.addEventListener("contextmenu", (event) => { + event.preventDefault(); + }); + + const self = this; + targetCanvas.addEventListener('pointermove', (event) => this.updateBrushPreview(self,event)); + targetCanvas.addEventListener('pointerdown', (event) => this.handlePointerDown(self,event)); + targetCanvas.addEventListener('pointerover', (event) => { this.brush.style.display = "block"; }); + targetCanvas.addEventListener('pointerleave', (event) => { this.brush.style.display = "none"; }); + document.addEventListener('keydown', SAM3EditorDialog.handleKeyDown); + } + + static handleKeyDown(event) { + const self = SAM3EditorDialog.instance; + if (event.key === '=') { // positive + brush.style.backgroundColor = "blue"; + brush.style.outline = "2px solid pink"; + self.is_positive_mode = true; + } else if (event.key === '-') { // negative + brush.style.backgroundColor = "red"; + brush.style.outline = "2px solid skyblue"; + self.is_positive_mode = false; + } + } + + is_positive_mode = true; + prompt_points = []; + confidence = 70; + + invalidatePointsCanvas(self) { + const ctx = self.pointsCtx; + + for (const i in self.prompt_points) { + const [is_positive, x, y] = self.prompt_points[i]; + + const scaledX = x * ctx.canvas.width / self.image.width; + const scaledY = y * ctx.canvas.height / self.image.height; + + if(is_positive) + ctx.fillStyle = "blue"; + else + ctx.fillStyle = "red"; + ctx.beginPath(); + ctx.arc(scaledX, scaledY, 3, 0, 3 * Math.PI); + ctx.fill(); + } + } + + invalidateMaskCanvas(self) { + if(self.mask_image) { + self.maskCtx.clearRect(0, 0, self.maskCanvas.width, self.maskCanvas.height); + self.maskCtx.drawImage(self.mask_image, 0, 0, self.maskCanvas.width, self.maskCanvas.height); + } + } + + async load_sam(url) { + const parsedUrl = new URL(url); + const searchParams = new URLSearchParams(parsedUrl.search); + + const filename = searchParams.get("filename") || ""; + const fileType = searchParams.get("type") || ""; + const subfolder = searchParams.get("subfolder") || ""; + + const data = { + sam_model_name: "auto", + filename: filename, + type: fileType, + subfolder: subfolder + }; + + api.fetchApi('/sam3/prepare', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(data) + }); + } + + async detect(self) { + const positive_points = []; + const negative_points = []; + + for(const i in self.prompt_points) { + const [is_positive, x, y] = self.prompt_points[i]; + const point = [x,y]; + if(is_positive) { + positive_points.push(point); + } + else + negative_points.push(point); + } + + const data = { + positive_points: positive_points, + negative_points: negative_points, + threshold: self.confidence/100 + }; + + const response = await api.fetchApi('/sam3/detect', { + method: 'POST', + headers: { 'Content-Type': 'image/png' }, + body: JSON.stringify(data) + }); + + const blob = await response.blob(); + const url = URL.createObjectURL(blob); + + return new Promise((resolve, reject) => { + const image = new Image(); + image.onload = () => resolve(image); + image.onerror = reject; + image.src = url; + }); + } + + handlePointerDown(self, event) { + if ([0, 2, 5].includes(event.button)) { + event.preventDefault(); + const x = event.offsetX || event.targetTouches[0].clientX - maskRect.left; + const y = event.offsetY || event.targetTouches[0].clientY - maskRect.top; + + const originalX = x * self.image.width / self.pointsCanvas.clientWidth; + const originalY = y * self.image.height / self.pointsCanvas.clientHeight; + + var point = null; + if (event.button == 0) { + // positive + point = [true, originalX, originalY]; + } else { + // negative + point = [false, originalX, originalY]; + } + + self.prompt_points.push(point); + + self.invalidatePointsCanvas(self); + } + } + + async save(self) { + if(!self.mask_image) { + this.close(); + return; + } + + const save_canvas = document.createElement('canvas'); + + const save_ctx = save_canvas.getContext('2d', {willReadFrequently:true}); + save_canvas.width = self.mask_image.width; + save_canvas.height = self.mask_image.height; + + save_ctx.drawImage(self.mask_image, 0, 0, save_canvas.width, save_canvas.height); + + const save_data = save_ctx.getImageData(0, 0, save_canvas.width, save_canvas.height); + + // refine mask image + for (let i = 0; i < save_data.data.length; i += 4) { + if(save_data.data[i]) { + save_data.data[i+3] = 0; + } + else { + save_data.data[i+3] = 255; + } + + save_data.data[i] = 0; + save_data.data[i+1] = 0; + save_data.data[i+2] = 0; + } + + save_ctx.globalCompositeOperation = 'source-over'; + save_ctx.putImageData(save_data, 0, 0); + + const formData = new FormData(); + const filename = "clipspace-mask-" + performance.now() + ".png"; + + const item = + { + "filename": filename, + "subfolder": "", + "type": "temp", + }; + + if(ComfyApp.clipspace.images) + ComfyApp.clipspace.images[0] = item; + + if(ComfyApp.clipspace.widgets) { + const index = ComfyApp.clipspace.widgets.findIndex(obj => obj.name === 'image'); + + if(index >= 0) + ComfyApp.clipspace.widgets[index].value = `${filename} [temp]`; + } + + const dataURL = save_canvas.toDataURL(); + const blob = dataURLToBlob(dataURL); + + let original_url = new URL(this.image.src); + + const original_ref = { filename: original_url.searchParams.get('filename') }; + + let original_subfolder = original_url.searchParams.get("subfolder"); + if(original_subfolder) + original_ref.subfolder = original_subfolder; + + let original_type = original_url.searchParams.get("type"); + if(original_type) + original_ref.type = original_type; + + formData.append('image', blob, filename); + formData.append('original_ref', JSON.stringify(original_ref)); + formData.append('type', "temp"); + + await uploadMask(item, formData); + ComfyApp.onClipspaceEditorSave(); + this.close(); + } +} + +app.registerExtension({ + name: "Comfy.SAM3.Editor", + init(app) { + const callback = + function () { + let dlg = SAM3EditorDialog.getInstance(); + dlg.show(); + }; + + const context_predicate = () => ComfyApp.clipspace && ComfyApp.clipspace.imgs && ComfyApp.clipspace.imgs.length > 0 + ClipspaceDialog.registerButton("SAM3 Detector", context_predicate, callback); + }, + + async beforeRegisterNodeDef(nodeType, nodeData, app) { + if (Array.isArray(nodeData.output) && (nodeData.output.includes("MASK") || nodeData.output.includes("IMAGE"))) { + addMenuHandler(nodeType, function (_, options) { + options.unshift({ + content: "Open in SAM3 Detector", + callback: () => { + ComfyApp.copyToClipspace(this); + ComfyApp.clipspace_return_node = this; + + let dlg = SAM3EditorDialog.getInstance(); + dlg.show(); + }, + }); + }); + } + } +}); diff --git a/web/sam3_points_widget.js b/web/sam3_points_widget.js new file mode 100644 index 0000000..25bb862 --- /dev/null +++ b/web/sam3_points_widget.js @@ -0,0 +1,423 @@ +/** + * SAM3 Simple Point Collector + * Uses plain HTML5 Canvas instead of Protovis for better compatibility + * Version: 2025-01-20-v8-SERIALIZE-FIX + */ + +import { app } from "../../scripts/app.js"; +import { ComfyWidgets } from "../../scripts/widgets.js"; + +console.log("[SAM3] ===== VERSION 8 - SERIALIZATION ENABLED ====="); + +// Helper function to properly hide widgets (enhanced for complete hiding) +function hideWidgetForGood(node, widget, suffix = '') { + if (!widget) return; + + // Save original properties + widget.origType = widget.type; + widget.origComputeSize = widget.computeSize; + widget.origSerializeValue = widget.serializeValue; + + // Multiple hiding approaches to ensure widget is fully hidden + widget.computeSize = () => [0, -4]; // -4 compensates for litegraph's automatic widget gap + widget.type = "converted-widget" + suffix; + widget.hidden = true; // Mark as hidden + + // IMPORTANT: Keep serialization enabled so values are sent to backend + // (We just hide it visually, but it still needs to send data) + + // Make the widget completely invisible in the DOM if it has element + if (widget.element) { + widget.element.style.display = "none"; + widget.element.style.visibility = "hidden"; + } + + // Handle linked widgets recursively + if (widget.linkedWidgets) { + for (const w of widget.linkedWidgets) { + hideWidgetForGood(node, w, ':' + widget.name); + } + } +} + +app.registerExtension({ + name: "Comfy.SAM3.SimplePointCollector", + + async beforeRegisterNodeDef(nodeType, nodeData, app) { + console.log("[SAM3] beforeRegisterNodeDef called for:", nodeData.name); + + if (nodeData.name === "SAM3PointCollector") { + console.log("[SAM3] Registering SAM3PointCollector node"); + const onNodeCreated = nodeType.prototype.onNodeCreated; + + nodeType.prototype.onNodeCreated = function () { + console.log("[SAM3] onNodeCreated called for SAM3PointCollector"); + + // Call original onNodeCreated FIRST to create all widgets + const result = onNodeCreated?.apply(this, arguments); + + console.log("[SAM3] Widgets after creation:", this.widgets?.map(w => w.name)); + + console.log("[SAM3] Creating canvas container"); + // Create canvas container - dynamically sized based on node height + const container = document.createElement("div"); + container.style.cssText = "position: relative; width: 100%; background: #222; overflow: hidden; box-sizing: border-box; margin: 0; padding: 0; display: flex; align-items: center; justify-content: center;"; + + // Create info/button bar + const infoBar = document.createElement("div"); + infoBar.style.cssText = "position: absolute; top: 5px; left: 5px; right: 5px; z-index: 10; display: flex; justify-content: space-between; align-items: center;"; + container.appendChild(infoBar); + + // Create points counter + const pointsCounter = document.createElement("div"); + pointsCounter.style.cssText = "padding: 5px 10px; background: rgba(0,0,0,0.7); color: #fff; border-radius: 3px; font-size: 12px; font-family: monospace;"; + pointsCounter.textContent = "Points: 0 pos, 0 neg"; + infoBar.appendChild(pointsCounter); + + // Create clear button + const clearButton = document.createElement("button"); + clearButton.textContent = "Clear All"; + clearButton.style.cssText = "padding: 5px 10px; background: #d44; color: #fff; border: 1px solid #a22; border-radius: 3px; cursor: pointer; font-size: 12px; font-weight: bold;"; + clearButton.onmouseover = () => clearButton.style.background = "#e55"; + clearButton.onmouseout = () => clearButton.style.background = "#d44"; + infoBar.appendChild(clearButton); + + // Create canvas for image and points + const canvas = document.createElement("canvas"); + canvas.width = 512; + canvas.height = 512; + // Use max-width and max-height instead of width/height 100% to prevent overflow + canvas.style.cssText = "display: block; max-width: 100%; max-height: 100%; object-fit: contain; cursor: crosshair; margin: 0 auto;"; + container.appendChild(canvas); + + const ctx = canvas.getContext("2d"); + console.log("[SAM3] Canvas created:", canvas); + + // Store state + this.canvasWidget = { + canvas: canvas, + ctx: ctx, + container: container, + image: null, + positivePoints: [], + negativePoints: [], + hoveredPoint: null, + pointsCounter: pointsCounter + }; + + // Add as DOM widget + console.log("[SAM3] Adding DOM widget via addDOMWidget"); + const widget = this.addDOMWidget("canvas", "customCanvas", container); + console.log("[SAM3] addDOMWidget returned:", widget); + + // Store widget reference for updates + this.canvasWidget.domWidget = widget; + + // Make widget dynamically sized - override computeSize + widget.computeSize = (width) => { + // Widget height = node height - title bar/padding (approx 80px) + const nodeHeight = this.size ? this.size[1] : 480; + const widgetHeight = Math.max(200, nodeHeight - 80); + return [width, widgetHeight]; + }; + + // Clear button handler + clearButton.addEventListener("click", (e) => { + e.preventDefault(); + e.stopPropagation(); + console.log("[SAM3] Clearing all points"); + this.canvasWidget.positivePoints = []; + this.canvasWidget.negativePoints = []; + this.updatePoints(); + this.redrawCanvas(); + }); + + // Hide the string storage widgets - multiple approaches + console.log("[SAM3] Attempting to hide widgets..."); + console.log("[SAM3] Widgets before hiding:", this.widgets.map(w => w.name)); + + const coordsWidget = this.widgets.find(w => w.name === "coordinates"); + const negCoordsWidget = this.widgets.find(w => w.name === "neg_coordinates"); + const storeWidget = this.widgets.find(w => w.name === "points_store"); + + console.log("[SAM3] Found widgets to hide:", { coordsWidget, negCoordsWidget, storeWidget }); + + // Initialize default values BEFORE hiding + if (coordsWidget) { + coordsWidget.value = coordsWidget.value || "[]"; + } + if (negCoordsWidget) { + negCoordsWidget.value = negCoordsWidget.value || "[]"; + } + if (storeWidget) { + storeWidget.value = storeWidget.value || "{}"; + } + + // Store references before hiding + this._hiddenWidgets = { + coordinates: coordsWidget, + neg_coordinates: negCoordsWidget, + points_store: storeWidget + }; + + // Apply hiding + if (coordsWidget) { + hideWidgetForGood(this, coordsWidget); + console.log("[SAM3] coordinates - type:", coordsWidget.type, "hidden:", coordsWidget.hidden, "value:", coordsWidget.value); + } + if (negCoordsWidget) { + hideWidgetForGood(this, negCoordsWidget); + console.log("[SAM3] neg_coordinates - type:", negCoordsWidget.type, "hidden:", negCoordsWidget.hidden, "value:", negCoordsWidget.value); + } + if (storeWidget) { + hideWidgetForGood(this, storeWidget); + console.log("[SAM3] points_store - type:", storeWidget.type, "hidden:", storeWidget.hidden, "value:", storeWidget.value); + } + + // CRITICAL FIX: Override onDrawForeground to skip rendering hidden widgets + const originalDrawForeground = this.onDrawForeground; + this.onDrawForeground = function(ctx) { + // Temporarily hide converted widgets from rendering + const hiddenWidgets = this.widgets.filter(w => w.type?.includes("converted-widget")); + const originalTypes = hiddenWidgets.map(w => w.type); + + // Temporarily set to null to prevent rendering + hiddenWidgets.forEach(w => w.type = null); + + // Call original draw + if (originalDrawForeground) { + originalDrawForeground.apply(this, arguments); + } + + // Restore types + hiddenWidgets.forEach((w, i) => w.type = originalTypes[i]); + + // Update container height based on current node size + const containerHeight = Math.max(200, this.size[1] - 80); + if (container.style.height !== containerHeight + "px") { + container.style.height = containerHeight + "px"; + } + }; + + console.log("[SAM3] Widgets after hiding:", this.widgets.map(w => `${w.name}(${w.type})`)); + console.log("[SAM3] All widgets processing complete"); + + // Mouse event handlers + canvas.addEventListener("click", (e) => { + const rect = canvas.getBoundingClientRect(); + // Calculate coordinates relative to the actual image dimensions + // The canvas might be scaled, so we need to map from display coords to image coords + const x = ((e.clientX - rect.left) / rect.width) * canvas.width; + const y = ((e.clientY - rect.top) / rect.height) * canvas.height; + console.log(`[SAM3] Click at canvas coords: (${x.toFixed(1)}, ${y.toFixed(1)}), canvas size: ${canvas.width}x${canvas.height}`); + + // Check if clicking existing point to delete + const clickedPoint = this.findPointAt(x, y); + if (clickedPoint && e.button === 2) { + // Right-click on existing point = delete + if (clickedPoint.type === 'positive') { + this.canvasWidget.positivePoints = this.canvasWidget.positivePoints.filter(p => p !== clickedPoint.point); + } else { + this.canvasWidget.negativePoints = this.canvasWidget.negativePoints.filter(p => p !== clickedPoint.point); + } + } else { + // Add new point + if (e.shiftKey || e.button === 2) { + // Negative point + this.canvasWidget.negativePoints.push({x, y}); + } else { + // Positive point + this.canvasWidget.positivePoints.push({x, y}); + } + } + + this.updatePoints(); + this.redrawCanvas(); + }); + + canvas.addEventListener("contextmenu", (e) => { + e.preventDefault(); + // Trigger click with right button flag + canvas.dispatchEvent(new MouseEvent('click', { + button: 2, + clientX: e.clientX, + clientY: e.clientY, + shiftKey: e.shiftKey + })); + }); + + canvas.addEventListener("mousemove", (e) => { + const rect = canvas.getBoundingClientRect(); + const x = ((e.clientX - rect.left) / rect.width) * canvas.width; + const y = ((e.clientY - rect.top) / rect.height) * canvas.height; + + const hovered = this.findPointAt(x, y); + if (hovered !== this.canvasWidget.hoveredPoint) { + this.canvasWidget.hoveredPoint = hovered; + this.redrawCanvas(); + } + }); + + // Handle image input changes + this.onExecuted = (message) => { + console.log("[SAM3] onExecuted called with message:", message); + if (message.bg_image && message.bg_image[0]) { + const img = new Image(); + img.onload = () => { + console.log(`[SAM3] Image loaded: ${img.width}x${img.height}`); + this.canvasWidget.image = img; + canvas.width = img.width; + canvas.height = img.height; + console.log(`[SAM3] Canvas resized to: ${canvas.width}x${canvas.height}`); + this.redrawCanvas(); + }; + img.src = "data:image/jpeg;base64," + message.bg_image[0]; + } + }; + + // Update container height dynamically when node size changes + const originalOnResize = this.onResize; + this.onResize = function(size) { + if (originalOnResize) { + originalOnResize.apply(this, arguments); + } + // Update container to match widget size + const containerHeight = Math.max(200, size[1] - 80); + container.style.height = containerHeight + "px"; + console.log(`[SAM3] Node resized to: ${size[0]}x${size[1]}, container height: ${containerHeight}px`); + }; + + // Also update on draw to handle any size changes + const originalOnDrawForeground = this.onDrawForeground; + this.onDrawForeground = function(ctx) { + if (originalOnDrawForeground) { + originalOnDrawForeground.apply(this, arguments); + } + // Update container height based on current node size + const containerHeight = Math.max(200, this.size[1] - 80); + if (container.style.height !== containerHeight + "px") { + container.style.height = containerHeight + "px"; + } + }; + + // Draw initial placeholder + console.log("[SAM3] Drawing initial placeholder"); + this.redrawCanvas(); + + // Set initial node size + const nodeWidth = Math.max(400, this.size[0] || 400); + const nodeHeight = 480; // Initial height: canvas (400) + space (80) + this.setSize([nodeWidth, nodeHeight]); + + // Set initial container height + container.style.height = "400px"; + + console.log("[SAM3] Node size set to:", [nodeWidth, nodeHeight]); + console.log("[SAM3] onNodeCreated complete"); + return result; + }; + + // Helper: Find point at coordinates + nodeType.prototype.findPointAt = function(x, y) { + const threshold = 10; + + for (const point of this.canvasWidget.positivePoints) { + if (Math.abs(point.x - x) < threshold && Math.abs(point.y - y) < threshold) { + return {type: 'positive', point}; + } + } + + for (const point of this.canvasWidget.negativePoints) { + if (Math.abs(point.x - x) < threshold && Math.abs(point.y - y) < threshold) { + return {type: 'negative', point}; + } + } + + return null; + }; + + // Helper: Update widget values + nodeType.prototype.updatePoints = function() { + // Use stored hidden widget references + const coordsWidget = this._hiddenWidgets?.coordinates || this.widgets.find(w => w.name === "coordinates"); + const negCoordsWidget = this._hiddenWidgets?.neg_coordinates || this.widgets.find(w => w.name === "neg_coordinates"); + const storeWidget = this._hiddenWidgets?.points_store || this.widgets.find(w => w.name === "points_store"); + + if (coordsWidget) { + coordsWidget.value = JSON.stringify(this.canvasWidget.positivePoints); + } + if (negCoordsWidget) { + negCoordsWidget.value = JSON.stringify(this.canvasWidget.negativePoints); + } + if (storeWidget) { + storeWidget.value = JSON.stringify({ + positive: this.canvasWidget.positivePoints, + negative: this.canvasWidget.negativePoints + }); + } + + // Update points counter display + const posCount = this.canvasWidget.positivePoints.length; + const negCount = this.canvasWidget.negativePoints.length; + this.canvasWidget.pointsCounter.textContent = `Points: ${posCount} pos, ${negCount} neg`; + }; + + // Helper: Redraw canvas + nodeType.prototype.redrawCanvas = function() { + const {canvas, ctx, image, positivePoints, negativePoints, hoveredPoint} = this.canvasWidget; + + // Clear + ctx.clearRect(0, 0, canvas.width, canvas.height); + + // Draw image if available + if (image) { + ctx.drawImage(image, 0, 0, canvas.width, canvas.height); + } else { + // Placeholder + ctx.fillStyle = "#333"; + ctx.fillRect(0, 0, canvas.width, canvas.height); + ctx.fillStyle = "#666"; + ctx.font = "16px sans-serif"; + ctx.textAlign = "center"; + ctx.fillText("Click to add points", canvas.width / 2, canvas.height / 2); + ctx.fillText("Left-click: Positive (green)", canvas.width / 2, canvas.height / 2 + 25); + ctx.fillText("Shift/Right-click: Negative (red)", canvas.width / 2, canvas.height / 2 + 50); + } + + // Draw canvas dimensions overlay (helpful for debugging) + if (image) { + ctx.fillStyle = "rgba(0,0,0,0.7)"; + ctx.fillRect(5, canvas.height - 25, 150, 20); + ctx.fillStyle = "#0f0"; + ctx.font = "12px monospace"; + ctx.textAlign = "left"; + ctx.fillText(`Image: ${canvas.width}x${canvas.height}`, 10, canvas.height - 10); + } + + // Draw positive points (green) + ctx.strokeStyle = "#0f0"; + ctx.fillStyle = "#0f0"; + for (const point of positivePoints) { + const isHovered = hoveredPoint?.point === point; + ctx.beginPath(); + ctx.arc(point.x, point.y, isHovered ? 8 : 6, 0, 2 * Math.PI); + ctx.fill(); + ctx.lineWidth = 2; + ctx.stroke(); + } + + // Draw negative points (red) + ctx.strokeStyle = "#f00"; + ctx.fillStyle = "#f00"; + for (const point of negativePoints) { + const isHovered = hoveredPoint?.point === point; + ctx.beginPath(); + ctx.arc(point.x, point.y, isHovered ? 8 : 6, 0, 2 * Math.PI); + ctx.fill(); + ctx.lineWidth = 2; + ctx.stroke(); + } + }; + } + } +}); diff --git a/workflows/example_segmentation_points.json b/workflows/example_segmentation_points.json new file mode 100644 index 0000000..c38d211 --- /dev/null +++ b/workflows/example_segmentation_points.json @@ -0,0 +1,395 @@ +{ + "id": "0b04cdef-0fe9-494a-baa9-ca11511022b9", + "revision": 0, + "last_node_id": 9, + "last_link_id": 8, + "nodes": [ + { + "id": 7, + "type": "MaskPreview", + "pos": [ + 1285.1917567959822, + -164.2376998289703 + ], + "size": [ + 140, + 246 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 5 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.70", + "Node name for S&R": "MaskPreview" + }, + "widgets_values": [] + }, + { + "id": 1, + "type": "LoadSAM3Model", + "pos": [ + 530.73828125, + 128.45703125 + ], + "size": [ + 270, + 106 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "sam3_model", + "type": "SAM3_MODEL", + "links": [ + 1 + ] + } + ], + "properties": { + "aux_id": "PozzettiAndrea/ComfyUI-SAM3", + "ver": "307654744fa45de98017f92710d4a9cc1b4d63de", + "Node name for S&R": "LoadSAM3Model" + }, + "widgets_values": [ + "auto", + "", + "YOUR_HF_TOKEN_HERE" + ] + }, + { + "id": 4, + "type": "LoadImage", + "pos": [ + 535.57421875, + 284.2890625 + ], + "size": [ + 274.080078125, + 314 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 2, + 7 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.70", + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "example_image.jpg", + "image" + ] + }, + { + "id": 5, + "type": "PreviewImage", + "pos": [ + 1710.4529605215105, + 147.12305834352858 + ], + "size": [ + 140, + 246 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 3 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.70", + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 6, + "type": "PreviewAny", + "pos": [ + 1578.2128366336826, + 509.43098550417255 + ], + "size": [ + 233.69546594584722, + 164.65461903779413 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 4 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.70", + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [] + }, + { + "id": 2, + "type": "SAM3Segmentation", + "pos": [ + 1413.3547712239274, + 213.48039870709997 + ], + "size": [ + 278.08203125, + 266 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "sam3_model", + "type": "SAM3_MODEL", + "link": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 2 + }, + { + "name": "positive_boxes", + "shape": 7, + "type": "SAM3_BOXES_PROMPT", + "link": null + }, + { + "name": "negative_boxes", + "shape": 7, + "type": "SAM3_BOXES_PROMPT", + "link": null + }, + { + "name": "positive_points", + "shape": 7, + "type": "SAM3_POINTS_PROMPT", + "link": 8 + }, + { + "name": "negative_points", + "shape": 7, + "type": "SAM3_POINTS_PROMPT", + "link": null + }, + { + "name": "mask_prompt", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "boxes_prompt", + "shape": 7, + "type": "SAM3_BOXES_PROMPT", + "link": null + }, + { + "name": "points_prompt", + "shape": 7, + "type": "SAM3_POINTS_PROMPT", + "link": null + } + ], + "outputs": [ + { + "name": "masks", + "type": "MASK", + "links": [ + 5 + ] + }, + { + "name": "visualization", + "type": "IMAGE", + "links": [ + 3 + ] + }, + { + "name": "boxes", + "type": "STRING", + "links": [ + 4 + ] + }, + { + "name": "scores", + "type": "STRING", + "links": null + } + ], + "properties": { + "aux_id": "PozzettiAndrea/ComfyUI-SAM3", + "ver": "307654744fa45de98017f92710d4a9cc1b4d63de", + "Node name for S&R": "SAM3Segmentation" + }, + "widgets_values": [ + 0.21, + "person", + -1 + ] + }, + { + "id": 9, + "type": "SAM3PointCollector", + "pos": [ + 929.3727386768797, + 184.97597369457938 + ], + "size": [ + 400, + 480 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 7 + } + ], + "outputs": [ + { + "name": "positive_points", + "type": "SAM3_POINTS_PROMPT", + "links": [ + 8 + ] + }, + { + "name": "negative_points", + "type": "SAM3_POINTS_PROMPT", + "links": null + } + ], + "properties": { + "aux_id": "PozzettiAndrea/ComfyUI-SAM3", + "ver": "64c86b0bf97593007c32df5dbc544a566d9efd9f", + "Node name for S&R": "SAM3PointCollector" + }, + "widgets_values": [ + "{}", + "[]", + "[]", + "" + ] + } + ], + "links": [ + [ + 1, + 1, + 0, + 2, + 0, + "SAM3_MODEL" + ], + [ + 2, + 4, + 0, + 2, + 1, + "IMAGE" + ], + [ + 3, + 2, + 1, + 5, + 0, + "IMAGE" + ], + [ + 4, + 2, + 2, + 6, + 0, + "*" + ], + [ + 5, + 2, + 0, + 7, + 0, + "MASK" + ], + [ + 7, + 4, + 0, + 9, + 0, + "IMAGE" + ], + [ + 8, + 9, + 0, + 2, + 4, + "SAM3_POINTS_PROMPT" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 1.0152559799477059, + "offset": [ + -484.16482173037514, + 102.63622043900284 + ] + }, + "frontendVersion": "1.28.8" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/workflows/video_tracking.json b/workflows/video_tracking.json new file mode 100644 index 0000000..05044d1 --- /dev/null +++ b/workflows/video_tracking.json @@ -0,0 +1,226 @@ +{ + "last_node_id": 8, + "last_link_id": 12, + "nodes": [ + { + "id": 1, + "type": "SAM3VideoModelLoader", + "pos": [100, 100], + "size": [400, 200], + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "video_model", + "type": "SAM3_VIDEO_MODEL", + "links": [1] + } + ], + "properties": {}, + "widgets_values": [ + "", + "", + "", + false, + "0" + ], + "title": "SAM3 Load Video Model" + }, + { + "id": 2, + "type": "VHS_LoadVideo", + "pos": [100, 350], + "size": [300, 150], + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [2] + } + ], + "properties": {}, + "widgets_values": [], + "title": "Load Video (VHS)" + }, + { + "id": 3, + "type": "SAM3InitVideoSession", + "pos": [550, 100], + "size": [350, 150], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "video_model", + "type": "SAM3_VIDEO_MODEL", + "link": 1 + }, + { + "name": "video_frames", + "type": "IMAGE", + "link": 2 + } + ], + "outputs": [ + { + "name": "session", + "type": "SAM3_VIDEO_SESSION", + "links": [3, 4] + }, + { + "name": "session_id", + "type": "STRING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [""], + "title": "SAM3 Init Video Session" + }, + { + "id": 4, + "type": "SAM3AddVideoPrompt", + "pos": [950, 100], + "size": [400, 250], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "session", + "type": "SAM3_VIDEO_SESSION", + "link": 3 + } + ], + "outputs": [ + { + "name": "session", + "type": "SAM3_VIDEO_SESSION", + "links": [5] + } + ], + "properties": {}, + "widgets_values": [ + 0, + "person", + 1 + ], + "title": "SAM3 Add Video Prompt" + }, + { + "id": 5, + "type": "SAM3PropagateVideo", + "pos": [1400, 100], + "size": [350, 200], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "session", + "type": "SAM3_VIDEO_SESSION", + "link": 5 + } + ], + "outputs": [ + { + "name": "video_masks", + "type": "SAM3_VIDEO_MASKS", + "links": [6] + } + ], + "properties": {}, + "widgets_values": [ + "both", + 0, + -1 + ], + "title": "SAM3 Propagate Video" + }, + { + "id": 6, + "type": "SAM3VideoOutput", + "pos": [1800, 100], + "size": [300, 150], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "video_masks", + "type": "SAM3_VIDEO_MASKS", + "link": 6 + } + ], + "outputs": [ + { + "name": "masks", + "type": "MASK", + "links": [7] + } + ], + "properties": {}, + "widgets_values": [-1], + "title": "SAM3 Video Output" + }, + { + "id": 7, + "type": "PreviewImage", + "pos": [2150, 100], + "size": [300, 300], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "MASK", + "link": 7 + } + ], + "properties": {}, + "title": "Preview Masks" + }, + { + "id": 8, + "type": "SAM3CloseVideoSession", + "pos": [950, 400], + "size": [300, 100], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "session", + "type": "SAM3_VIDEO_SESSION", + "link": 4 + } + ], + "properties": {}, + "title": "SAM3 Close Video Session" + } + ], + "links": [ + [1, 1, 0, 3, 0, "SAM3_VIDEO_MODEL"], + [2, 2, 0, 3, 1, "IMAGE"], + [3, 3, 0, 4, 0, "SAM3_VIDEO_SESSION"], + [4, 3, 0, 8, 0, "SAM3_VIDEO_SESSION"], + [5, 4, 0, 5, 0, "SAM3_VIDEO_SESSION"], + [6, 5, 0, 6, 0, "SAM3_VIDEO_MASKS"], + [7, 6, 0, 7, 0, "MASK"] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 1, + "offset": [0, 0] + } + }, + "version": 0.4 +}