updated viewer

README.md

@@ -206,6 +206,7 @@ ComfyUI-SAM3/
 
 - **SAM3**: Meta AI Research (https://github.com/facebookresearch/sam3)
 - **ComfyUI Integration**: ComfyUI-SAM3
+- **Interactive Points Editor**: Adapted from [ComfyUI-KJNodes](https://github.com/kijai/ComfyUI-KJNodes) by kijai (Apache 2.0 License). The SAM3PointsEditor node is based on the PointsEditor implementation from KJNodes, simplified for SAM3-specific point-based segmentation.
 
 ## License
 

__init__.py

@@ -14,6 +14,14 @@ Helper Nodes (Visual Prompt Creation):
 - SAM3CombineBoxes: Combine multiple box prompts (up to 5)
 - SAM3CombinePoints: Combine multiple point prompts (up to 10)
 
+Interactive Features:
+- Right-click any node with IMAGE/MASK output → "Open in SAM3 Detector"
+- Interactive point-and-click segmentation (left-click=positive, right-click=negative)
+
+Video Tracking:
+- SAM3VideoModelLoader, SAM3InitVideoSession, SAM3AddVideoPrompt
+- SAM3PropagateVideo, SAM3VideoOutput, SAM3CloseVideoSession
+
 Workflow Example:
   [SAM3CreateBox] → [SAM3CombineBoxes] → [SAM3Segmentation] ← [LoadImage]
   [SAM3CreatePoint] → [SAM3CombinePoints] ↗
@@ -22,20 +30,24 @@ All geometric refinement uses SAM3's grounding model approach.
 No JSON typing required - pure visual node-based workflow!
 
 Author: ComfyUI-SAM3
-Version: 2.0.0
+Version: 2.1.0
 License: MIT
 """
 
 from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
 
-# Web directory for custom UI (optional, not needed for basic functionality)
-WEB_DIRECTORY = None
+# Web directory for custom UI (interactive SAM3 detector)
+WEB_DIRECTORY = "./web"
+
+# Import server to register API endpoints
+from . import sam3_server
 
 # Version info
-__version__ = "2.0.0"
+__version__ = "2.1.0"
 
 # Export for ComfyUI
 __all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS', 'WEB_DIRECTORY']
 
 print(f"[SAM3] ComfyUI-SAM3 v{__version__} loaded successfully")
 print(f"[SAM3] Available nodes: {', '.join(NODE_CLASS_MAPPINGS.keys())}")
+print(f"[SAM3] Interactive SAM3 Detector: Right-click any IMAGE/MASK node → 'Open in SAM3 Detector'")

nodes/__init__.py

@@ -6,14 +6,22 @@ from .load_model import NODE_CLASS_MAPPINGS as LOAD_MODEL_MAPPINGS
 from .load_model import NODE_DISPLAY_NAME_MAPPINGS as LOAD_MODEL_DISPLAY_MAPPINGS
 from .segmentation import NODE_CLASS_MAPPINGS as SEGMENTATION_MAPPINGS
 from .segmentation import NODE_DISPLAY_NAME_MAPPINGS as SEGMENTATION_DISPLAY_MAPPINGS
+from .sam3_video_nodes import NODE_CLASS_MAPPINGS as VIDEO_MAPPINGS
+from .sam3_video_nodes import NODE_DISPLAY_NAME_MAPPINGS as VIDEO_DISPLAY_MAPPINGS
+from .sam3_interactive import NODE_CLASS_MAPPINGS as INTERACTIVE_MAPPINGS
+from .sam3_interactive import NODE_DISPLAY_NAME_MAPPINGS as INTERACTIVE_DISPLAY_MAPPINGS
 
 # Combine all node mappings
 NODE_CLASS_MAPPINGS = {}
 NODE_CLASS_MAPPINGS.update(LOAD_MODEL_MAPPINGS)
 NODE_CLASS_MAPPINGS.update(SEGMENTATION_MAPPINGS)
+NODE_CLASS_MAPPINGS.update(VIDEO_MAPPINGS)
+NODE_CLASS_MAPPINGS.update(INTERACTIVE_MAPPINGS)
 
 NODE_DISPLAY_NAME_MAPPINGS = {}
 NODE_DISPLAY_NAME_MAPPINGS.update(LOAD_MODEL_DISPLAY_MAPPINGS)
 NODE_DISPLAY_NAME_MAPPINGS.update(SEGMENTATION_DISPLAY_MAPPINGS)
+NODE_DISPLAY_NAME_MAPPINGS.update(VIDEO_DISPLAY_MAPPINGS)
+NODE_DISPLAY_NAME_MAPPINGS.update(INTERACTIVE_DISPLAY_MAPPINGS)
 
 __all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS']

nodes/sam3_interactive.py

@@ -0,0 +1,134 @@
+"""
+SAM3 Point Collector - Interactive Point Selection
+
+Point editor widget adapted from ComfyUI-KJNodes
+Original: https://github.com/kijai/ComfyUI-KJNodes
+Author: kijai
+License: Apache 2.0
+
+Modifications for SAM3:
+- Removed bounding box functionality
+- Simplified to positive/negative points only
+- Outputs point arrays for use with SAM3Segmentation node
+"""
+
+import torch
+import numpy as np
+import json
+import io
+import base64
+from PIL import Image
+
+
+class SAM3PointCollector:
+    """
+    Interactive Point Collector for SAM3
+
+    Displays image canvas in the node where users can click to add:
+    - Positive points (Left-click) - green circles
+    - Negative points (Shift+Left-click or Right-click) - red circles
+
+    Outputs point arrays to feed into SAM3Segmentation node.
+    """
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "image": ("IMAGE",),
+                "points_store": ("STRING", {"multiline": False, "default": "{}"}),
+                "coordinates": ("STRING", {"multiline": False, "default": "[]"}),
+                "neg_coordinates": ("STRING", {"multiline": False, "default": "[]"}),
+            },
+        }
+
+    RETURN_TYPES = ("SAM3_POINTS_PROMPT", "SAM3_POINTS_PROMPT")
+    RETURN_NAMES = ("positive_points", "negative_points")
+    FUNCTION = "collect_points"
+    CATEGORY = "SAM3"
+    OUTPUT_NODE = True  # Makes node executable even without outputs connected
+
+    def collect_points(self, image, points_store, coordinates, neg_coordinates):
+        """
+        Collect points from interactive canvas
+
+        Args:
+            image: ComfyUI image tensor [B, H, W, C]
+            points_store: Combined JSON storage (hidden widget)
+            coordinates: Positive points JSON (hidden widget)
+            neg_coordinates: Negative points JSON (hidden widget)
+
+        Returns:
+            Tuple of (positive_points, negative_points) as separate SAM3_POINTS_PROMPT outputs
+        """
+        # Parse coordinates from JSON
+        try:
+            pos_coords = json.loads(coordinates) if coordinates and coordinates.strip() else []
+            neg_coords = json.loads(neg_coordinates) if neg_coordinates and neg_coordinates.strip() else []
+        except json.JSONDecodeError:
+            pos_coords = []
+            neg_coords = []
+
+        print(f"[SAM3 Point Collector] Collected {len(pos_coords)} positive, {len(neg_coords)} negative points")
+
+        # Get image dimensions for normalization
+        img_height, img_width = image.shape[1], image.shape[2]
+        print(f"[SAM3 Point Collector] Image dimensions: {img_width}x{img_height}")
+
+        # Convert to SAM3 point format - separate positive and negative outputs
+        # SAM3 expects normalized coordinates (0-1), so divide by image dimensions
+        positive_points = {"points": [], "labels": []}
+        negative_points = {"points": [], "labels": []}
+
+        # Add positive points (label = 1) - normalize to 0-1
+        for p in pos_coords:
+            normalized_x = p['x'] / img_width
+            normalized_y = p['y'] / img_height
+            positive_points["points"].append([normalized_x, normalized_y])
+            positive_points["labels"].append(1)
+            print(f"[SAM3 Point Collector]   Positive point: ({p['x']:.1f}, {p['y']:.1f}) -> ({normalized_x:.3f}, {normalized_y:.3f})")
+
+        # Add negative points (label = 0) - normalize to 0-1
+        for n in neg_coords:
+            normalized_x = n['x'] / img_width
+            normalized_y = n['y'] / img_height
+            negative_points["points"].append([normalized_x, normalized_y])
+            negative_points["labels"].append(0)
+            print(f"[SAM3 Point Collector]   Negative point: ({n['x']:.1f}, {n['y']:.1f}) -> ({normalized_x:.3f}, {normalized_y:.3f})")
+
+        print(f"[SAM3 Point Collector] Output: {len(positive_points['points'])} positive, {len(negative_points['points'])} negative")
+
+        # Send image back to widget as base64
+        img_base64 = self.tensor_to_base64(image)
+
+        return {
+            "ui": {"bg_image": [img_base64]},
+            "result": (positive_points, negative_points)
+        }
+
+    def tensor_to_base64(self, tensor):
+        """Convert ComfyUI image tensor to base64 string for JavaScript widget"""
+        # Convert from [B, H, W, C] to PIL Image
+        # Take first image if batch
+        img_array = tensor[0].cpu().numpy()
+        # Convert from 0-1 float to 0-255 uint8
+        img_array = (img_array * 255).astype(np.uint8)
+        pil_img = Image.fromarray(img_array)
+
+        # Convert to base64
+        buffered = io.BytesIO()
+        pil_img.save(buffered, format="JPEG", quality=75)
+        img_bytes = buffered.getvalue()
+        img_base64 = base64.b64encode(img_bytes).decode('utf-8')
+
+        return img_base64
+
+
+# Node mappings for ComfyUI registration
+NODE_CLASS_MAPPINGS = {
+    "SAM3PointCollector": SAM3PointCollector,
+}
+
+NODE_DISPLAY_NAME_MAPPINGS = {
+    "SAM3PointCollector": "SAM3 Point Collector",
+}

nodes/sam3_video_nodes.py

@@ -0,0 +1,423 @@
+"""
+SAM3 Video Tracking Nodes for ComfyUI
+
+These nodes provide video object tracking and segmentation capabilities using SAM3.
+They use the full sam3 library (not the vendored image-only version).
+"""
+
+import sys
+from pathlib import Path
+import torch
+import numpy as np
+import folder_paths
+
+# Add the full sam3 library to the path
+sam3_root = Path(__file__).parent.parent.parent.parent.parent / "sam3"
+if str(sam3_root) not in sys.path:
+    sys.path.insert(0, str(sam3_root))
+
+from sam3.model_builder import build_sam3_video_predictor
+
+
+class SAM3VideoModelLoader:
+    """Load SAM3 model for video tracking"""
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "checkpoint_path": ("STRING", {
+                    "default": "",
+                    "multiline": False
+                }),
+                "bpe_path": ("STRING", {
+                    "default": "",
+                    "multiline": False
+                }),
+                "hf_token": ("STRING", {
+                    "default": "",
+                    "multiline": False
+                }),
+            },
+            "optional": {
+                "use_multi_gpu": ("BOOLEAN", {"default": False}),
+                "gpu_ids": ("STRING", {
+                    "default": "0",
+                    "multiline": False
+                }),
+            }
+        }
+
+    RETURN_TYPES = ("SAM3_VIDEO_MODEL",)
+    RETURN_NAMES = ("video_model",)
+    FUNCTION = "load_model"
+    CATEGORY = "SAM3/video"
+
+    def load_model(self, checkpoint_path, bpe_path, hf_token, use_multi_gpu=False, gpu_ids="0"):
+        """Load the SAM3 video model"""
+        import os
+        if hf_token:
+            os.environ["HF_TOKEN"] = hf_token
+
+        # Parse GPU IDs if using multi-GPU
+        gpus_to_use = None
+        if use_multi_gpu:
+            gpus_to_use = [int(x.strip()) for x in gpu_ids.split(",")]
+
+        print(f"[SAM3 Video] Loading video model from {checkpoint_path}")
+
+        # Build the video predictor
+        predictor = build_sam3_video_predictor(
+            checkpoint_path=checkpoint_path if checkpoint_path else None,
+            bpe_path=bpe_path if bpe_path else None,
+            gpus_to_use=gpus_to_use,
+        )
+
+        print(f"[SAM3 Video] Model loaded successfully")
+
+        return (predictor,)
+
+
+class SAM3InitVideoSession:
+    """Initialize a video tracking session"""
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "video_model": ("SAM3_VIDEO_MODEL",),
+                "video_frames": ("IMAGE",),  # ComfyUI video frames [B, H, W, C]
+            },
+            "optional": {
+                "session_id": ("STRING", {
+                    "default": "",
+                    "multiline": False
+                }),
+            }
+        }
+
+    RETURN_TYPES = ("SAM3_VIDEO_SESSION", "STRING")
+    RETURN_NAMES = ("session", "session_id")
+    FUNCTION = "init_session"
+    CATEGORY = "SAM3/video"
+
+    def init_session(self, video_model, video_frames, session_id=""):
+        """Initialize a tracking session with video frames"""
+        # Convert ComfyUI frames to temporary directory
+        import tempfile
+        import os
+        from PIL import Image
+
+        # Create a temporary directory for frames
+        temp_dir = tempfile.mkdtemp(prefix="sam3_video_")
+
+        # Save frames as JPEG
+        num_frames = video_frames.shape[0]
+        for i in range(num_frames):
+            frame = video_frames[i].cpu().numpy()
+            # Convert from [H, W, C] float32 0-1 to uint8 0-255
+            frame = (frame * 255).astype(np.uint8)
+            img = Image.fromarray(frame)
+            img.save(os.path.join(temp_dir, f"{i:05d}.jpg"))
+
+        print(f"[SAM3 Video] Saved {num_frames} frames to {temp_dir}")
+
+        # Start the session
+        response = video_model.start_session(
+            resource_path=temp_dir,
+            session_id=session_id if session_id else None
+        )
+
+        actual_session_id = response["session_id"]
+
+        session_data = {
+            "model": video_model,
+            "session_id": actual_session_id,
+            "temp_dir": temp_dir,
+            "num_frames": num_frames,
+            "height": video_frames.shape[1],
+            "width": video_frames.shape[2],
+        }
+
+        print(f"[SAM3 Video] Initialized session {actual_session_id}")
+
+        return (session_data, actual_session_id)
+
+
+class SAM3AddVideoPrompt:
+    """Add a text, box, or point prompt to a specific video frame"""
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "session": ("SAM3_VIDEO_SESSION",),
+                "frame_index": ("INT", {
+                    "default": 0,
+                    "min": 0,
+                    "max": 10000,
+                    "step": 1
+                }),
+            },
+            "optional": {
+                "text_prompt": ("STRING", {
+                    "default": "",
+                    "multiline": False
+                }),
+                "obj_id": ("INT", {
+                    "default": 1,
+                    "min": 1,
+                    "max": 100,
+                    "step": 1
+                }),
+                "boxes": ("SAM3_BOXES_PROMPT",),
+                "points": ("SAM3_POINTS_PROMPT",),
+            }
+        }
+
+    RETURN_TYPES = ("SAM3_VIDEO_SESSION",)
+    RETURN_NAMES = ("session",)
+    FUNCTION = "add_prompt"
+    CATEGORY = "SAM3/video"
+
+    def add_prompt(self, session, frame_index, text_prompt="", obj_id=1, boxes=None, points=None):
+        """Add a prompt on a specific frame"""
+        video_model = session["model"]
+        session_id = session["session_id"]
+
+        # Prepare prompt parameters
+        bounding_boxes = None
+        bounding_box_labels = None
+        point_coords = None
+        point_labels = None
+
+        if boxes is not None:
+            bounding_boxes = boxes["boxes"]
+            bounding_box_labels = boxes.get("labels", None)
+
+        if points is not None:
+            point_coords = points["points"]
+            point_labels = points["labels"]
+
+        print(f"[SAM3 Video] Adding prompt on frame {frame_index}: text='{text_prompt}', obj_id={obj_id}")
+
+        # Add the prompt
+        response = video_model.add_prompt(
+            session_id=session_id,
+            frame_idx=frame_index,
+            text=text_prompt if text_prompt else None,
+            points=point_coords,
+            point_labels=point_labels,
+            bounding_boxes=bounding_boxes,
+            bounding_box_labels=bounding_box_labels,
+            obj_id=obj_id,
+        )
+
+        print(f"[SAM3 Video] Prompt added successfully")
+
+        return (session,)
+
+
+class SAM3PropagateVideo:
+    """Propagate masks through the entire video"""
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "session": ("SAM3_VIDEO_SESSION",),
+            },
+            "optional": {
+                "propagation_direction": (["both", "forward", "backward"], {"default": "both"}),
+                "start_frame_index": ("INT", {
+                    "default": 0,
+                    "min": 0,
+                    "max": 10000,
+                    "step": 1
+                }),
+                "max_frames": ("INT", {
+                    "default": -1,
+                    "min": -1,
+                    "max": 10000,
+                    "step": 1
+                }),
+            }
+        }
+
+    RETURN_TYPES = ("SAM3_VIDEO_MASKS",)
+    RETURN_NAMES = ("video_masks",)
+    FUNCTION = "propagate"
+    CATEGORY = "SAM3/video"
+
+    def propagate(self, session, propagation_direction="both", start_frame_index=0, max_frames=-1):
+        """Propagate the prompts through all video frames"""
+        video_model = session["model"]
+        session_id = session["session_id"]
+        num_frames = session["num_frames"]
+
+        max_frame_num_to_track = max_frames if max_frames > 0 else num_frames
+
+        print(f"[SAM3 Video] Propagating in {propagation_direction} direction from frame {start_frame_index}")
+
+        # Collect all frames' masks
+        all_masks = {}
+        all_obj_ids = None
+
+        request = {
+            "type": "propagate_in_video",
+            "session_id": session_id,
+            "propagation_direction": propagation_direction,
+            "start_frame_index": start_frame_index,
+            "max_frame_num_to_track": max_frame_num_to_track,
+        }
+
+        for response in video_model.handle_stream_request(request):
+            frame_idx = response["frame_index"]
+            outputs = response["outputs"]
+
+            all_masks[frame_idx] = outputs
+            if all_obj_ids is None:
+                all_obj_ids = outputs.get("obj_ids", [])
+
+        print(f"[SAM3 Video] Propagation complete. Tracked {len(all_masks)} frames, {len(all_obj_ids)} objects")
+
+        video_masks = {
+            "session": session,
+            "masks": all_masks,
+            "obj_ids": all_obj_ids,
+            "num_frames": num_frames,
+        }
+
+        return (video_masks,)
+
+
+class SAM3VideoOutput:
+    """Output video masks as ComfyUI tensors"""
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "video_masks": ("SAM3_VIDEO_MASKS",),
+            },
+            "optional": {
+                "obj_id_filter": ("INT", {
+                    "default": -1,
+                    "min": -1,
+                    "max": 100,
+                    "step": 1
+                }),
+            }
+        }
+
+    RETURN_TYPES = ("MASK",)
+    RETURN_NAMES = ("masks",)
+    FUNCTION = "output_masks"
+    CATEGORY = "SAM3/video"
+
+    def output_masks(self, video_masks, obj_id_filter=-1):
+        """Convert video masks to ComfyUI mask format"""
+        masks_dict = video_masks["masks"]
+        obj_ids = video_masks["obj_ids"]
+        num_frames = video_masks["num_frames"]
+        session = video_masks["session"]
+        height = session["height"]
+        width = session["width"]
+
+        # Create output tensor [N, H, W] where N is number of frames
+        output_masks = torch.zeros((num_frames, height, width), dtype=torch.float32)
+
+        for frame_idx in range(num_frames):
+            if frame_idx in masks_dict:
+                frame_output = masks_dict[frame_idx]
+
+                # Get masks for this frame
+                if "video_res_masks" in frame_output:
+                    frame_masks = frame_output["video_res_masks"]
+                elif "pred_masks" in frame_output:
+                    # Resize if needed
+                    frame_masks = frame_output["pred_masks"]
+                    if frame_masks.shape[-2:] != (height, width):
+                        frame_masks = torch.nn.functional.interpolate(
+                            frame_masks,
+                            size=(height, width),
+                            mode="bilinear",
+                            align_corners=False,
+                        )
+                else:
+                    continue
+
+                # Filter by object ID if specified
+                if obj_id_filter > 0:
+                    try:
+                        obj_idx = obj_ids.index(obj_id_filter)
+                        mask = frame_masks[obj_idx, 0] > 0.0
+                    except (ValueError, IndexError):
+                        mask = torch.zeros((height, width), dtype=torch.bool)
+                else:
+                    # Combine all object masks
+                    mask = (frame_masks[:, 0] > 0.0).any(dim=0)
+
+                output_masks[frame_idx] = mask.float().cpu()
+
+        print(f"[SAM3 Video] Output {num_frames} mask frames")
+
+        return (output_masks,)
+
+
+class SAM3CloseVideoSession:
+    """Close a video tracking session and cleanup resources"""
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "session": ("SAM3_VIDEO_SESSION",),
+            },
+        }
+
+    RETURN_TYPES = ()
+    OUTPUT_NODE = True
+    FUNCTION = "close_session"
+    CATEGORY = "SAM3/video"
+
+    def close_session(self, session):
+        """Close the session and cleanup temporary files"""
+        import shutil
+
+        video_model = session["model"]
+        session_id = session["session_id"]
+        temp_dir = session["temp_dir"]
+
+        # Close the session
+        video_model.close_session(session_id)
+
+        # Clean up temporary directory
+        if temp_dir and Path(temp_dir).exists():
+            shutil.rmtree(temp_dir)
+            print(f"[SAM3 Video] Cleaned up temp directory: {temp_dir}")
+
+        print(f"[SAM3 Video] Closed session {session_id}")
+
+        return ()
+
+
+# Node class mappings
+NODE_CLASS_MAPPINGS = {
+    "SAM3VideoModelLoader": SAM3VideoModelLoader,
+    "SAM3InitVideoSession": SAM3InitVideoSession,
+    "SAM3AddVideoPrompt": SAM3AddVideoPrompt,
+    "SAM3PropagateVideo": SAM3PropagateVideo,
+    "SAM3VideoOutput": SAM3VideoOutput,
+    "SAM3CloseVideoSession": SAM3CloseVideoSession,
+}
+
+# Node display name mappings
+NODE_DISPLAY_NAME_MAPPINGS = {
+    "SAM3VideoModelLoader": "SAM3 Load Video Model",
+    "SAM3InitVideoSession": "SAM3 Init Video Session",
+    "SAM3AddVideoPrompt": "SAM3 Add Video Prompt",
+    "SAM3PropagateVideo": "SAM3 Propagate Video",
+    "SAM3VideoOutput": "SAM3 Video Output",
+    "SAM3CloseVideoSession": "SAM3 Close Video Session",
+}

nodes/segmentation.py

@@ -41,8 +41,10 @@ class SAM3Segmentation:
                     "multiline": False,
                     "placeholder": "e.g., 'cat', 'person in red', 'car'"
                 }),
-                "boxes_prompt": ("SAM3_BOXES_PROMPT",),
-                "points_prompt": ("SAM3_POINTS_PROMPT",),
+                "positive_boxes": ("SAM3_BOXES_PROMPT",),
+                "negative_boxes": ("SAM3_BOXES_PROMPT",),
+                "positive_points": ("SAM3_POINTS_PROMPT",),
+                "negative_points": ("SAM3_POINTS_PROMPT",),
                 "mask_prompt": ("MASK",),
                 "max_detections": ("INT", {
                     "default": -1,
@@ -60,7 +62,8 @@ class SAM3Segmentation:
     CATEGORY = "SAM3"
 
     def segment(self, sam3_model, image, confidence_threshold=0.2, text_prompt="",
-                boxes_prompt=None, points_prompt=None, mask_prompt=None, max_detections=-1):
+                positive_boxes=None, negative_boxes=None, positive_points=None, negative_points=None,
+                mask_prompt=None, max_detections=-1):
         """
         Perform SAM3 segmentation with optional geometric prompts
 
@@ -69,8 +72,10 @@ class SAM3Segmentation:
             image: ComfyUI image tensor [B, H, W, C]
             confidence_threshold: Minimum confidence score for detections
             text_prompt: Optional text description of objects to segment
-            boxes_prompt: Optional box prompts from SAM3CombineBoxes
-            points_prompt: Optional point prompts from SAM3CombinePoints
+            positive_boxes: Optional positive box prompts
+            negative_boxes: Optional negative box prompts
+            positive_points: Optional positive point prompts
+            negative_points: Optional negative point prompts
             mask_prompt: Optional mask prompt
             max_detections: Maximum number of detections to return
 
@@ -84,10 +89,14 @@ class SAM3Segmentation:
         print(f"[SAM3] Running segmentation")
         if text_prompt:
             print(f"[SAM3]   Text prompt: '{text_prompt}'")
-        if boxes_prompt:
-            print(f"[SAM3]   Boxes: {len(boxes_prompt['boxes'])}")
-        if points_prompt:
-            print(f"[SAM3]   Points: {len(points_prompt['points'])}")
+        if positive_boxes:
+            print(f"[SAM3]   Positive boxes: {len(positive_boxes['boxes'])}")
+        if negative_boxes:
+            print(f"[SAM3]   Negative boxes: {len(negative_boxes['boxes'])}")
+        if positive_points:
+            print(f"[SAM3]   Positive points: {len(positive_points['points'])}")
+        if negative_points:
+            print(f"[SAM3]   Negative points: {len(negative_points['points'])}")
         if mask_prompt is not None:
             print(f"[SAM3]   Mask prompt provided")
         print(f"[SAM3] Confidence threshold: {confidence_threshold}")
@@ -107,20 +116,42 @@ class SAM3Segmentation:
             print(f"[SAM3] Adding text prompt...")
             state = processor.set_text_prompt(text_prompt.strip(), state)
 
-        # Add geometric prompts
-        if boxes_prompt is not None and len(boxes_prompt['boxes']) > 0:
-            print(f"[SAM3] Adding {len(boxes_prompt['boxes'])} box prompts...")
+        # Add geometric prompts - combine positive and negative
+        all_boxes = []
+        all_box_labels = []
+
+        if positive_boxes is not None and len(positive_boxes['boxes']) > 0:
+            all_boxes.extend(positive_boxes['boxes'])
+            all_box_labels.extend(positive_boxes['labels'])
+
+        if negative_boxes is not None and len(negative_boxes['boxes']) > 0:
+            all_boxes.extend(negative_boxes['boxes'])
+            all_box_labels.extend(negative_boxes['labels'])
+
+        if len(all_boxes) > 0:
+            print(f"[SAM3] Adding {len(all_boxes)} box prompts...")
             state = processor.add_multiple_box_prompts(
-                boxes_prompt['boxes'],
-                boxes_prompt['labels'],
+                all_boxes,
+                all_box_labels,
                 state
             )
 
-        if points_prompt is not None and len(points_prompt['points']) > 0:
-            print(f"[SAM3] Adding {len(points_prompt['points'])} point prompts...")
+        all_points = []
+        all_point_labels = []
+
+        if positive_points is not None and len(positive_points['points']) > 0:
+            all_points.extend(positive_points['points'])
+            all_point_labels.extend(positive_points['labels'])
+
+        if negative_points is not None and len(negative_points['points']) > 0:
+            all_points.extend(negative_points['points'])
+            all_point_labels.extend(negative_points['labels'])
+
+        if len(all_points) > 0:
+            print(f"[SAM3] Adding {len(all_points)} point prompts...")
             state = processor.add_point_prompt(
-                points_prompt['points'],
-                points_prompt['labels'],
+                all_points,
+                all_point_labels,
                 state
             )
 

prestartup_script.py

@@ -71,13 +71,29 @@ def main():
     else:
         print(f"[SAM3] No assets folder found")
 
-    # Copy workflows to user workflows folder
+    # Copy workflows to user workflows folder with SAM3- prefix
     workflows_src = script_dir / "workflows"
     workflows_dst = comfy_root / "user" / "default" / "workflows"
 
     if workflows_src.exists():
         print(f"[SAM3] Copying workflows to {workflows_dst}...")
-        copied, skipped = copy_files_skip_existing(str(workflows_src), str(workflows_dst))
+        os.makedirs(str(workflows_dst), exist_ok=True)
+        copied = 0
+        skipped = 0
+
+        for item in os.listdir(str(workflows_src)):
+            src_path = workflows_src / item
+            if src_path.is_file():
+                # Add SAM3- prefix to workflow filenames
+                prefixed_name = f"SAM3-{item}" if not item.startswith("SAM3-") else item
+                dst_path = workflows_dst / prefixed_name
+
+                if dst_path.exists():
+                    skipped += 1
+                else:
+                    shutil.copy2(str(src_path), str(dst_path))
+                    copied += 1
+
         print(f"[SAM3] Workflows: {copied} copied, {skipped} skipped")
     else:
         print(f"[SAM3] No workflows folder found")

requirements.txt

@@ -9,6 +9,12 @@ regex
 iopath>=0.1.10
 huggingface_hub
 
+# Video support dependencies
+decord>=0.6.0
+opencv-python>=4.8.0
+psutil>=5.9.0
+pycocotools>=2.0.6
+
 # Standard dependencies
 numpy>=1.26
 Pillow

sam3_lib/sam3_video_predictor.py

@@ -0,0 +1,521 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
+
+import datetime
+import gc
+import multiprocessing as mp
+import os
+import queue
+import socket
+import sys
+import time
+import uuid
+from contextlib import closing
+from typing import List, Optional
+
+import psutil
+import torch
+
+# Adapted imports for vendored version
+import logging
+logger = logging.getLogger(__name__)
+
+
+class Sam3VideoPredictor:
+    # a global dictionary that holds all inference states for this model (key is session_id)
+    _ALL_INFERENCE_STATES = {}
+
+    def __init__(
+        self,
+        checkpoint_path=None,
+        bpe_path=None,
+        has_presence_token=True,
+        geo_encoder_use_img_cross_attn=True,
+        strict_state_dict_loading=True,
+        async_loading_frames=False,
+        video_loader_type="cv2",
+        apply_temporal_disambiguation: bool = True,
+    ):
+        self.async_loading_frames = async_loading_frames
+        self.video_loader_type = video_loader_type
+        from .model_builder import build_sam3_video_model
+
+        self.model = (
+            build_sam3_video_model(
+                checkpoint_path=checkpoint_path,
+                bpe_path=bpe_path,
+                has_presence_token=has_presence_token,
+                geo_encoder_use_img_cross_attn=geo_encoder_use_img_cross_attn,
+                strict_state_dict_loading=strict_state_dict_loading,
+                apply_temporal_disambiguation=apply_temporal_disambiguation,
+            )
+            .cuda()
+            .eval()
+        )
+
+    @torch.inference_mode()
+    def handle_request(self, request):
+        """Dispatch a request based on its type."""
+        request_type = request["type"]
+        if request_type == "start_session":
+            return self.start_session(
+                resource_path=request["resource_path"],
+                session_id=request.get("session_id", None),
+            )
+        elif request_type == "add_prompt":
+            return self.add_prompt(
+                session_id=request["session_id"],
+                frame_idx=request["frame_index"],
+                text=request.get("text", None),
+                points=request.get("points", None),
+                point_labels=request.get("point_labels", None),
+                bounding_boxes=request.get("bounding_boxes", None),
+                bounding_box_labels=request.get("bounding_box_labels", None),
+                obj_id=request.get("obj_id", None),
+            )
+        elif request_type == "remove_object":
+            return self.remove_object(
+                session_id=request["session_id"],
+                obj_id=request["obj_id"],
+                is_user_action=request.get("is_user_action", True),
+            )
+        elif request_type == "reset_session":
+            return self.reset_session(session_id=request["session_id"])
+        elif request_type == "close_session":
+            return self.close_session(session_id=request["session_id"])
+        else:
+            raise RuntimeError(f"invalid request type: {request_type}")
+
+    @torch.inference_mode()
+    def handle_stream_request(self, request):
+        """Dispatch a stream request based on its type."""
+        request_type = request["type"]
+        if request_type == "propagate_in_video":
+            yield from self.propagate_in_video(
+                session_id=request["session_id"],
+                propagation_direction=request.get("propagation_direction", "both"),
+                start_frame_idx=request.get("start_frame_index", None),
+                max_frame_num_to_track=request.get("max_frame_num_to_track", None),
+            )
+        else:
+            raise RuntimeError(f"invalid request type: {request_type}")
+
+    def start_session(self, resource_path, session_id=None):
+        """
+        Start a new inference session on an image or a video. Here `resource_path`
+        can be either a path to an image file (for image inference) or an MP4 file
+        or directory with JPEG video frames (for video inference).
+
+        If `session_id` is defined, it will be used as identifier for the
+        session. If it is not defined, the start_session function will create
+        a session id and return it.
+        """
+        # get an initial inference_state from the model
+        inference_state = self.model.init_state(
+            resource_path=resource_path,
+            async_loading_frames=self.async_loading_frames,
+            video_loader_type=self.video_loader_type,
+        )
+        if not session_id:
+            session_id = str(uuid.uuid4())
+        self._ALL_INFERENCE_STATES[session_id] = {
+            "state": inference_state,
+            "session_id": session_id,
+            "start_time": time.time(),
+        }
+        logger.debug(
+            f"started new session {session_id}; {self._get_session_stats()}; "
+            f"{self._get_torch_and_gpu_properties()}"
+        )
+        return {"session_id": session_id}
+
+    def add_prompt(
+        self,
+        session_id: str,
+        frame_idx: int,
+        text: Optional[str] = None,
+        points: Optional[List[List[float]]] = None,
+        point_labels: Optional[List[int]] = None,
+        bounding_boxes: Optional[List[List[float]]] = None,
+        bounding_box_labels: Optional[List[int]] = None,
+        obj_id: Optional[int] = None,
+    ):
+        """Add text, box and/or point prompt on a specific video frame."""
+        logger.debug(
+            f"add prompt on frame {frame_idx} in session {session_id}: "
+            f"{text=}, {points=}, {point_labels=}, "
+            f"{bounding_boxes=}, {bounding_box_labels=}"
+        )
+        session = self._get_session(session_id)
+        inference_state = session["state"]
+
+        frame_idx, outputs = self.model.add_prompt(
+            inference_state=inference_state,
+            frame_idx=frame_idx,
+            text_str=text,
+            points=points,
+            point_labels=point_labels,
+            boxes_xywh=bounding_boxes,
+            box_labels=bounding_box_labels,
+            obj_id=obj_id,
+        )
+        return {"frame_index": frame_idx, "outputs": outputs}
+
+    def remove_object(
+        self,
+        session_id: str,
+        obj_id: int,
+        is_user_action: bool = True,
+    ):
+        """Remove an object from tracking."""
+        logger.debug(
+            f"remove object {obj_id} in session {session_id}: " f"{is_user_action=}"
+        )
+        session = self._get_session(session_id)
+        inference_state = session["state"]
+
+        self.model.remove_object(
+            inference_state=inference_state,
+            obj_id=obj_id,
+            is_user_action=is_user_action,
+        )
+        return {"is_success": True}
+
+    def propagate_in_video(
+        self,
+        session_id,
+        propagation_direction,
+        start_frame_idx,
+        max_frame_num_to_track,
+    ):
+        """Propagate the added prompts to get grounding results on all video frames."""
+        logger.debug(
+            f"propagate in video in session {session_id}: "
+            f"{propagation_direction=}, {start_frame_idx=}, {max_frame_num_to_track=}"
+        )
+        try:
+            session = self._get_session(session_id)
+            inference_state = session["state"]
+            if propagation_direction not in ["both", "forward", "backward"]:
+                raise ValueError(
+                    f"invalid propagation direction: {propagation_direction}"
+                )
+
+            # First doing the forward propagation
+            if propagation_direction in ["both", "forward"]:
+                for frame_idx, outputs in self.model.propagate_in_video(
+                    inference_state=inference_state,
+                    start_frame_idx=start_frame_idx,
+                    max_frame_num_to_track=max_frame_num_to_track,
+                    reverse=False,
+                ):
+                    yield {"frame_index": frame_idx, "outputs": outputs}
+            # Then doing the backward propagation (reverse in time)
+            if propagation_direction in ["both", "backward"]:
+                for frame_idx, outputs in self.model.propagate_in_video(
+                    inference_state=inference_state,
+                    start_frame_idx=start_frame_idx,
+                    max_frame_num_to_track=max_frame_num_to_track,
+                    reverse=True,
+                ):
+                    yield {"frame_index": frame_idx, "outputs": outputs}
+        finally:
+            # Log upon completion (so that e.g. we can see if two propagations happen in parallel).
+            # Using `finally` here to log even when the tracking is aborted with GeneratorExit.
+            logger.debug(
+                f"propagation ended in session {session_id}; {self._get_session_stats()}"
+            )
+
+    def reset_session(self, session_id):
+        """Reset the session to its initial state (as when it's initial opened)."""
+        logger.debug(f"reset session {session_id}")
+        session = self._get_session(session_id)
+        inference_state = session["state"]
+        self.model.reset_state(inference_state)
+        return {"is_success": True}
+
+    def close_session(self, session_id):
+        """
+        Close a session. This method is idempotent and can be called multiple
+        times on the same "session_id".
+        """
+        session = self._ALL_INFERENCE_STATES.pop(session_id, None)
+        if session is None:
+            logger.warning(
+                f"cannot close session {session_id} as it does not exist (it might have expired); "
+                f"{self._get_session_stats()}"
+            )
+        else:
+            del session
+            gc.collect()
+            logger.info(f"removed session {session_id}; {self._get_session_stats()}")
+        return {"is_success": True}
+
+    def _get_session(self, session_id):
+        session = self._ALL_INFERENCE_STATES.get(session_id, None)
+        if session is None:
+            raise RuntimeError(
+                f"Cannot find session {session_id}; it might have expired"
+            )
+        return session
+
+    def _get_session_stats(self):
+        """Get a statistics string for live sessions and their GPU usage."""
+        # print both the session ids and their video frame numbers
+        live_session_strs = [
+            f"'{session_id}' ({session['state']['num_frames']} frames)"
+            for session_id, session in self._ALL_INFERENCE_STATES.items()
+        ]
+        session_stats_str = (
+            f"live sessions: [{', '.join(live_session_strs)}], GPU memory: "
+            f"{torch.cuda.memory_allocated() // 1024**2} MiB used and "
+            f"{torch.cuda.memory_reserved() // 1024**2} MiB reserved"
+            f" (max over time: {torch.cuda.max_memory_allocated() // 1024**2} MiB used "
+            f"and {torch.cuda.max_memory_reserved() // 1024**2} MiB reserved)"
+        )
+        return session_stats_str
+
+    def _get_torch_and_gpu_properties(self):
+        """Get a string for PyTorch and GPU properties (for logging and debugging)."""
+        torch_and_gpu_str = (
+            f"torch: {torch.__version__} with CUDA arch {torch.cuda.get_arch_list()}, "
+            f"GPU device: {torch.cuda.get_device_properties(torch.cuda.current_device())}"
+        )
+        return torch_and_gpu_str
+
+    def shutdown(self):
+        """Shutdown the predictor and clear all sessions."""
+        self._ALL_INFERENCE_STATES.clear()
+
+
+class Sam3VideoPredictorMultiGPU(Sam3VideoPredictor):
+    def __init__(self, *model_args, gpus_to_use=None, **model_kwargs):
+        if gpus_to_use is None:
+            # if not specified, use only the current GPU by default
+            gpus_to_use = [torch.cuda.current_device()]
+
+        IS_MAIN_PROCESS = os.getenv("IS_MAIN_PROCESS", "1") == "1"
+        if IS_MAIN_PROCESS:
+            gpus_to_use = sorted(set(gpus_to_use))
+            logger.info(f"using the following GPU IDs: {gpus_to_use}")
+            assert len(gpus_to_use) > 0 and all(isinstance(i, int) for i in gpus_to_use)
+            assert all(0 <= i < torch.cuda.device_count() for i in gpus_to_use)
+            os.environ["MASTER_ADDR"] = "localhost"
+            os.environ["MASTER_PORT"] = f"{self._find_free_port()}"
+            os.environ["RANK"] = "0"
+            os.environ["WORLD_SIZE"] = f"{len(gpus_to_use)}"
+
+        self.gpus_to_use = gpus_to_use
+        self.rank = int(os.environ["RANK"])
+        self.world_size = int(os.environ["WORLD_SIZE"])
+        self.rank_str = f"rank={self.rank} with world_size={self.world_size}"
+        self.device = torch.device(f"cuda:{self.gpus_to_use[self.rank]}")
+        torch.cuda.set_device(self.device)
+        self.has_shutdown = False
+        if self.rank == 0:
+            logger.info("\n\n\n\t*** START loading model on all ranks ***\n\n")
+
+        logger.info(f"loading model on {self.rank_str} -- this could take a while ...")
+        super().__init__(*model_args, **model_kwargs)
+        logger.info(f"loading model on {self.rank_str} -- DONE locally")
+
+        if self.world_size > 1 and self.rank == 0:
+            # start the worker processes *after* the model is loaded in the main process
+            # so that the main process can run torch.compile and fill the cache first
+            self._start_worker_processes(*model_args, **model_kwargs)
+            for rank in range(1, self.world_size):
+                self.command_queues[rank].put(("start_nccl_process_group", None))
+            self._start_nccl_process_group()
+
+        if self.rank == 0:
+            logger.info("\n\n\n\t*** DONE loading model on all ranks ***\n\n")
+
+    @torch.inference_mode()
+    def handle_request(self, request):
+        """Dispatch a request based on its type."""
+        if self.has_shutdown:
+            raise RuntimeError(
+                "cannot handle request after the predictor has shutdown; please create a new predictor"
+            )
+
+        # when starting a session, we need to create a session id before dispatching
+        # the request to the workers
+        if request["type"] == "start_session" and request.get("session_id") is None:
+            request["session_id"] = str(uuid.uuid4())
+        # dispatch the request to all worker processes
+        if self.world_size > 1 and self.rank == 0:
+            for rank in range(1, self.world_size):
+                self.command_queues[rank].put((request, False))
+
+        response = super().handle_request(request)
+
+        if self.world_size > 1:
+            torch.distributed.barrier()  # wait for all ranks to finish
+        return response
+
+    @torch.inference_mode()
+    def handle_stream_request(self, request):
+        """Dispatch a stream request based on its type."""
+        if self.has_shutdown:
+            raise RuntimeError(
+                "cannot handle request after the predictor has shutdown; please create a new predictor"
+            )
+
+        # dispatch the request to all worker processes
+        if self.world_size > 1 and self.rank == 0:
+            for rank in range(1, self.world_size):
+                self.command_queues[rank].put((request, True))
+
+        yield from super().handle_stream_request(request)
+
+        if self.world_size > 1:
+            torch.distributed.barrier()  # wait for all ranks to finish
+
+    def _start_worker_processes(self, *model_args, **model_kwargs):
+        """Start worker processes for handling model inference."""
+        world_size = self.world_size
+        logger.info(f"spawning {world_size - 1} worker processes")
+        # Use "spawn" (instead of "fork") for different PyTorch or CUDA context
+        mp_ctx = mp.get_context("spawn")
+        self.command_queues = {rank: mp_ctx.Queue() for rank in range(1, world_size)}
+        self.result_queues = {rank: mp_ctx.Queue() for rank in range(1, world_size)}
+        parent_pid = os.getpid()
+        for rank in range(1, world_size):
+            # set the environment variables for each worker process
+            os.environ["IS_MAIN_PROCESS"] = "0"  # mark this as a worker process
+            os.environ["RANK"] = f"{rank}"
+            worker_process = mp_ctx.Process(
+                target=Sam3VideoPredictorMultiGPU._worker_process_command_loop,
+                args=(
+                    rank,
+                    world_size,
+                    self.command_queues[rank],
+                    self.result_queues[rank],
+                    model_args,
+                    model_kwargs,
+                    self.gpus_to_use,
+                    parent_pid,
+                ),
+                daemon=True,
+            )
+            worker_process.start()
+        # revert the environment variables for the main process
+        os.environ["IS_MAIN_PROCESS"] = "1"
+        os.environ["RANK"] = "0"
+        # wait for all the worker processes to load the model and collect their PIDs
+        self.worker_pids = {}
+        for rank in range(1, self.world_size):
+            # a large timeout to cover potentially long model loading time due to compilation
+            _, worker_pid = self.result_queues[rank].get(timeout=7200)
+            self.worker_pids[rank] = worker_pid
+        logger.info(f"spawned {world_size - 1} worker processes")
+
+    def _start_nccl_process_group(self):
+        rank = int(os.environ["RANK"])
+        world_size = int(os.environ["WORLD_SIZE"])
+        if world_size == 1:
+            return
+
+        logger.debug(f"starting NCCL process group on {rank=} with {world_size=}")
+        assert not torch.distributed.is_initialized()
+        # use the "env://" init method with environment variables set in start_worker_processes
+        # a short 3-min timeout to quickly detect any synchronization failures
+        timeout_sec = int(os.getenv("SAM3_COLLECTIVE_OP_TIMEOUT_SEC", "180"))
+        timeout = datetime.timedelta(seconds=timeout_sec)
+        torch.distributed.init_process_group(
+            backend="nccl",
+            init_method="env://",
+            timeout=timeout,
+            device_id=self.device,
+        )
+        # warm-up the NCCL process group by running a dummy all-reduce
+        tensor = torch.ones(1024, 1024).cuda()
+        torch.distributed.all_reduce(tensor)
+        logger.debug(f"started NCCL process group on {rank=} with {world_size=}")
+
+    def _find_free_port(self) -> int:
+        """
+        Find a free port (a random free port from 1024 to 65535 will be selected)
+        https://stackoverflow.com/questions/1365265/on-localhost-how-do-i-pick-a-free-port-number)
+        """
+        with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+            s.bind(("", 0))
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            return s.getsockname()[1]
+
+    @staticmethod
+    def _worker_process_command_loop(
+        rank,
+        world_size,
+        command_queue,
+        result_queue,
+        model_args,
+        model_kwargs,
+        gpus_to_use,
+        parent_pid,
+    ):
+        """
+        The command loop for each worker process. It listens to commands from the main process
+        and executes them using the model.
+        """
+        logger.info(f"starting worker process {rank=} with {world_size=}")
+        # verify that the environment variables are set correctly
+        assert int(os.environ["IS_MAIN_PROCESS"]) == 0
+        assert int(os.environ["RANK"]) == rank
+        assert int(os.environ["WORLD_SIZE"]) == world_size
+        # load the model in this worker process
+        predictor = Sam3VideoPredictorMultiGPU(
+            *model_args, gpus_to_use=gpus_to_use, **model_kwargs
+        )
+        logger.info(f"started worker {rank=} with {world_size=}")
+        # return the worker process id to the main process for bookkeeping
+        worker_pid = os.getpid()
+        result_queue.put(("load_model", worker_pid))
+
+        # wait for the command to start the NCCL process group
+        request_type, _ = command_queue.get(timeout=7200)
+        assert request_type == "start_nccl_process_group"
+        predictor._start_nccl_process_group()
+
+        # keep listening to commands from the main process
+        while True:
+            try:
+                request, is_stream_request = command_queue.get(timeout=5.0)
+                if request == "shutdown":
+                    logger.info(f"worker {rank=} shutting down")
+                    torch.distributed.destroy_process_group()
+                    result_queue.put(("shutdown", True))  # acknowledge the shutdown
+                    sys.exit(0)
+
+                logger.debug(f"worker {rank=} received request {request['type']=}")
+                if is_stream_request:
+                    for _ in predictor.handle_stream_request(request):
+                        pass  # handle stream requests in a generator fashion
+                else:
+                    predictor.handle_request(request)
+            except queue.Empty:
+                # Usually Python's multiprocessing module will shutdown all the daemon worker
+                # processes when the main process exits gracefully. However, the user may kill
+                # the main process using SIGKILL and thereby leaving no chance for the main process
+                # to clean up its daemon child processes. So here we manually check whether the
+                # parent process still exists (every 5 sec as in `command_queue.get` timeout).
+                if not psutil.pid_exists(parent_pid):
+                    logger.info(
+                        f"stopping worker {rank=} as its parent process has exited"
+                    )
+                    sys.exit(1)
+            except Exception as e:
+                logger.error(f"worker {rank=} exception: {e}", exc_info=True)
+
+    def shutdown(self):
+        """Shutdown all worker processes."""
+        if self.rank == 0 and self.world_size > 1:
+            logger.info(f"shutting down {self.world_size - 1} worker processes")
+            for rank in range(1, self.world_size):
+                self.command_queues[rank].put(("shutdown", False))
+            torch.distributed.destroy_process_group()
+            for rank in range(1, self.world_size):
+                self.result_queues[rank].get()  # wait for the worker to acknowledge
+            logger.info(f"shut down {self.world_size - 1} worker processes")
+        self.has_shutdown = True
+
+        super().shutdown()

sam3_server.py

@@ -0,0 +1,258 @@
+"""
+SAM3 Server - API endpoints for interactive point selection
+
+Provides endpoints for the SAM3 interactive editor:
+- /sam3/prepare: Load model and image
+- /sam3/detect: Run SAM3 with point prompts
+"""
+
+import io
+import json
+import os
+import threading
+import numpy as np
+import torch
+from PIL import Image
+from aiohttp import web
+from server import PromptServer
+import folder_paths
+
+# Global state for SAM3 predictor
+sam3_predictor = None
+sam3_lock = threading.Condition()
+
+
+def load_image_from_path(filename, file_type, subfolder=""):
+    """Load image from ComfyUI storage"""
+    # Get the appropriate directory based on type
+    if file_type == "input":
+        input_dir = folder_paths.get_input_directory()
+    elif file_type == "temp":
+        input_dir = folder_paths.get_temp_directory()
+    elif file_type == "output":
+        input_dir = folder_paths.get_output_directory()
+    else:
+        input_dir = folder_paths.get_input_directory()
+
+    # Build full path
+    if subfolder:
+        full_path = os.path.join(input_dir, subfolder, filename)
+    else:
+        full_path = os.path.join(input_dir, filename)
+
+    print(f"[SAM3 Server] Loading image from: {full_path}")
+
+    # Load image
+    image = Image.open(full_path)
+    if image.mode != 'RGB':
+        image = image.convert('RGB')
+
+    return image
+
+
+@PromptServer.instance.routes.post("/sam3/prepare")
+async def sam3_prepare(request):
+    """
+    Load SAM3 model and prepare image for interactive editing
+
+    Request JSON:
+    {
+        "sam_model_name": "auto",
+        "filename": "image.png",
+        "type": "temp",
+        "subfolder": ""
+    }
+    """
+    global sam3_predictor
+
+    try:
+        data = await request.json()
+        filename = data.get("filename", "")
+        file_type = data.get("type", "temp")
+        subfolder = data.get("subfolder", "")
+
+        print(f"[SAM3 Server] Preparing SAM3 for image: {filename}")
+
+        # Load image
+        image = load_image_from_path(filename, file_type, subfolder)
+
+        # Load SAM3 model if not already loaded
+        # For now, we'll load the model on-demand when /sam3/detect is called
+        # This is because the model is managed by LoadSAM3Model node
+
+        with sam3_lock:
+            sam3_predictor = {
+                "image": image,
+                "filename": filename,
+                "type": file_type,
+                "subfolder": subfolder,
+            }
+            print(f"[SAM3 Server] Image prepared: {image.size}")
+
+        return web.Response(status=200)
+
+    except Exception as e:
+        print(f"[SAM3 Server] Error in /sam3/prepare: {e}")
+        import traceback
+        traceback.print_exc()
+        return web.Response(status=500, text=str(e))
+
+
+@PromptServer.instance.routes.post("/sam3/detect")
+async def sam3_detect(request):
+    """
+    Run SAM3 detection with point prompts
+
+    Request JSON:
+    {
+        "positive_points": [[x1, y1], [x2, y2], ...],
+        "negative_points": [[x3, y3], ...],
+        "threshold": 0.7
+    }
+
+    Returns: PNG image of the mask
+    """
+    global sam3_predictor
+
+    try:
+        data = await request.json()
+        positive_points = data.get("positive_points", [])
+        negative_points = data.get("negative_points", [])
+        threshold = data.get("threshold", 0.7)
+
+        print(f"[SAM3 Server] Detecting with {len(positive_points)} positive and {len(negative_points)} negative points")
+
+        if sam3_predictor is None:
+            return web.Response(status=400, text="SAM3 not prepared. Call /sam3/prepare first.")
+
+        # Get the stored image
+        image = sam3_predictor.get("image")
+        if image is None:
+            return web.Response(status=400, text="No image loaded")
+
+        # Load SAM3 model on-demand
+        # We need to import here to avoid circular imports
+        from .sam3_lib.model_builder import build_sam3_image_model
+        from .sam3_lib.model.sam3_image_processor import SAM3ImageProcessor
+
+        # Build model (cached globally if possible)
+        if "model" not in sam3_predictor:
+            print(f"[SAM3 Server] Loading SAM3 model...")
+            model = build_sam3_image_model()
+            device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+            model = model.to(device).eval()
+
+            processor = SAM3ImageProcessor(model, device=device)
+
+            sam3_predictor["model"] = model
+            sam3_predictor["processor"] = processor
+            sam3_predictor["device"] = device
+            print(f"[SAM3 Server] SAM3 model loaded on {device}")
+
+        processor = sam3_predictor["processor"]
+        device = sam3_predictor["device"]
+
+        # Set image and extract features
+        print(f"[SAM3 Server] Setting image...")
+        state = processor.set_image(image)
+
+        # Prepare points and labels
+        all_points = []
+        all_labels = []
+
+        # Add positive points (label=1)
+        for point in positive_points:
+            all_points.append(point)
+            all_labels.append(1)
+
+        # Add negative points (label=0)
+        for point in negative_points:
+            all_points.append(point)
+            all_labels.append(0)
+
+        print(f"[SAM3 Server] Total points: {len(all_points)}")
+
+        # Run detection with points if we have any
+        if len(all_points) > 0:
+            # Convert to tensors
+            points_tensor = torch.tensor(all_points, dtype=torch.float32).unsqueeze(0)  # [1, N, 2]
+            labels_tensor = torch.tensor(all_labels, dtype=torch.int32).unsqueeze(0)  # [1, N]
+
+            print(f"[SAM3 Server] Running point prompt detection...")
+            state = processor.add_point_prompt(points_tensor, labels_tensor, state)
+
+            # Get masks from state
+            masks = state.get("masks", None)
+            scores = state.get("scores", None)
+
+            if masks is None or len(masks) == 0:
+                print(f"[SAM3 Server] No masks generated")
+                # Return empty mask
+                mask_array = np.zeros((image.height, image.width), dtype=np.uint8)
+            else:
+                print(f"[SAM3 Server] Got {len(masks)} masks, scores: {scores}")
+
+                # Filter by threshold if scores available
+                if scores is not None:
+                    valid_indices = scores > threshold
+                    if valid_indices.sum() > 0:
+                        masks = masks[valid_indices]
+                        scores = scores[valid_indices]
+                        print(f"[SAM3 Server] Filtered to {len(masks)} masks above threshold {threshold}")
+                    else:
+                        # Take best mask if none above threshold
+                        best_idx = torch.argmax(scores)
+                        masks = masks[best_idx:best_idx+1]
+                        scores = scores[best_idx:best_idx+1]
+                        print(f"[SAM3 Server] No masks above threshold, using best mask with score {scores[0]:.3f}")
+
+                # Combine multiple masks using bitwise OR
+                mask_array = np.zeros((image.height, image.width), dtype=np.uint8)
+                for mask in masks:
+                    # Convert tensor to numpy
+                    if isinstance(mask, torch.Tensor):
+                        mask_np = mask.cpu().numpy()
+                    else:
+                        mask_np = mask
+
+                    # Ensure correct shape
+                    if mask_np.ndim == 3:
+                        mask_np = mask_np[0]  # Remove batch dimension
+
+                    # Resize if needed
+                    if mask_np.shape != (image.height, image.width):
+                        from PIL import Image as PILImage
+                        mask_pil = PILImage.fromarray((mask_np > 0).astype(np.uint8) * 255)
+                        mask_pil = mask_pil.resize((image.width, image.height), PILImage.NEAREST)
+                        mask_np = np.array(mask_pil) > 0
+
+                    # Combine with OR
+                    mask_array = np.logical_or(mask_array, mask_np > 0).astype(np.uint8) * 255
+
+        else:
+            print(f"[SAM3 Server] No points provided, returning empty mask")
+            mask_array = np.zeros((image.height, image.width), dtype=np.uint8)
+
+        # Convert mask to PIL Image
+        mask_image = Image.fromarray(mask_array, mode='L')
+
+        # Convert to PNG bytes
+        img_buffer = io.BytesIO()
+        mask_image.save(img_buffer, format='PNG')
+        img_buffer.seek(0)
+
+        print(f"[SAM3 Server] Returning mask PNG")
+
+        return web.Response(
+            body=img_buffer.getvalue(),
+            headers={'Content-Type': 'image/png'}
+        )
+
+    except Exception as e:
+        print(f"[SAM3 Server] Error in /sam3/detect: {e}")
+        import traceback
+        traceback.print_exc()
+        return web.Response(status=500, text=str(e))
+
+
+print("[SAM3 Server] Registered routes: /sam3/prepare, /sam3/detect")

web/sam3-editor.js

@@ -0,0 +1,640 @@
+import { app } from "../../scripts/app.js";
+import { api } from "../../scripts/api.js";
+import { ComfyDialog, $el } from "../../scripts/ui.js";
+import { ComfyApp } from "../../scripts/app.js";
+import { ClipspaceDialog } from "../../extensions/core/clipspace.js";
+
+function addMenuHandler(nodeType, cb) {
+	const getOpts = nodeType.prototype.getExtraMenuOptions;
+	nodeType.prototype.getExtraMenuOptions = function () {
+		const r = getOpts.apply(this, arguments);
+		cb.apply(this, arguments);
+		return r;
+	};
+}
+
+// Helper function to convert a data URL to a Blob object
+function dataURLToBlob(dataURL) {
+	const parts = dataURL.split(';base64,');
+	const contentType = parts[0].split(':')[1];
+	const byteString = atob(parts[1]);
+	const arrayBuffer = new ArrayBuffer(byteString.length);
+	const uint8Array = new Uint8Array(arrayBuffer);
+	for (let i = 0; i < byteString.length; i++) {
+		uint8Array[i] = byteString.charCodeAt(i);
+	}
+	return new Blob([arrayBuffer], { type: contentType });
+}
+
+function loadedImageToBlob(image) {
+	const canvas = document.createElement('canvas');
+
+	canvas.width = image.width;
+	canvas.height = image.height;
+
+	const ctx = canvas.getContext('2d');
+
+	ctx.drawImage(image, 0, 0);
+
+	const dataURL = canvas.toDataURL('image/png', 1);
+	const blob = dataURLToBlob(dataURL);
+
+	return blob;
+}
+
+async function uploadMask(filepath, formData) {
+	await api.fetchApi('/upload/mask', {
+		method: 'POST',
+		body: formData
+	}).then(response => {}).catch(error => {
+		console.error('Error:', error);
+	});
+
+	ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']] = new Image();
+	ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src = `view?filename=${filepath.filename}&type=${filepath.type}`;
+
+	if(ComfyApp.clipspace.images)
+		ComfyApp.clipspace.images[ComfyApp.clipspace['selectedIndex']] = filepath;
+
+	ClipspaceDialog.invalidatePreview();
+}
+
+class SAM3EditorDialog extends ComfyDialog {
+	static instance = null;
+
+	static getInstance() {
+		if(!SAM3EditorDialog.instance) {
+			SAM3EditorDialog.instance = new SAM3EditorDialog();
+		}
+
+		return SAM3EditorDialog.instance;
+	}
+
+	constructor() {
+		super();
+		this.element = $el("div.comfy-modal", { parent: document.body },
+			[ $el("div.comfy-modal-content",
+				[...this.createButtons()]),
+			]);
+	}
+
+	createButtons() {
+		return [];
+	}
+
+	createButton(name, callback) {
+		var button = document.createElement("button");
+		button.innerText = name;
+		button.addEventListener("click", callback);
+		return button;
+	}
+
+	createLeftButton(name, callback) {
+		var button = this.createButton(name, callback);
+		button.style.cssFloat = "left";
+		button.style.marginRight = "4px";
+		return button;
+	}
+
+	createRightButton(name, callback) {
+		var button = this.createButton(name, callback);
+		button.style.cssFloat = "right";
+		button.style.marginLeft = "4px";
+		return button;
+	}
+
+	createLeftSlider(self, name, callback) {
+		const divElement = document.createElement('div');
+		divElement.id = "sam-confidence-slider";
+		divElement.style.cssFloat = "left";
+		divElement.style.fontFamily = "sans-serif";
+		divElement.style.marginRight = "4px";
+		divElement.style.color = "var(--input-text)";
+		divElement.style.backgroundColor = "var(--comfy-input-bg)";
+		divElement.style.borderRadius = "8px";
+		divElement.style.borderColor = "var(--border-color)";
+		divElement.style.borderStyle = "solid";
+		divElement.style.fontSize = "15px";
+		divElement.style.height = "21px";
+		divElement.style.padding = "1px 6px";
+		divElement.style.display = "flex";
+		divElement.style.position = "relative";
+		divElement.style.top = "2px";
+		self.confidence_slider_input = document.createElement('input');
+		self.confidence_slider_input.setAttribute('type', 'range');
+		self.confidence_slider_input.setAttribute('min', '0');
+		self.confidence_slider_input.setAttribute('max', '100');
+		self.confidence_slider_input.setAttribute('value', '70');
+		const labelElement = document.createElement("label");
+		labelElement.textContent = name;
+
+		divElement.appendChild(labelElement);
+		divElement.appendChild(self.confidence_slider_input);
+
+		self.confidence_slider_input.addEventListener("change", callback);
+
+		return divElement;
+	}
+
+	async detect_and_invalidate_mask_canvas(self) {
+		const mask_img = await self.detect(self);
+
+		const canvas = self.maskCtx.canvas;
+		const ctx = self.maskCtx;
+
+		ctx.clearRect(0, 0, canvas.width, canvas.height);
+
+		await new Promise((resolve, reject) => {
+						self.mask_image = new Image();
+						self.mask_image.onload = function() {
+							ctx.drawImage(self.mask_image, 0, 0, canvas.width, canvas.height);
+							resolve();
+						};
+						self.mask_image.onerror = reject;
+						self.mask_image.src = mask_img.src;
+				});
+	}
+
+	setlayout(imgCanvas, maskCanvas, pointsCanvas) {
+		const self = this;
+
+		// If it is specified as relative, using it only as a hidden placeholder for padding is recommended
+		// to prevent anomalies where it exceeds a certain size and goes outside of the window.
+		var placeholder = document.createElement("div");
+		placeholder.style.position = "relative";
+		placeholder.style.height = "50px";
+
+		var bottom_panel = document.createElement("div");
+		bottom_panel.style.position = "absolute";
+		bottom_panel.style.bottom = "0px";
+		bottom_panel.style.left = "20px";
+		bottom_panel.style.right = "20px";
+		bottom_panel.style.height = "50px";
+
+		var brush = document.createElement("div");
+		brush.id = "sam-brush";
+		brush.style.backgroundColor = "blue";
+		brush.style.outline = "2px solid pink";
+		brush.style.borderRadius = "50%";
+		brush.style.MozBorderRadius = "50%";
+		brush.style.WebkitBorderRadius = "50%";
+		brush.style.position = "absolute";
+		brush.style.zIndex = 100;
+		brush.style.pointerEvents = "none";
+		this.brush = brush;
+		this.element.appendChild(imgCanvas);
+		this.element.appendChild(maskCanvas);
+		this.element.appendChild(pointsCanvas);
+		this.element.appendChild(placeholder); // must below z-index than bottom_panel to avoid covering button
+		this.element.appendChild(bottom_panel);
+		document.body.appendChild(brush);
+		this.brush_size = 5;
+
+		var confidence_slider = this.createLeftSlider(self, "Confidence", (event) => {
+			self.confidence = event.target.value;
+		});
+
+		var clearButton = this.createLeftButton("Clear", () => {
+				self.maskCtx.clearRect(0, 0, self.maskCanvas.width, self.maskCanvas.height);
+				self.pointsCtx.clearRect(0, 0, self.pointsCanvas.width, self.pointsCanvas.height);
+
+				self.prompt_points = [];
+
+				self.invalidatePointsCanvas(self);
+			});
+
+		var detectButton = this.createLeftButton("Detect", () => self.detect_and_invalidate_mask_canvas(self));
+
+		var cancelButton = this.createRightButton("Cancel", () => {
+				document.removeEventListener("mouseup", SAM3EditorDialog.handleMouseUp);
+				document.removeEventListener("keydown", SAM3EditorDialog.handleKeyDown);
+				self.close();
+			});
+
+		self.saveButton = this.createRightButton("Save", () => {
+				document.removeEventListener("mouseup", SAM3EditorDialog.handleMouseUp);
+				document.removeEventListener("keydown", SAM3EditorDialog.handleKeyDown);
+				self.save(self);
+			});
+
+		var undoButton = this.createLeftButton("Undo", () => {
+				if(self.prompt_points.length > 0) {
+					self.prompt_points.pop();
+					self.pointsCtx.clearRect(0, 0, self.pointsCanvas.width, self.pointsCanvas.height);
+					self.invalidatePointsCanvas(self);
+				}
+			});
+
+		bottom_panel.appendChild(clearButton);
+		bottom_panel.appendChild(detectButton);
+		bottom_panel.appendChild(self.saveButton);
+		bottom_panel.appendChild(cancelButton);
+		bottom_panel.appendChild(confidence_slider);
+		bottom_panel.appendChild(undoButton);
+
+		imgCanvas.style.position = "relative";
+		imgCanvas.style.top = "200";
+		imgCanvas.style.left = "0";
+
+		maskCanvas.style.position = "absolute";
+		maskCanvas.style.opacity = 0.5;
+		pointsCanvas.style.position = "absolute";
+	}
+
+	show() {
+		this.mask_image = null;
+		self.prompt_points = [];
+
+		this.message_box = $el("p", ["Please wait a moment while the SAM3 model and the image are being loaded."]);
+		this.element.appendChild(this.message_box);
+
+		if(self.imgCtx) {
+			self.imgCtx.clearRect(0, 0, self.imageCanvas.width, self.imageCanvas.height);
+		}
+
+		const target_image_path = ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src;
+		this.load_sam(target_image_path);
+
+		if(!this.is_layout_created) {
+			// layout
+			const imgCanvas = document.createElement('canvas');
+			const maskCanvas = document.createElement('canvas');
+			const pointsCanvas = document.createElement('canvas');
+
+			imgCanvas.id = "imageCanvas";
+			maskCanvas.id = "samEditorMaskCanvas";
+			pointsCanvas.id = "pointsCanvas";
+
+			this.setlayout(imgCanvas, maskCanvas, pointsCanvas);
+
+			// prepare content
+			this.imgCanvas = imgCanvas;
+			this.maskCanvas = maskCanvas;
+			this.pointsCanvas = pointsCanvas;
+			this.maskCtx = maskCanvas.getContext('2d');
+			this.pointsCtx = pointsCanvas.getContext('2d');
+
+			this.is_layout_created = true;
+
+			// replacement of onClose hook since close is not real close
+			const self = this;
+			const observer = new MutationObserver(function(mutations) {
+			mutations.forEach(function(mutation) {
+					if (mutation.type === 'attributes' && mutation.attributeName === 'style') {
+						if(self.last_display_style && self.last_display_style != 'none' && self.element.style.display == 'none') {
+							ComfyApp.onClipspaceEditorClosed();
+						}
+
+						self.last_display_style = self.element.style.display;
+					}
+				});
+			});
+
+			const config = { attributes: true };
+			observer.observe(this.element, config);
+		}
+
+		this.setImages(target_image_path, this.imgCanvas, this.pointsCanvas);
+
+		if(ComfyApp.clipspace_return_node) {
+			this.saveButton.innerText = "Save to node";
+		}
+		else {
+			this.saveButton.innerText = "Save";
+		}
+		this.saveButton.disabled = true;
+
+		this.element.style.display = "block";
+		this.element.style.zIndex = 8888; // NOTE: alert dialog must be high priority.
+	}
+
+	updateBrushPreview(self, event) {
+		event.preventDefault();
+
+		const centerX = event.pageX;
+		const centerY = event.pageY;
+
+		const brush = self.brush;
+
+		brush.style.width = self.brush_size * 2 + "px";
+		brush.style.height = self.brush_size * 2 + "px";
+		brush.style.left = (centerX - self.brush_size) + "px";
+		brush.style.top = (centerY - self.brush_size) + "px";
+	}
+
+	setImages(target_image_path, imgCanvas, pointsCanvas) {
+		const imgCtx = imgCanvas.getContext('2d');
+		const maskCtx = this.maskCtx;
+		const maskCanvas = this.maskCanvas;
+
+		const self = this;
+
+		// image load
+		const orig_image = new Image();
+		window.addEventListener("resize", () => {
+			// repositioning
+			imgCanvas.width = window.innerWidth - 250;
+			imgCanvas.height = window.innerHeight - 200;
+
+			// redraw image
+			let drawWidth = orig_image.width;
+			let drawHeight = orig_image.height;
+
+			if (orig_image.width > imgCanvas.width) {
+				drawWidth = imgCanvas.width;
+				drawHeight = (drawWidth / orig_image.width) * orig_image.height;
+			}
+
+			if (drawHeight > imgCanvas.height) {
+				drawHeight = imgCanvas.height;
+				drawWidth = (drawHeight / orig_image.height) * orig_image.width;
+			}
+
+			imgCtx.drawImage(orig_image, 0, 0, drawWidth, drawHeight);
+
+			// update mask
+			let w = (drawWidth * imgCanvas.clientWidth/imgCanvas.width) + "px";
+			let h = (drawHeight * imgCanvas.clientHeight/imgCanvas.height) + "px";
+
+			pointsCanvas.width = drawWidth * imgCanvas.clientWidth/imgCanvas.width;
+			pointsCanvas.height = drawHeight * imgCanvas.clientHeight/imgCanvas.height;
+			pointsCanvas.style.top = imgCanvas.offsetTop + "px";
+			pointsCanvas.style.left = imgCanvas.offsetLeft + "px";
+
+			maskCanvas.width = pointsCanvas.width;
+			maskCanvas.height = pointsCanvas.height;
+			maskCanvas.style.top = imgCanvas.offsetTop + "px";
+			maskCanvas.style.left = imgCanvas.offsetLeft + "px";
+
+			self.invalidateMaskCanvas(self);
+			self.invalidatePointsCanvas(self);
+		});
+
+		// original image load
+		orig_image.onload = () => self.onLoaded(self);
+		const rgb_url = new URL(target_image_path);
+		rgb_url.searchParams.delete('channel');
+		rgb_url.searchParams.set('channel', 'rgb');
+		orig_image.src = rgb_url;
+		self.image = orig_image;
+	}
+
+	onLoaded(self) {
+		if(self.message_box) {
+			self.element.removeChild(self.message_box);
+			self.message_box = null;
+		}
+
+		window.dispatchEvent(new Event('resize'));
+
+		self.setEventHandler(pointsCanvas);
+		self.saveButton.disabled = false;
+	}
+
+	setEventHandler(targetCanvas) {
+		targetCanvas.addEventListener("contextmenu", (event) => {
+			event.preventDefault();
+		});
+
+		const self = this;
+		targetCanvas.addEventListener('pointermove', (event) => this.updateBrushPreview(self,event));
+		targetCanvas.addEventListener('pointerdown', (event) => this.handlePointerDown(self,event));
+		targetCanvas.addEventListener('pointerover', (event) => { this.brush.style.display = "block"; });
+		targetCanvas.addEventListener('pointerleave', (event) => { this.brush.style.display = "none"; });
+		document.addEventListener('keydown', SAM3EditorDialog.handleKeyDown);
+	}
+
+	static handleKeyDown(event) {
+		const self = SAM3EditorDialog.instance;
+		if (event.key === '=') { // positive
+			brush.style.backgroundColor = "blue";
+			brush.style.outline = "2px solid pink";
+			self.is_positive_mode = true;
+		} else if (event.key === '-') { // negative
+			brush.style.backgroundColor = "red";
+			brush.style.outline = "2px solid skyblue";
+			self.is_positive_mode = false;
+		}
+	}
+
+	is_positive_mode = true;
+	prompt_points = [];
+	confidence = 70;
+
+	invalidatePointsCanvas(self) {
+		const ctx = self.pointsCtx;
+
+		for (const i in self.prompt_points) {
+			const [is_positive, x, y] = self.prompt_points[i];
+
+			const scaledX = x * ctx.canvas.width / self.image.width;
+			const scaledY = y * ctx.canvas.height / self.image.height;
+
+			if(is_positive)
+				ctx.fillStyle = "blue";
+			else
+				ctx.fillStyle = "red";
+			ctx.beginPath();
+			ctx.arc(scaledX, scaledY, 3, 0, 3 * Math.PI);
+			ctx.fill();
+		}
+	}
+
+	invalidateMaskCanvas(self) {
+		if(self.mask_image) {
+			self.maskCtx.clearRect(0, 0, self.maskCanvas.width, self.maskCanvas.height);
+			self.maskCtx.drawImage(self.mask_image, 0, 0, self.maskCanvas.width, self.maskCanvas.height);
+		}
+	}
+
+	async load_sam(url) {
+		const parsedUrl = new URL(url);
+		const searchParams = new URLSearchParams(parsedUrl.search);
+
+		const filename = searchParams.get("filename") || "";
+		const fileType = searchParams.get("type") || "";
+		const subfolder = searchParams.get("subfolder") || "";
+
+		const data = {
+				sam_model_name: "auto",
+				filename: filename,
+				type: fileType,
+				subfolder: subfolder
+			};
+
+		api.fetchApi('/sam3/prepare', {
+			method: 'POST',
+			headers: { 'Content-Type': 'application/json' },
+			body: JSON.stringify(data)
+		});
+	}
+
+	async detect(self) {
+		const positive_points = [];
+		const negative_points = [];
+
+		for(const i in self.prompt_points) {
+			const [is_positive, x, y] = self.prompt_points[i];
+			const point = [x,y];
+			if(is_positive) {
+				positive_points.push(point);
+			}
+			else
+				negative_points.push(point);
+		}
+
+		const data = {
+			positive_points: positive_points,
+			negative_points: negative_points,
+			threshold: self.confidence/100
+		};
+
+		const response = await api.fetchApi('/sam3/detect', {
+			method: 'POST',
+			headers: { 'Content-Type': 'image/png' },
+			body: JSON.stringify(data)
+		});
+
+		const blob = await response.blob();
+		const url = URL.createObjectURL(blob);
+
+		return new Promise((resolve, reject) => {
+			const image = new Image();
+			image.onload = () => resolve(image);
+			image.onerror = reject;
+			image.src = url;
+		});
+	}
+
+	handlePointerDown(self, event) {
+		if ([0, 2, 5].includes(event.button)) {
+			event.preventDefault();
+			const x = event.offsetX || event.targetTouches[0].clientX - maskRect.left;
+			const y = event.offsetY || event.targetTouches[0].clientY - maskRect.top;
+
+			const originalX = x * self.image.width / self.pointsCanvas.clientWidth;
+			const originalY = y * self.image.height / self.pointsCanvas.clientHeight;
+
+			var point = null;
+			if (event.button == 0) {
+				// positive
+				point = [true, originalX, originalY];
+			} else {
+				// negative
+				point = [false, originalX, originalY];
+			}
+
+			self.prompt_points.push(point);
+
+			self.invalidatePointsCanvas(self);
+		}
+	}
+
+	async save(self) {
+		if(!self.mask_image) {
+			this.close();
+			return;
+		}
+
+		const save_canvas = document.createElement('canvas');
+
+		const save_ctx = save_canvas.getContext('2d', {willReadFrequently:true});
+		save_canvas.width = self.mask_image.width;
+		save_canvas.height = self.mask_image.height;
+
+		save_ctx.drawImage(self.mask_image, 0, 0, save_canvas.width, save_canvas.height);
+
+		const save_data = save_ctx.getImageData(0, 0, save_canvas.width, save_canvas.height);
+
+		// refine mask image
+		for (let i = 0; i < save_data.data.length; i += 4) {
+			if(save_data.data[i]) {
+				save_data.data[i+3] = 0;
+			}
+			else {
+				save_data.data[i+3] = 255;
+			}
+
+			save_data.data[i] = 0;
+			save_data.data[i+1] = 0;
+			save_data.data[i+2] = 0;
+		}
+
+		save_ctx.globalCompositeOperation = 'source-over';
+		save_ctx.putImageData(save_data, 0, 0);
+
+		const formData = new FormData();
+		const filename = "clipspace-mask-" + performance.now() + ".png";
+
+		const item =
+			{
+				"filename": filename,
+				"subfolder": "",
+				"type": "temp",
+			};
+
+		if(ComfyApp.clipspace.images)
+			ComfyApp.clipspace.images[0] = item;
+
+		if(ComfyApp.clipspace.widgets) {
+			const index = ComfyApp.clipspace.widgets.findIndex(obj => obj.name === 'image');
+
+			if(index >= 0)
+				ComfyApp.clipspace.widgets[index].value = `${filename} [temp]`;
+		}
+
+		const dataURL = save_canvas.toDataURL();
+		const blob = dataURLToBlob(dataURL);
+
+		let original_url = new URL(this.image.src);
+
+		const original_ref = { filename: original_url.searchParams.get('filename') };
+
+		let original_subfolder = original_url.searchParams.get("subfolder");
+		if(original_subfolder)
+			original_ref.subfolder = original_subfolder;
+
+		let original_type = original_url.searchParams.get("type");
+		if(original_type)
+			original_ref.type = original_type;
+
+		formData.append('image', blob, filename);
+		formData.append('original_ref', JSON.stringify(original_ref));
+		formData.append('type', "temp");
+
+		await uploadMask(item, formData);
+		ComfyApp.onClipspaceEditorSave();
+		this.close();
+	}
+}
+
+app.registerExtension({
+	name: "Comfy.SAM3.Editor",
+	init(app) {
+		const callback =
+			function () {
+				let dlg = SAM3EditorDialog.getInstance();
+				dlg.show();
+			};
+
+		const context_predicate = () => ComfyApp.clipspace && ComfyApp.clipspace.imgs && ComfyApp.clipspace.imgs.length > 0
+		ClipspaceDialog.registerButton("SAM3 Detector", context_predicate, callback);
+	},
+
+	async beforeRegisterNodeDef(nodeType, nodeData, app) {
+		if (Array.isArray(nodeData.output) && (nodeData.output.includes("MASK") || nodeData.output.includes("IMAGE"))) {
+			addMenuHandler(nodeType, function (_, options) {
+				options.unshift({
+					content: "Open in SAM3 Detector",
+					callback: () => {
+						ComfyApp.copyToClipspace(this);
+						ComfyApp.clipspace_return_node = this;
+
+						let dlg = SAM3EditorDialog.getInstance();
+						dlg.show();
+					},
+				});
+			});
+		}
+	}
+});

web/sam3_points_widget.js

@@ -0,0 +1,423 @@
+/**
+ * SAM3 Simple Point Collector
+ * Uses plain HTML5 Canvas instead of Protovis for better compatibility
+ * Version: 2025-01-20-v8-SERIALIZE-FIX
+ */
+
+import { app } from "../../scripts/app.js";
+import { ComfyWidgets } from "../../scripts/widgets.js";
+
+console.log("[SAM3] ===== VERSION 8 - SERIALIZATION ENABLED =====");
+
+// Helper function to properly hide widgets (enhanced for complete hiding)
+function hideWidgetForGood(node, widget, suffix = '') {
+    if (!widget) return;
+
+    // Save original properties
+    widget.origType = widget.type;
+    widget.origComputeSize = widget.computeSize;
+    widget.origSerializeValue = widget.serializeValue;
+
+    // Multiple hiding approaches to ensure widget is fully hidden
+    widget.computeSize = () => [0, -4];  // -4 compensates for litegraph's automatic widget gap
+    widget.type = "converted-widget" + suffix;
+    widget.hidden = true;  // Mark as hidden
+
+    // IMPORTANT: Keep serialization enabled so values are sent to backend
+    // (We just hide it visually, but it still needs to send data)
+
+    // Make the widget completely invisible in the DOM if it has element
+    if (widget.element) {
+        widget.element.style.display = "none";
+        widget.element.style.visibility = "hidden";
+    }
+
+    // Handle linked widgets recursively
+    if (widget.linkedWidgets) {
+        for (const w of widget.linkedWidgets) {
+            hideWidgetForGood(node, w, ':' + widget.name);
+        }
+    }
+}
+
+app.registerExtension({
+    name: "Comfy.SAM3.SimplePointCollector",
+
+    async beforeRegisterNodeDef(nodeType, nodeData, app) {
+        console.log("[SAM3] beforeRegisterNodeDef called for:", nodeData.name);
+
+        if (nodeData.name === "SAM3PointCollector") {
+            console.log("[SAM3] Registering SAM3PointCollector node");
+            const onNodeCreated = nodeType.prototype.onNodeCreated;
+
+            nodeType.prototype.onNodeCreated = function () {
+                console.log("[SAM3] onNodeCreated called for SAM3PointCollector");
+
+                // Call original onNodeCreated FIRST to create all widgets
+                const result = onNodeCreated?.apply(this, arguments);
+
+                console.log("[SAM3] Widgets after creation:", this.widgets?.map(w => w.name));
+
+                console.log("[SAM3] Creating canvas container");
+                // Create canvas container - dynamically sized based on node height
+                const container = document.createElement("div");
+                container.style.cssText = "position: relative; width: 100%; background: #222; overflow: hidden; box-sizing: border-box; margin: 0; padding: 0; display: flex; align-items: center; justify-content: center;";
+
+                // Create info/button bar
+                const infoBar = document.createElement("div");
+                infoBar.style.cssText = "position: absolute; top: 5px; left: 5px; right: 5px; z-index: 10; display: flex; justify-content: space-between; align-items: center;";
+                container.appendChild(infoBar);
+
+                // Create points counter
+                const pointsCounter = document.createElement("div");
+                pointsCounter.style.cssText = "padding: 5px 10px; background: rgba(0,0,0,0.7); color: #fff; border-radius: 3px; font-size: 12px; font-family: monospace;";
+                pointsCounter.textContent = "Points: 0 pos, 0 neg";
+                infoBar.appendChild(pointsCounter);
+
+                // Create clear button
+                const clearButton = document.createElement("button");
+                clearButton.textContent = "Clear All";
+                clearButton.style.cssText = "padding: 5px 10px; background: #d44; color: #fff; border: 1px solid #a22; border-radius: 3px; cursor: pointer; font-size: 12px; font-weight: bold;";
+                clearButton.onmouseover = () => clearButton.style.background = "#e55";
+                clearButton.onmouseout = () => clearButton.style.background = "#d44";
+                infoBar.appendChild(clearButton);
+
+                // Create canvas for image and points
+                const canvas = document.createElement("canvas");
+                canvas.width = 512;
+                canvas.height = 512;
+                // Use max-width and max-height instead of width/height 100% to prevent overflow
+                canvas.style.cssText = "display: block; max-width: 100%; max-height: 100%; object-fit: contain; cursor: crosshair; margin: 0 auto;";
+                container.appendChild(canvas);
+
+                const ctx = canvas.getContext("2d");
+                console.log("[SAM3] Canvas created:", canvas);
+
+                // Store state
+                this.canvasWidget = {
+                    canvas: canvas,
+                    ctx: ctx,
+                    container: container,
+                    image: null,
+                    positivePoints: [],
+                    negativePoints: [],
+                    hoveredPoint: null,
+                    pointsCounter: pointsCounter
+                };
+
+                // Add as DOM widget
+                console.log("[SAM3] Adding DOM widget via addDOMWidget");
+                const widget = this.addDOMWidget("canvas", "customCanvas", container);
+                console.log("[SAM3] addDOMWidget returned:", widget);
+
+                // Store widget reference for updates
+                this.canvasWidget.domWidget = widget;
+
+                // Make widget dynamically sized - override computeSize
+                widget.computeSize = (width) => {
+                    // Widget height = node height - title bar/padding (approx 80px)
+                    const nodeHeight = this.size ? this.size[1] : 480;
+                    const widgetHeight = Math.max(200, nodeHeight - 80);
+                    return [width, widgetHeight];
+                };
+
+                // Clear button handler
+                clearButton.addEventListener("click", (e) => {
+                    e.preventDefault();
+                    e.stopPropagation();
+                    console.log("[SAM3] Clearing all points");
+                    this.canvasWidget.positivePoints = [];
+                    this.canvasWidget.negativePoints = [];
+                    this.updatePoints();
+                    this.redrawCanvas();
+                });
+
+                // Hide the string storage widgets - multiple approaches
+                console.log("[SAM3] Attempting to hide widgets...");
+                console.log("[SAM3] Widgets before hiding:", this.widgets.map(w => w.name));
+
+                const coordsWidget = this.widgets.find(w => w.name === "coordinates");
+                const negCoordsWidget = this.widgets.find(w => w.name === "neg_coordinates");
+                const storeWidget = this.widgets.find(w => w.name === "points_store");
+
+                console.log("[SAM3] Found widgets to hide:", { coordsWidget, negCoordsWidget, storeWidget });
+
+                // Initialize default values BEFORE hiding
+                if (coordsWidget) {
+                    coordsWidget.value = coordsWidget.value || "[]";
+                }
+                if (negCoordsWidget) {
+                    negCoordsWidget.value = negCoordsWidget.value || "[]";
+                }
+                if (storeWidget) {
+                    storeWidget.value = storeWidget.value || "{}";
+                }
+
+                // Store references before hiding
+                this._hiddenWidgets = {
+                    coordinates: coordsWidget,
+                    neg_coordinates: negCoordsWidget,
+                    points_store: storeWidget
+                };
+
+                // Apply hiding
+                if (coordsWidget) {
+                    hideWidgetForGood(this, coordsWidget);
+                    console.log("[SAM3] coordinates - type:", coordsWidget.type, "hidden:", coordsWidget.hidden, "value:", coordsWidget.value);
+                }
+                if (negCoordsWidget) {
+                    hideWidgetForGood(this, negCoordsWidget);
+                    console.log("[SAM3] neg_coordinates - type:", negCoordsWidget.type, "hidden:", negCoordsWidget.hidden, "value:", negCoordsWidget.value);
+                }
+                if (storeWidget) {
+                    hideWidgetForGood(this, storeWidget);
+                    console.log("[SAM3] points_store - type:", storeWidget.type, "hidden:", storeWidget.hidden, "value:", storeWidget.value);
+                }
+
+                // CRITICAL FIX: Override onDrawForeground to skip rendering hidden widgets
+                const originalDrawForeground = this.onDrawForeground;
+                this.onDrawForeground = function(ctx) {
+                    // Temporarily hide converted widgets from rendering
+                    const hiddenWidgets = this.widgets.filter(w => w.type?.includes("converted-widget"));
+                    const originalTypes = hiddenWidgets.map(w => w.type);
+
+                    // Temporarily set to null to prevent rendering
+                    hiddenWidgets.forEach(w => w.type = null);
+
+                    // Call original draw
+                    if (originalDrawForeground) {
+                        originalDrawForeground.apply(this, arguments);
+                    }
+
+                    // Restore types
+                    hiddenWidgets.forEach((w, i) => w.type = originalTypes[i]);
+
+                    // Update container height based on current node size
+                    const containerHeight = Math.max(200, this.size[1] - 80);
+                    if (container.style.height !== containerHeight + "px") {
+                        container.style.height = containerHeight + "px";
+                    }
+                };
+
+                console.log("[SAM3] Widgets after hiding:", this.widgets.map(w => `${w.name}(${w.type})`));
+                console.log("[SAM3] All widgets processing complete");
+
+                // Mouse event handlers
+                canvas.addEventListener("click", (e) => {
+                    const rect = canvas.getBoundingClientRect();
+                    // Calculate coordinates relative to the actual image dimensions
+                    // The canvas might be scaled, so we need to map from display coords to image coords
+                    const x = ((e.clientX - rect.left) / rect.width) * canvas.width;
+                    const y = ((e.clientY - rect.top) / rect.height) * canvas.height;
+                    console.log(`[SAM3] Click at canvas coords: (${x.toFixed(1)}, ${y.toFixed(1)}), canvas size: ${canvas.width}x${canvas.height}`);
+
+                    // Check if clicking existing point to delete
+                    const clickedPoint = this.findPointAt(x, y);
+                    if (clickedPoint && e.button === 2) {
+                        // Right-click on existing point = delete
+                        if (clickedPoint.type === 'positive') {
+                            this.canvasWidget.positivePoints = this.canvasWidget.positivePoints.filter(p => p !== clickedPoint.point);
+                        } else {
+                            this.canvasWidget.negativePoints = this.canvasWidget.negativePoints.filter(p => p !== clickedPoint.point);
+                        }
+                    } else {
+                        // Add new point
+                        if (e.shiftKey || e.button === 2) {
+                            // Negative point
+                            this.canvasWidget.negativePoints.push({x, y});
+                        } else {
+                            // Positive point
+                            this.canvasWidget.positivePoints.push({x, y});
+                        }
+                    }
+
+                    this.updatePoints();
+                    this.redrawCanvas();
+                });
+
+                canvas.addEventListener("contextmenu", (e) => {
+                    e.preventDefault();
+                    // Trigger click with right button flag
+                    canvas.dispatchEvent(new MouseEvent('click', {
+                        button: 2,
+                        clientX: e.clientX,
+                        clientY: e.clientY,
+                        shiftKey: e.shiftKey
+                    }));
+                });
+
+                canvas.addEventListener("mousemove", (e) => {
+                    const rect = canvas.getBoundingClientRect();
+                    const x = ((e.clientX - rect.left) / rect.width) * canvas.width;
+                    const y = ((e.clientY - rect.top) / rect.height) * canvas.height;
+
+                    const hovered = this.findPointAt(x, y);
+                    if (hovered !== this.canvasWidget.hoveredPoint) {
+                        this.canvasWidget.hoveredPoint = hovered;
+                        this.redrawCanvas();
+                    }
+                });
+
+                // Handle image input changes
+                this.onExecuted = (message) => {
+                    console.log("[SAM3] onExecuted called with message:", message);
+                    if (message.bg_image && message.bg_image[0]) {
+                        const img = new Image();
+                        img.onload = () => {
+                            console.log(`[SAM3] Image loaded: ${img.width}x${img.height}`);
+                            this.canvasWidget.image = img;
+                            canvas.width = img.width;
+                            canvas.height = img.height;
+                            console.log(`[SAM3] Canvas resized to: ${canvas.width}x${canvas.height}`);
+                            this.redrawCanvas();
+                        };
+                        img.src = "data:image/jpeg;base64," + message.bg_image[0];
+                    }
+                };
+
+                // Update container height dynamically when node size changes
+                const originalOnResize = this.onResize;
+                this.onResize = function(size) {
+                    if (originalOnResize) {
+                        originalOnResize.apply(this, arguments);
+                    }
+                    // Update container to match widget size
+                    const containerHeight = Math.max(200, size[1] - 80);
+                    container.style.height = containerHeight + "px";
+                    console.log(`[SAM3] Node resized to: ${size[0]}x${size[1]}, container height: ${containerHeight}px`);
+                };
+
+                // Also update on draw to handle any size changes
+                const originalOnDrawForeground = this.onDrawForeground;
+                this.onDrawForeground = function(ctx) {
+                    if (originalOnDrawForeground) {
+                        originalOnDrawForeground.apply(this, arguments);
+                    }
+                    // Update container height based on current node size
+                    const containerHeight = Math.max(200, this.size[1] - 80);
+                    if (container.style.height !== containerHeight + "px") {
+                        container.style.height = containerHeight + "px";
+                    }
+                };
+
+                // Draw initial placeholder
+                console.log("[SAM3] Drawing initial placeholder");
+                this.redrawCanvas();
+
+                // Set initial node size
+                const nodeWidth = Math.max(400, this.size[0] || 400);
+                const nodeHeight = 480; // Initial height: canvas (400) + space (80)
+                this.setSize([nodeWidth, nodeHeight]);
+
+                // Set initial container height
+                container.style.height = "400px";
+
+                console.log("[SAM3] Node size set to:", [nodeWidth, nodeHeight]);
+                console.log("[SAM3] onNodeCreated complete");
+                return result;
+            };
+
+            // Helper: Find point at coordinates
+            nodeType.prototype.findPointAt = function(x, y) {
+                const threshold = 10;
+
+                for (const point of this.canvasWidget.positivePoints) {
+                    if (Math.abs(point.x - x) < threshold && Math.abs(point.y - y) < threshold) {
+                        return {type: 'positive', point};
+                    }
+                }
+
+                for (const point of this.canvasWidget.negativePoints) {
+                    if (Math.abs(point.x - x) < threshold && Math.abs(point.y - y) < threshold) {
+                        return {type: 'negative', point};
+                    }
+                }
+
+                return null;
+            };
+
+            // Helper: Update widget values
+            nodeType.prototype.updatePoints = function() {
+                // Use stored hidden widget references
+                const coordsWidget = this._hiddenWidgets?.coordinates || this.widgets.find(w => w.name === "coordinates");
+                const negCoordsWidget = this._hiddenWidgets?.neg_coordinates || this.widgets.find(w => w.name === "neg_coordinates");
+                const storeWidget = this._hiddenWidgets?.points_store || this.widgets.find(w => w.name === "points_store");
+
+                if (coordsWidget) {
+                    coordsWidget.value = JSON.stringify(this.canvasWidget.positivePoints);
+                }
+                if (negCoordsWidget) {
+                    negCoordsWidget.value = JSON.stringify(this.canvasWidget.negativePoints);
+                }
+                if (storeWidget) {
+                    storeWidget.value = JSON.stringify({
+                        positive: this.canvasWidget.positivePoints,
+                        negative: this.canvasWidget.negativePoints
+                    });
+                }
+
+                // Update points counter display
+                const posCount = this.canvasWidget.positivePoints.length;
+                const negCount = this.canvasWidget.negativePoints.length;
+                this.canvasWidget.pointsCounter.textContent = `Points: ${posCount} pos, ${negCount} neg`;
+            };
+
+            // Helper: Redraw canvas
+            nodeType.prototype.redrawCanvas = function() {
+                const {canvas, ctx, image, positivePoints, negativePoints, hoveredPoint} = this.canvasWidget;
+
+                // Clear
+                ctx.clearRect(0, 0, canvas.width, canvas.height);
+
+                // Draw image if available
+                if (image) {
+                    ctx.drawImage(image, 0, 0, canvas.width, canvas.height);
+                } else {
+                    // Placeholder
+                    ctx.fillStyle = "#333";
+                    ctx.fillRect(0, 0, canvas.width, canvas.height);
+                    ctx.fillStyle = "#666";
+                    ctx.font = "16px sans-serif";
+                    ctx.textAlign = "center";
+                    ctx.fillText("Click to add points", canvas.width / 2, canvas.height / 2);
+                    ctx.fillText("Left-click: Positive (green)", canvas.width / 2, canvas.height / 2 + 25);
+                    ctx.fillText("Shift/Right-click: Negative (red)", canvas.width / 2, canvas.height / 2 + 50);
+                }
+
+                // Draw canvas dimensions overlay (helpful for debugging)
+                if (image) {
+                    ctx.fillStyle = "rgba(0,0,0,0.7)";
+                    ctx.fillRect(5, canvas.height - 25, 150, 20);
+                    ctx.fillStyle = "#0f0";
+                    ctx.font = "12px monospace";
+                    ctx.textAlign = "left";
+                    ctx.fillText(`Image: ${canvas.width}x${canvas.height}`, 10, canvas.height - 10);
+                }
+
+                // Draw positive points (green)
+                ctx.strokeStyle = "#0f0";
+                ctx.fillStyle = "#0f0";
+                for (const point of positivePoints) {
+                    const isHovered = hoveredPoint?.point === point;
+                    ctx.beginPath();
+                    ctx.arc(point.x, point.y, isHovered ? 8 : 6, 0, 2 * Math.PI);
+                    ctx.fill();
+                    ctx.lineWidth = 2;
+                    ctx.stroke();
+                }
+
+                // Draw negative points (red)
+                ctx.strokeStyle = "#f00";
+                ctx.fillStyle = "#f00";
+                for (const point of negativePoints) {
+                    const isHovered = hoveredPoint?.point === point;
+                    ctx.beginPath();
+                    ctx.arc(point.x, point.y, isHovered ? 8 : 6, 0, 2 * Math.PI);
+                    ctx.fill();
+                    ctx.lineWidth = 2;
+                    ctx.stroke();
+                }
+            };
+        }
+    }
+});

workflows/example_segmentation_points.json

@@ -0,0 +1,395 @@
+{
+  "id": "0b04cdef-0fe9-494a-baa9-ca11511022b9",
+  "revision": 0,
+  "last_node_id": 9,
+  "last_link_id": 8,
+  "nodes": [
+    {
+      "id": 7,
+      "type": "MaskPreview",
+      "pos": [
+        1285.1917567959822,
+        -164.2376998289703
+      ],
+      "size": [
+        140,
+        246
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "mask",
+          "type": "MASK",
+          "link": 5
+        }
+      ],
+      "outputs": [],
+      "properties": {
+        "cnr_id": "comfy-core",
+        "ver": "0.3.70",
+        "Node name for S&R": "MaskPreview"
+      },
+      "widgets_values": []
+    },
+    {
+      "id": 1,
+      "type": "LoadSAM3Model",
+      "pos": [
+        530.73828125,
+        128.45703125
+      ],
+      "size": [
+        270,
+        106
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "sam3_model",
+          "type": "SAM3_MODEL",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "aux_id": "PozzettiAndrea/ComfyUI-SAM3",
+        "ver": "307654744fa45de98017f92710d4a9cc1b4d63de",
+        "Node name for S&R": "LoadSAM3Model"
+      },
+      "widgets_values": [
+        "auto",
+        "",
+        "YOUR_HF_TOKEN_HERE"
+      ]
+    },
+    {
+      "id": 4,
+      "type": "LoadImage",
+      "pos": [
+        535.57421875,
+        284.2890625
+      ],
+      "size": [
+        274.080078125,
+        314
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [
+            2,
+            7
+          ]
+        },
+        {
+          "name": "MASK",
+          "type": "MASK",
+          "links": null
+        }
+      ],
+      "properties": {
+        "cnr_id": "comfy-core",
+        "ver": "0.3.70",
+        "Node name for S&R": "LoadImage"
+      },
+      "widgets_values": [
+        "example_image.jpg",
+        "image"
+      ]
+    },
+    {
+      "id": 5,
+      "type": "PreviewImage",
+      "pos": [
+        1710.4529605215105,
+        147.12305834352858
+      ],
+      "size": [
+        140,
+        246
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 3
+        }
+      ],
+      "outputs": [],
+      "properties": {
+        "cnr_id": "comfy-core",
+        "ver": "0.3.70",
+        "Node name for S&R": "PreviewImage"
+      },
+      "widgets_values": []
+    },
+    {
+      "id": 6,
+      "type": "PreviewAny",
+      "pos": [
+        1578.2128366336826,
+        509.43098550417255
+      ],
+      "size": [
+        233.69546594584722,
+        164.65461903779413
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "source",
+          "type": "*",
+          "link": 4
+        }
+      ],
+      "outputs": [],
+      "properties": {
+        "cnr_id": "comfy-core",
+        "ver": "0.3.70",
+        "Node name for S&R": "PreviewAny"
+      },
+      "widgets_values": []
+    },
+    {
+      "id": 2,
+      "type": "SAM3Segmentation",
+      "pos": [
+        1413.3547712239274,
+        213.48039870709997
+      ],
+      "size": [
+        278.08203125,
+        266
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "sam3_model",
+          "type": "SAM3_MODEL",
+          "link": 1
+        },
+        {
+          "name": "image",
+          "type": "IMAGE",
+          "link": 2
+        },
+        {
+          "name": "positive_boxes",
+          "shape": 7,
+          "type": "SAM3_BOXES_PROMPT",
+          "link": null
+        },
+        {
+          "name": "negative_boxes",
+          "shape": 7,
+          "type": "SAM3_BOXES_PROMPT",
+          "link": null
+        },
+        {
+          "name": "positive_points",
+          "shape": 7,
+          "type": "SAM3_POINTS_PROMPT",
+          "link": 8
+        },
+        {
+          "name": "negative_points",
+          "shape": 7,
+          "type": "SAM3_POINTS_PROMPT",
+          "link": null
+        },
+        {
+          "name": "mask_prompt",
+          "shape": 7,
+          "type": "MASK",
+          "link": null
+        },
+        {
+          "name": "boxes_prompt",
+          "shape": 7,
+          "type": "SAM3_BOXES_PROMPT",
+          "link": null
+        },
+        {
+          "name": "points_prompt",
+          "shape": 7,
+          "type": "SAM3_POINTS_PROMPT",
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "masks",
+          "type": "MASK",
+          "links": [
+            5
+          ]
+        },
+        {
+          "name": "visualization",
+          "type": "IMAGE",
+          "links": [
+            3
+          ]
+        },
+        {
+          "name": "boxes",
+          "type": "STRING",
+          "links": [
+            4
+          ]
+        },
+        {
+          "name": "scores",
+          "type": "STRING",
+          "links": null
+        }
+      ],
+      "properties": {
+        "aux_id": "PozzettiAndrea/ComfyUI-SAM3",
+        "ver": "307654744fa45de98017f92710d4a9cc1b4d63de",
+        "Node name for S&R": "SAM3Segmentation"
+      },
+      "widgets_values": [
+        0.21,
+        "person",
+        -1
+      ]
+    },
+    {
+      "id": 9,
+      "type": "SAM3PointCollector",
+      "pos": [
+        929.3727386768797,
+        184.97597369457938
+      ],
+      "size": [
+        400,
+        480
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image",
+          "type": "IMAGE",
+          "link": 7
+        }
+      ],
+      "outputs": [
+        {
+          "name": "positive_points",
+          "type": "SAM3_POINTS_PROMPT",
+          "links": [
+            8
+          ]
+        },
+        {
+          "name": "negative_points",
+          "type": "SAM3_POINTS_PROMPT",
+          "links": null
+        }
+      ],
+      "properties": {
+        "aux_id": "PozzettiAndrea/ComfyUI-SAM3",
+        "ver": "64c86b0bf97593007c32df5dbc544a566d9efd9f",
+        "Node name for S&R": "SAM3PointCollector"
+      },
+      "widgets_values": [
+        "{}",
+        "[]",
+        "[]",
+        ""
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      1,
+      0,
+      2,
+      0,
+      "SAM3_MODEL"
+    ],
+    [
+      2,
+      4,
+      0,
+      2,
+      1,
+      "IMAGE"
+    ],
+    [
+      3,
+      2,
+      1,
+      5,
+      0,
+      "IMAGE"
+    ],
+    [
+      4,
+      2,
+      2,
+      6,
+      0,
+      "*"
+    ],
+    [
+      5,
+      2,
+      0,
+      7,
+      0,
+      "MASK"
+    ],
+    [
+      7,
+      4,
+      0,
+      9,
+      0,
+      "IMAGE"
+    ],
+    [
+      8,
+      9,
+      0,
+      2,
+      4,
+      "SAM3_POINTS_PROMPT"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 1.0152559799477059,
+      "offset": [
+        -484.16482173037514,
+        102.63622043900284
+      ]
+    },
+    "frontendVersion": "1.28.8"
+  },
+  "version": 0.4
+}

workflows/video_tracking.json

@@ -0,0 +1,226 @@
+{
+  "last_node_id": 8,
+  "last_link_id": 12,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "SAM3VideoModelLoader",
+      "pos": [100, 100],
+      "size": [400, 200],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "video_model",
+          "type": "SAM3_VIDEO_MODEL",
+          "links": [1]
+        }
+      ],
+      "properties": {},
+      "widgets_values": [
+        "",
+        "",
+        "",
+        false,
+        "0"
+      ],
+      "title": "SAM3 Load Video Model"
+    },
+    {
+      "id": 2,
+      "type": "VHS_LoadVideo",
+      "pos": [100, 350],
+      "size": [300, 150],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [2]
+        }
+      ],
+      "properties": {},
+      "widgets_values": [],
+      "title": "Load Video (VHS)"
+    },
+    {
+      "id": 3,
+      "type": "SAM3InitVideoSession",
+      "pos": [550, 100],
+      "size": [350, 150],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video_model",
+          "type": "SAM3_VIDEO_MODEL",
+          "link": 1
+        },
+        {
+          "name": "video_frames",
+          "type": "IMAGE",
+          "link": 2
+        }
+      ],
+      "outputs": [
+        {
+          "name": "session",
+          "type": "SAM3_VIDEO_SESSION",
+          "links": [3, 4]
+        },
+        {
+          "name": "session_id",
+          "type": "STRING",
+          "links": []
+        }
+      ],
+      "properties": {},
+      "widgets_values": [""],
+      "title": "SAM3 Init Video Session"
+    },
+    {
+      "id": 4,
+      "type": "SAM3AddVideoPrompt",
+      "pos": [950, 100],
+      "size": [400, 250],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "session",
+          "type": "SAM3_VIDEO_SESSION",
+          "link": 3
+        }
+      ],
+      "outputs": [
+        {
+          "name": "session",
+          "type": "SAM3_VIDEO_SESSION",
+          "links": [5]
+        }
+      ],
+      "properties": {},
+      "widgets_values": [
+        0,
+        "person",
+        1
+      ],
+      "title": "SAM3 Add Video Prompt"
+    },
+    {
+      "id": 5,
+      "type": "SAM3PropagateVideo",
+      "pos": [1400, 100],
+      "size": [350, 200],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "session",
+          "type": "SAM3_VIDEO_SESSION",
+          "link": 5
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video_masks",
+          "type": "SAM3_VIDEO_MASKS",
+          "links": [6]
+        }
+      ],
+      "properties": {},
+      "widgets_values": [
+        "both",
+        0,
+        -1
+      ],
+      "title": "SAM3 Propagate Video"
+    },
+    {
+      "id": 6,
+      "type": "SAM3VideoOutput",
+      "pos": [1800, 100],
+      "size": [300, 150],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video_masks",
+          "type": "SAM3_VIDEO_MASKS",
+          "link": 6
+        }
+      ],
+      "outputs": [
+        {
+          "name": "masks",
+          "type": "MASK",
+          "links": [7]
+        }
+      ],
+      "properties": {},
+      "widgets_values": [-1],
+      "title": "SAM3 Video Output"
+    },
+    {
+      "id": 7,
+      "type": "PreviewImage",
+      "pos": [2150, 100],
+      "size": [300, 300],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "MASK",
+          "link": 7
+        }
+      ],
+      "properties": {},
+      "title": "Preview Masks"
+    },
+    {
+      "id": 8,
+      "type": "SAM3CloseVideoSession",
+      "pos": [950, 400],
+      "size": [300, 100],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "session",
+          "type": "SAM3_VIDEO_SESSION",
+          "link": 4
+        }
+      ],
+      "properties": {},
+      "title": "SAM3 Close Video Session"
+    }
+  ],
+  "links": [
+    [1, 1, 0, 3, 0, "SAM3_VIDEO_MODEL"],
+    [2, 2, 0, 3, 1, "IMAGE"],
+    [3, 3, 0, 4, 0, "SAM3_VIDEO_SESSION"],
+    [4, 3, 0, 8, 0, "SAM3_VIDEO_SESSION"],
+    [5, 4, 0, 5, 0, "SAM3_VIDEO_SESSION"],
+    [6, 5, 0, 6, 0, "SAM3_VIDEO_MASKS"],
+    [7, 6, 0, 7, 0, "MASK"]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 1,
+      "offset": [0, 0]
+    }
+  },
+  "version": 0.4
+}