Apache0ne-ComfyUI_efficient_sam_node/efficient_sam_node.py

import os
import re
import torch
import numpy as np
import folder_paths
from PIL import Image

# Required imports from the EfficientViT-SAM library
# Make sure 'efficientvit' is installed in your ComfyUI environment
# pip install git+https://github.com/mit-han-lab/efficientvit.git
from efficientvit.sam_model_zoo import create_efficientvit_sam_model
from efficientvit.models.efficientvit.sam import EfficientViTSamPredictor, EfficientViTSamAutomaticMaskGenerator

# Helper: Find EfficientViT-SAM models in ComfyUI/models/sam or /sams
def find_efficientvit_sam_models():
    """
    Scans for EfficientViT-SAM models inside ComfyUI/models/sam and ComfyUI/models/sams
    """
    model_filenames = set()
    
    models_dir = folder_paths.models_dir  

    sam_dir_names = ["sam", "sams"]

    for dir_name in sam_dir_names:
        full_path = os.path.join(models_dir, dir_name)
        if os.path.isdir(full_path):
            print(f"[EfficientViT-SAM Loader] Scanning for models in: {full_path}")
            for filename in os.listdir(full_path):
                if filename.startswith("efficientvit_sam_") and filename.endswith((".pt", ".pth", ".safetensors")):
                    model_filenames.add(filename)
    
    return sorted(list(model_filenames))

available_models = find_efficientvit_sam_models()

# NODE 1: Model Loader
class EfficientViTSAMLoader:
    @classmethod
    def INPUT_TYPES(s):
        if not available_models:
            raise RuntimeError(
                "No EfficientViT-SAM models found. "
                "Please place models like 'efficientvit_sam_l0.pt' inside 'ComfyUI/models/sam/'."
            )
        return {
            "required": {
                "model_name": (available_models,),
                "device": (["cuda", "cpu"],),
            }
        }

    RETURN_TYPES = ("SAM_MODEL",)
    FUNCTION = "load_model"
    CATEGORY = "EfficientViT-SAM"

    def load_model(self, model_name, device):
        ckpt_path = None
        models_dir = folder_paths.models_dir
        sam_dir_names = ["sam", "sams"]

        for dir_name in sam_dir_names:
            potential_path = os.path.join(models_dir, dir_name, model_name)
            if os.path.isfile(potential_path):
                ckpt_path = potential_path
                break

        if not ckpt_path:
            raise FileNotFoundError(
                f"Could not find checkpoint '{model_name}' in 'sam' or 'sams' under '{models_dir}'."
            )

        match = re.search(r'efficientvit_sam_(.*)\.(pt|pth|safetensors)', model_name)
        if not match:
            raise ValueError(f"Invalid model filename: {model_name}")
        model_type = match.group(1)

        full_model_name = f"efficientvit-sam-{model_type}"

        print(f"Loading EfficientViT-SAM model: {full_model_name} from {ckpt_path}")
        
        sam_model = create_efficientvit_sam_model(name=full_model_name, pretrained=False)
        sam_model.load_state_dict(torch.load(ckpt_path, map_location="cpu"))
        sam_model = sam_model.to(device).eval()

        sam_predictor = EfficientViTSamPredictor(sam_model)
        return (sam_predictor,)

# NODE 2: The Segmentation Predictor (BBox)
class EfficientViTSAMPredictorNode:
    """
    This node uses a loaded EfficientViT-SAM model to predict a segmentation
    mask based on a bounding box (BBOX) input.
    """
    @classmethod
    def INPUT_TYPES(s):
        return {
            "required": {
                "sam_model": ("SAM_MODEL",),
                "image": ("IMAGE",),
                "bbox": ("BBOX",), 
            }
        }

    RETURN_TYPES = ("MASK", "IMAGE",)
    FUNCTION = "predict_segmentation"
    CATEGORY = "EfficientViT-SAM"

    def predict_segmentation(self, sam_model, image, bbox):
            (h, w) = image.shape[1:3]
            empty_mask_tensor = torch.zeros(1, h, w, dtype=torch.float32, device="cpu")
            empty_image_tensor = torch.zeros(1, h, w, 3, dtype=torch.float32, device="cpu")
            empty_return = (empty_mask_tensor, empty_image_tensor)

            if not isinstance(bbox, list) or not bbox:
                print("[EfficientViTSAM Bbox] Warning: BBOX input is not a valid list or is empty. Returning a black mask.")
                return empty_return

            box_data = bbox[0]

            if not isinstance(box_data, (list, tuple)) or len(box_data) < 4:
                print(f"[EfficientViTSAM Bbox] Warning: Received invalid data for BBOX. Expected a list of 4 numbers, but got data of type '{type(box_data)}'. This is likely point data connected to the wrong input. Returning a black mask.")
                return empty_return
            
            predictor = sam_model
            image_np = (image[0].numpy() * 255).astype(np.uint8)
            predictor.set_image(image_np)
            bbox_np = np.array([box_data[:4]], dtype=np.float32)

            masks, scores, logits = predictor.predict(
                box=bbox_np,
                multimask_output=False,
            )

            mask_np = masks[0].astype(np.float32)
            mask_tensor = torch.from_numpy(mask_np).unsqueeze(0)

            seg_image = mask_tensor.unsqueeze(-1).expand(-1, -1, -1, 3)

            return (mask_tensor, seg_image)

# NODE 3: The Point-Based Segmentation Predictor
class EfficientViTSAMPointPredictorNode:
    """
    This node uses a loaded EfficientViT-SAM model to predict a segmentation
    mask based on foreground and background point prompts.
    """
    @classmethod
    def INPUT_TYPES(s):
        return {
            "required": {
                "sam_model": ("SAM_MODEL",),
                "image": ("IMAGE",),
                "points": ("SAM_PROMPT",),
            }
        }

    RETURN_TYPES = ("MASK", "IMAGE",)
    FUNCTION = "predict_segmentation_with_points"
    CATEGORY = "EfficientViT-SAM"

    def predict_segmentation_with_points(self, sam_model, image, points):
        predictor = sam_model

        image_np = (image[0].numpy() * 255).astype(np.uint8)
        predictor.set_image(image_np)

        point_coords = np.array(points.get("points", []), dtype=np.float32)
        point_labels = np.array(points.get("labels", []), dtype=np.int32)

        if point_coords.shape[0] == 0:
            h, w = image_np.shape[:2]
            empty_mask = torch.zeros((1, h, w), dtype=torch.float32, device="cpu")
            empty_image = torch.zeros((1, h, w, 3), dtype=torch.float32, device="cpu")
            return (empty_mask, empty_image)

        masks, scores, logits = predictor.predict(
            point_coords=point_coords,
            point_labels=point_labels,
            multimask_output=False,
        )

        mask_np = masks[0].astype(np.float32)
        mask_tensor = torch.from_numpy(mask_np).unsqueeze(0)

        seg_image = mask_tensor.unsqueeze(-1).expand(-1, -1, -1, 3)

        return (mask_tensor, seg_image)

# NODE 4: The Automatic Mask Generator
class EfficientViTSAMAutoMaskGeneratorNode:
    """
    This node automatically generates segmentation masks for all objects
    detected in an image, without needing any input prompts.
    """
    @classmethod
    def INPUT_TYPES(s):
        return {
            "required": {
                "sam_model": ("SAM_MODEL",),
                "image": ("IMAGE",),
            }
        }

    RETURN_TYPES = ("MASK", "IMAGE",)
    FUNCTION = "generate_masks"
    CATEGORY = "EfficientViT-SAM"

    def generate_masks(self, sam_model, image):
        raw_model = sam_model.model
        mask_generator = EfficientViTSamAutomaticMaskGenerator(raw_model)
        image_np = (image[0].numpy() * 255).astype(np.uint8)
        masks_list = mask_generator.generate(image_np)

        if not masks_list:
            h, w = image_np.shape[:2]
            empty_mask = torch.zeros((1, h, w), dtype=torch.float32, device="cpu")
            empty_image = torch.zeros((1, h, w, 3), dtype=torch.float32, device="cpu")
            return (empty_mask, empty_image)

        all_masks = [torch.from_numpy(m['segmentation']) for m in masks_list]
        mask_batch = torch.stack(all_masks).float()

        seg_images = mask_batch.unsqueeze(-1).expand(-1, -1, -1, 3)

        return (mask_batch, seg_images)

# NODE 5: Video BBox Segmentation Predictor
class EfficientViTSAMVideoPredictorNode:
    """
    Processes a batch of images (video frames) to predict segmentation masks based on a bounding box.
    """
    @classmethod
    def INPUT_TYPES(s):
        return {
            "required": {
                "sam_model": ("SAM_MODEL",),
                "image": ("IMAGE",),
                "bbox": ("BBOX",),
            }
        }

    RETURN_TYPES = ("MASK", "IMAGE",)
    FUNCTION = "predict_video_segmentation"
    CATEGORY = "EfficientViT-SAM"

    def predict_video_segmentation(self, sam_model, image, bbox):
        (b, h, w, c) = image.shape
        empty_mask_tensor = torch.zeros(b, h, w, dtype=torch.float32, device="cpu")
        empty_image_tensor = torch.zeros(b, h, w, 3, dtype=torch.float32, device="cpu")
        empty_return = (empty_mask_tensor, empty_image_tensor)

        if not isinstance(bbox, list) or not bbox:
            print("[EfficientViTSAM Video Bbox] Warning: BBOX input is not a valid list or is empty.")
            return empty_return

        box_data = bbox[0]
        if not isinstance(box_data, (list, tuple)) or len(box_data) < 4:
            print(f"[EfficientViTSAM Video Bbox] Warning: Invalid BBOX data type '{type(box_data)}'.")
            return empty_return

        predictor = sam_model
        bbox_np = np.array([box_data[:4]], dtype=np.float32)
        
        masks = []
        seg_images = []

        for i in range(b):
            image_np = (image[i].numpy() * 255).astype(np.uint8)
            predictor.set_image(image_np)

            frame_masks, _, _ = predictor.predict(box=bbox_np, multimask_output=False)
            
            mask_np = frame_masks[0].astype(np.float32)
            mask_tensor = torch.from_numpy(mask_np)
            masks.append(mask_tensor)

            seg_image = mask_tensor.unsqueeze(-1).expand(-1, -1, 3)
            seg_images.append(seg_image)

        return (torch.stack(masks), torch.stack(seg_images))

# NODE 6: Video Point-Based Segmentation Predictor
class EfficientViTSAMVideoPointPredictorNode:
    """
    Processes a batch of images (video frames) to predict segmentation masks based on point prompts.
    """
    @classmethod
    def INPUT_TYPES(s):
        return {
            "required": {
                "sam_model": ("SAM_MODEL",),
                "image": ("IMAGE",),
                "points": ("SAM_PROMPT",),
            }
        }

    RETURN_TYPES = ("MASK", "IMAGE",)
    FUNCTION = "predict_video_segmentation_with_points"
    CATEGORY = "EfficientViT-SAM"

    def predict_video_segmentation_with_points(self, sam_model, image, points):
        predictor = sam_model
        (b, h, w, c) = image.shape

        point_coords = np.array(points.get("points", []), dtype=np.float32)
        point_labels = np.array(points.get("labels", []), dtype=np.int32)

        if point_coords.shape[0] == 0:
            empty_mask = torch.zeros((b, h, w), dtype=torch.float32, device="cpu")
            empty_image = torch.zeros((b, h, w, 3), dtype=torch.float32, device="cpu")
            return (empty_mask, empty_image)

        masks = []
        seg_images = []

        for i in range(b):
            image_np = (image[i].numpy() * 255).astype(np.uint8)
            predictor.set_image(image_np)

            frame_masks, _, _ = predictor.predict(
                point_coords=point_coords,
                point_labels=point_labels,
                multimask_output=False,
            )

            mask_np = frame_masks[0].astype(np.float32)
            mask_tensor = torch.from_numpy(mask_np)
            masks.append(mask_tensor)

            seg_image = mask_tensor.unsqueeze(-1).expand(-1, -1, 3)
            seg_images.append(seg_image)

        return (torch.stack(masks), torch.stack(seg_images))

# NODE 7: Video Automatic Mask Generator
class EfficientViTSAMVideoAutoMaskGeneratorNode:
    """
    Automatically generates segmentation masks for all objects in a batch of images (video frames).
    """
    @classmethod
    def INPUT_TYPES(s):
        return {
            "required": {
                "sam_model": ("SAM_MODEL",),
                "image": ("IMAGE",),
            }
        }

    RETURN_TYPES = ("MASK", "IMAGE",)
    FUNCTION = "generate_video_masks"
    CATEGORY = "EfficientViT-SAM"

    def generate_video_masks(self, sam_model, image):
        raw_model = sam_model.model
        mask_generator = EfficientViTSamAutomaticMaskGenerator(raw_model)
        (b, h, w, c) = image.shape

        all_masks = []
        all_seg_images = []

        for i in range(b):
            image_np = (image[i].numpy() * 255).astype(np.uint8)
            masks_list = mask_generator.generate(image_np)

            if not masks_list:
                continue

            for m in masks_list:
                mask_tensor = torch.from_numpy(m['segmentation']).float()
                all_masks.append(mask_tensor)
                
                seg_image = mask_tensor.unsqueeze(-1).expand(-1, -1, 3)
                all_seg_images.append(seg_image)

        if not all_masks:
            empty_mask = torch.zeros((1, h, w), dtype=torch.float32, device="cpu")
            empty_image = torch.zeros((1, h, w, 3), dtype=torch.float32, device="cpu")
            return (empty_mask, empty_image)

        return (torch.stack(all_masks), torch.stack(all_seg_images))