From ba792e13c6da49d1803b433beaf985b7a36c1ff7 Mon Sep 17 00:00:00 2001 From: Andrea Pozzetti Date: Fri, 19 Dec 2025 01:01:42 +0100 Subject: [PATCH] first draft --- .github/workflows/publish.yml | 25 +++++ .github/workflows/test.yml | 23 +++++ .gitignore | 26 +++++ __init__.py | 11 +++ nodes/__init__.py | 18 ++++ nodes/load_model.py | 107 ++++++++++++++++++++ nodes/predict.py | 177 ++++++++++++++++++++++++++++++++++ requirements.txt | 13 +++ utils/__init__.py | 5 + utils/image.py | 39 ++++++++ workflows/sharp_basic.json | 148 ++++++++++++++++++++++++++++ 11 files changed, 592 insertions(+) create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/test.yml create mode 100644 .gitignore create mode 100644 __init__.py create mode 100644 nodes/__init__.py create mode 100644 nodes/load_model.py create mode 100644 nodes/predict.py create mode 100644 requirements.txt create mode 100644 utils/__init__.py create mode 100644 utils/image.py create mode 100644 workflows/sharp_basic.json diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..56dcaea --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,25 @@ +name: Publish to Comfy registry + +on: + workflow_dispatch: + push: + branches: + - main + paths-ignore: + - '**.md' + - 'workflows/**' + +permissions: + issues: write + +jobs: + publish-node: + name: Publish Custom Node to registry + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Publish Custom Node + uses: Comfy-Org/publish-node-action@v1 + with: + personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..2b5ca85 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,23 @@ +name: Test Installation + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + test-syntax: + name: Test Python Syntax + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Check syntax + run: | + python -m py_compile __init__.py + python -m py_compile nodes/*.py + python -m py_compile utils/*.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5e57d2c --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so + +# Jupyter +.ipynb_checkpoints/ + +# Environment +.env +.venv/ +venv/ + +# IDE +.vscode/ +.idea/ + +# Model checkpoints (large files) +*.pt +*.pth +*.ckpt + +# Output files +output/ +*.ply diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..d70b1f0 --- /dev/null +++ b/__init__.py @@ -0,0 +1,11 @@ +"""ComfyUI-Sharp: SHARP monocular 3D Gaussian Splatting for ComfyUI. + +SHARP (Sharp Monocular View Synthesis) takes a single image and produces +3D Gaussian Splatting representations in under 1 second. + +Based on Apple's SHARP model: https://arxiv.org/abs/2512.10685 +""" + +from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS + +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] diff --git a/nodes/__init__.py b/nodes/__init__.py new file mode 100644 index 0000000..b77380c --- /dev/null +++ b/nodes/__init__.py @@ -0,0 +1,18 @@ +"""ComfyUI-Sharp nodes aggregation.""" + +from .load_model import NODE_CLASS_MAPPINGS as LOAD_MODEL_MAPPINGS +from .load_model import NODE_DISPLAY_NAME_MAPPINGS as LOAD_MODEL_DISPLAY_MAPPINGS +from .predict import NODE_CLASS_MAPPINGS as PREDICT_MAPPINGS +from .predict import NODE_DISPLAY_NAME_MAPPINGS as PREDICT_DISPLAY_MAPPINGS + +# Aggregate all node mappings +NODE_CLASS_MAPPINGS = {} +NODE_DISPLAY_NAME_MAPPINGS = {} + +NODE_CLASS_MAPPINGS.update(LOAD_MODEL_MAPPINGS) +NODE_CLASS_MAPPINGS.update(PREDICT_MAPPINGS) + +NODE_DISPLAY_NAME_MAPPINGS.update(LOAD_MODEL_DISPLAY_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(PREDICT_DISPLAY_MAPPINGS) + +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] diff --git a/nodes/load_model.py b/nodes/load_model.py new file mode 100644 index 0000000..e989605 --- /dev/null +++ b/nodes/load_model.py @@ -0,0 +1,107 @@ +"""LoadSharpModel node for ComfyUI-Sharp.""" + +import os +import sys + +import torch + +# Add ml-sharp to path +# Priority: SHARP_PATH env var > /home/shadeform/ml-sharp/src > relative path +ML_SHARP_PATH = os.environ.get("SHARP_PATH") +if not ML_SHARP_PATH: + # Try /home/shadeform/ml-sharp/src first + default_path = "/home/shadeform/ml-sharp/src" + if os.path.exists(default_path): + ML_SHARP_PATH = default_path + else: + # Fallback to relative path from this file + ML_SHARP_PATH = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), + "ml-sharp", "src" + ) + +if ML_SHARP_PATH not in sys.path: + sys.path.insert(0, ML_SHARP_PATH) + +# Model cache +_MODEL_CACHE = {} + +# Default model URL +DEFAULT_MODEL_URL = "https://ml-site.cdn-apple.com/models/sharp/sharp_2572gikvuh.pt" + + +class LoadSharpModel: + """Load and cache the SHARP model for 3D Gaussian prediction.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "device": (["auto", "cuda", "mps", "cpu"], { + "default": "auto", + "tooltip": "Device to run inference on. 'auto' selects best available." + }), + }, + "optional": { + "checkpoint_path": ("STRING", { + "default": "", + "tooltip": "Path to .pt checkpoint. Leave empty to auto-download from Apple CDN." + }), + } + } + + RETURN_TYPES = ("SHARP_MODEL",) + RETURN_NAMES = ("model",) + FUNCTION = "load_model" + CATEGORY = "SHARP" + DESCRIPTION = "Load the SHARP model for monocular 3D Gaussian Splatting prediction." + + def load_model(self, device: str, checkpoint_path: str = ""): + """Load and cache the SHARP model.""" + from sharp.models import PredictorParams, create_predictor + + # Resolve device + if device == "auto": + if torch.cuda.is_available(): + device = "cuda" + elif hasattr(torch, "mps") and torch.mps.is_available(): + device = "mps" + else: + device = "cpu" + + # Create cache key + cache_key = f"{checkpoint_path or 'default'}_{device}" + + if cache_key in _MODEL_CACHE: + return (_MODEL_CACHE[cache_key],) + + # Load or download checkpoint + if checkpoint_path and os.path.exists(checkpoint_path): + state_dict = torch.load(checkpoint_path, weights_only=True) + else: + print(f"[SHARP] Downloading model from {DEFAULT_MODEL_URL}") + state_dict = torch.hub.load_state_dict_from_url(DEFAULT_MODEL_URL, progress=True) + + # Create predictor + predictor = create_predictor(PredictorParams()) + predictor.load_state_dict(state_dict) + predictor.eval() + predictor.to(device) + + model_dict = { + "predictor": predictor, + "device": device, + } + + _MODEL_CACHE[cache_key] = model_dict + + return (model_dict,) + + +NODE_CLASS_MAPPINGS = { + "LoadSharpModel": LoadSharpModel, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "LoadSharpModel": "Load SHARP Model", +} diff --git a/nodes/predict.py b/nodes/predict.py new file mode 100644 index 0000000..a77365e --- /dev/null +++ b/nodes/predict.py @@ -0,0 +1,177 @@ +"""SharpPredict node for ComfyUI-Sharp.""" + +import os +import sys +import time +from pathlib import Path + +import numpy as np +import torch +import torch.nn.functional as F + +# Add ml-sharp to path +# Priority: SHARP_PATH env var > /home/shadeform/ml-sharp/src > relative path +ML_SHARP_PATH = os.environ.get("SHARP_PATH") +if not ML_SHARP_PATH: + # Try /home/shadeform/ml-sharp/src first + default_path = "/home/shadeform/ml-sharp/src" + if os.path.exists(default_path): + ML_SHARP_PATH = default_path + else: + # Fallback to relative path from this file + ML_SHARP_PATH = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), + "ml-sharp", "src" + ) + +if ML_SHARP_PATH not in sys.path: + sys.path.insert(0, ML_SHARP_PATH) + +# Try to import ComfyUI folder_paths for output directory +try: + import folder_paths + OUTPUT_DIR = folder_paths.get_output_directory() +except ImportError: + OUTPUT_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "output") + +from ..utils.image import comfy_to_numpy_rgb, convert_focallength + + +class SharpPredict: + """Run SHARP inference to generate 3D Gaussians from a single image.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "model": ("SHARP_MODEL",), + "image": ("IMAGE",), + }, + "optional": { + "focal_length_mm": ("FLOAT", { + "default": 0.0, + "min": 0.0, + "max": 500.0, + "step": 0.1, + "tooltip": "Focal length in mm (35mm equivalent). 0 = auto (defaults to 30mm)." + }), + "output_prefix": ("STRING", { + "default": "sharp", + "tooltip": "Prefix for output PLY filename." + }), + } + } + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("ply_path",) + FUNCTION = "predict" + CATEGORY = "SHARP" + OUTPUT_NODE = True + DESCRIPTION = "Generate 3D Gaussian Splatting PLY file from a single image using SHARP." + + @torch.no_grad() + def predict( + self, + model: dict, + image: torch.Tensor, + focal_length_mm: float = 0.0, + output_prefix: str = "sharp", + ): + """Run SHARP inference and save PLY file.""" + from sharp.utils.gaussians import Gaussians3D, save_ply, unproject_gaussians + + predictor = model["predictor"] + device = torch.device(model["device"]) + + # Convert ComfyUI image to numpy RGB + image_np = comfy_to_numpy_rgb(image) + height, width = image_np.shape[:2] + + # Determine focal length in pixels + if focal_length_mm > 0: + f_px = convert_focallength(width, height, focal_length_mm) + else: + # Default to 30mm equivalent + f_px = convert_focallength(width, height, 30.0) + + # Run inference + gaussians = self._predict_image(predictor, image_np, f_px, device) + + # Ensure output directory exists + os.makedirs(OUTPUT_DIR, exist_ok=True) + + # Generate output filename + timestamp = int(time.time() * 1000) + output_filename = f"{output_prefix}_{timestamp}.ply" + output_path = os.path.join(OUTPUT_DIR, output_filename) + + # Save PLY + save_ply(gaussians, f_px, (height, width), Path(output_path)) + + print(f"[SHARP] Saved PLY to: {output_path}") + + return (output_path,) + + def _predict_image( + self, + predictor, + image: np.ndarray, + f_px: float, + device: torch.device, + ): + """Predict Gaussians from an image. + + Based on sharp/cli/predict.py:predict_image() + """ + from sharp.utils.gaussians import unproject_gaussians + + internal_shape = (1536, 1536) + + # Convert to tensor and normalize + image_pt = torch.from_numpy(image.copy()).float().to(device).permute(2, 0, 1) / 255.0 + _, height, width = image_pt.shape + disparity_factor = torch.tensor([f_px / width]).float().to(device) + + # Resize to internal resolution + image_resized_pt = F.interpolate( + image_pt[None], + size=(internal_shape[1], internal_shape[0]), + mode="bilinear", + align_corners=True, + ) + + # Predict Gaussians in NDC space + gaussians_ndc = predictor(image_resized_pt, disparity_factor) + + # Build intrinsics for unprojection + intrinsics = ( + torch.tensor( + [ + [f_px, 0, width / 2, 0], + [0, f_px, height / 2, 0], + [0, 0, 1, 0], + [0, 0, 0, 1], + ] + ) + .float() + .to(device) + ) + intrinsics_resized = intrinsics.clone() + intrinsics_resized[0] *= internal_shape[0] / width + intrinsics_resized[1] *= internal_shape[1] / height + + # Convert Gaussians to metric space + gaussians = unproject_gaussians( + gaussians_ndc, torch.eye(4).to(device), intrinsics_resized, internal_shape + ) + + return gaussians + + +NODE_CLASS_MAPPINGS = { + "SharpPredict": SharpPredict, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "SharpPredict": "SHARP Predict (Image to PLY)", +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d70454a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +# Core dependencies for SHARP +torch>=2.0.0 +torchvision>=0.15.0 +numpy>=1.21.0 +pillow>=9.0.0 + +# SHARP specific +timm>=0.9.0 +plyfile>=1.0.0 +scipy>=1.10.0 +pillow-heif>=0.10.0 +imageio>=2.0.0 +imageio-ffmpeg>=0.4.0 diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..9ab0767 --- /dev/null +++ b/utils/__init__.py @@ -0,0 +1,5 @@ +"""ComfyUI-Sharp utilities.""" + +from .image import comfy_to_numpy_rgb, convert_focallength + +__all__ = ["comfy_to_numpy_rgb", "convert_focallength"] diff --git a/utils/image.py b/utils/image.py new file mode 100644 index 0000000..d390525 --- /dev/null +++ b/utils/image.py @@ -0,0 +1,39 @@ +"""Image conversion utilities for ComfyUI-Sharp.""" + +import numpy as np +import torch + + +def comfy_to_numpy_rgb(image_tensor: torch.Tensor) -> np.ndarray: + """Convert ComfyUI image tensor to numpy RGB array. + + Args: + image_tensor: ComfyUI image tensor with shape [B, H, W, C] in range [0, 1]. + + Returns: + Numpy array with shape [H, W, C] in range [0, 255] as uint8. + """ + if image_tensor.dim() == 4: + img = image_tensor[0] # Take first image from batch + else: + img = image_tensor + + img_np = img.cpu().numpy() + img_np = (img_np * 255).astype(np.uint8) + + return img_np + + +def convert_focallength(width: float, height: float, f_mm: float = 30.0) -> float: + """Convert focal length from mm (35mm film equivalent) to pixels. + + Args: + width: Image width in pixels. + height: Image height in pixels. + f_mm: Focal length in mm (35mm film equivalent). Default 30mm. + + Returns: + Focal length in pixels. + """ + # 35mm film dimensions: 36mm x 24mm + return f_mm * np.sqrt(width**2 + height**2) / np.sqrt(36**2 + 24**2) diff --git a/workflows/sharp_basic.json b/workflows/sharp_basic.json new file mode 100644 index 0000000..3c6e311 --- /dev/null +++ b/workflows/sharp_basic.json @@ -0,0 +1,148 @@ +{ + "id": "sharp-basic-workflow", + "revision": 0, + "last_node_id": 3, + "last_link_id": 2, + "nodes": [ + { + "id": 1, + "type": "LoadImage", + "pos": [ + -150, + -100 + ], + "size": [ + 274, + 314 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 2 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [] + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "example.png", + "image" + ] + }, + { + "id": 2, + "type": "LoadSharpModel", + "pos": [ + -150, + -400 + ], + "size": [ + 270, + 106 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "model", + "type": "SHARP_MODEL", + "links": [ + 1 + ] + } + ], + "properties": { + "Node name for S&R": "LoadSharpModel" + }, + "widgets_values": [ + "auto", + "" + ] + }, + { + "id": 3, + "type": "SharpPredict", + "pos": [ + 250, + -250 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "SHARP_MODEL", + "link": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 2 + } + ], + "outputs": [ + { + "name": "ply_path", + "type": "STRING", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SharpPredict" + }, + "widgets_values": [ + 0, + "sharp" + ] + } + ], + "links": [ + [ + 1, + 2, + 0, + 3, + 0, + "SHARP_MODEL" + ], + [ + 2, + 1, + 0, + 3, + 1, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 1.0, + "offset": [ + 400, + 600 + ] + } + }, + "version": 0.4 +}