diff --git a/comfy-env-root.toml b/comfy-env-root.toml index be07239..a1f2972 100644 --- a/comfy-env-root.toml +++ b/comfy-env-root.toml @@ -1,2 +1,3 @@ [node_reqs] -ComfyUI-GeometryPack = "PozzettiAndrea/ComfyUI-GeometryPack" \ No newline at end of file +ComfyUI-GeometryPack = "PozzettiAndrea/ComfyUI-GeometryPack" +ComfyUI-GaussianPack = "PozzettiAndrea/ComfyUI-GaussianPack" \ No newline at end of file diff --git a/nodes/__init__.py b/nodes/__init__.py index 4809561..fdb4e44 100644 --- a/nodes/__init__.py +++ b/nodes/__init__.py @@ -28,6 +28,14 @@ from .panorama_icosahedron_split import NODE_CLASS_MAPPINGS as ICOSAHEDRON_SPLIT from .panorama_icosahedron_split import NODE_DISPLAY_NAME_MAPPINGS as ICOSAHEDRON_SPLIT_DISPLAY_MAPPINGS from .build_mesh import NODE_CLASS_MAPPINGS as BUILD_MESH_MAPPINGS from .build_mesh import NODE_DISPLAY_NAME_MAPPINGS as BUILD_MESH_DISPLAY_MAPPINGS +from .predict_metric_depth import NODE_CLASS_MAPPINGS as PREDICT_MDEPTH_MAPPINGS +from .predict_metric_depth import NODE_DISPLAY_NAME_MAPPINGS as PREDICT_MDEPTH_DISPLAY_MAPPINGS +from .predict_gaussians_from_depth import NODE_CLASS_MAPPINGS as PREDICT_GFD_MAPPINGS +from .predict_gaussians_from_depth import NODE_DISPLAY_NAME_MAPPINGS as PREDICT_GFD_DISPLAY_MAPPINGS +from .predict_foreground import NODE_CLASS_MAPPINGS as PREDICT_FG_MAPPINGS +from .predict_foreground import NODE_DISPLAY_NAME_MAPPINGS as PREDICT_FG_DISPLAY_MAPPINGS +from .image_attrs_to_ply import NODE_CLASS_MAPPINGS as IMAGE_ATTRS_MAPPINGS +from .image_attrs_to_ply import NODE_DISPLAY_NAME_MAPPINGS as IMAGE_ATTRS_DISPLAY_MAPPINGS # Aggregate all node mappings NODE_CLASS_MAPPINGS = {} @@ -47,6 +55,10 @@ NODE_CLASS_MAPPINGS.update(REFINED_DEPTH_MAPPINGS) NODE_CLASS_MAPPINGS.update(PREDICT_ATTRS_MAPPINGS) NODE_CLASS_MAPPINGS.update(ICOSAHEDRON_SPLIT_MAPPINGS) NODE_CLASS_MAPPINGS.update(BUILD_MESH_MAPPINGS) +NODE_CLASS_MAPPINGS.update(PREDICT_MDEPTH_MAPPINGS) +NODE_CLASS_MAPPINGS.update(PREDICT_GFD_MAPPINGS) +NODE_CLASS_MAPPINGS.update(PREDICT_FG_MAPPINGS) +NODE_CLASS_MAPPINGS.update(IMAGE_ATTRS_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(LOAD_MODEL_DISPLAY_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(PREDICT_DISPLAY_MAPPINGS) @@ -62,5 +74,9 @@ NODE_DISPLAY_NAME_MAPPINGS.update(REFINED_DEPTH_DISPLAY_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(PREDICT_ATTRS_DISPLAY_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(ICOSAHEDRON_SPLIT_DISPLAY_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(BUILD_MESH_DISPLAY_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(PREDICT_MDEPTH_DISPLAY_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(PREDICT_GFD_DISPLAY_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(PREDICT_FG_DISPLAY_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(IMAGE_ATTRS_DISPLAY_MAPPINGS) __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] diff --git a/nodes/image_attrs_to_ply.py b/nodes/image_attrs_to_ply.py new file mode 100644 index 0000000..065553b --- /dev/null +++ b/nodes/image_attrs_to_ply.py @@ -0,0 +1,275 @@ +"""SharpImageAttrsToPLY — write a PLY from a MULTIBAND_IMAGE of gaussian attrs. + +Consumes the `first_layer_gaussian_attrs` / `second_layer_gaussian_attrs` +output of `SharpPredictGaussianAttrs` (or any 14-channel MULTIBAND_IMAGE +in SHARP's gaussian schema) and emits a PLY file in world coordinates. + +Channel order is fixed by `SharpPredictGaussianAttrs.GAUSS_ATTR_CHANNEL_NAMES`: + + 0: position_x (NDC) + 1: position_y (NDC) + 2: position_z (NDC / camera-space z) + 3: scale_x (Σ-singular value, world units after unprojection) + 4: scale_y + 5: scale_z + 6: quaternion_w + 7: quaternion_x + 8: quaternion_y + 9: quaternion_z + 10: color_r (linear or SH-DC) + 11: color_g + 12: color_b + 13: opacity (post-activation, in [0, 1]) + +We rebuild a `Gaussians3D` namedtuple in NDC, then call SHARP's +`unproject_gaussians` to transform to world coordinates using the +per-face extrinsics + intrinsics. Output convention matches SharpPredict: +single batch → one .ply; N>1 batch → folder of NNN.ply. +""" + +import logging +import os +import sys +import time +from pathlib import Path + +import numpy as np +import torch + +from comfy_api.latest import io + +log = logging.getLogger("sharp") + +try: + import folder_paths + OUTPUT_DIR = folder_paths.get_output_directory() +except ImportError: + OUTPUT_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "output") + + +def _p(msg: str) -> None: + print(f"[SharpImageAttrsToPLY] {msg}", file=sys.stderr, flush=True) + + +# Channel offsets within the 14-channel multiband — must match +# `predict_gaussian_attrs.GAUSS_ATTR_CHANNEL_NAMES`. +POS = slice(0, 3) +SCALE = slice(3, 6) +QUAT = slice(6, 10) +COLOR = slice(10, 13) +OPACITY = 13 + + +class SharpImageAttrsToPLY(io.ComfyNode): + """MULTIBAND_IMAGE of gaussian attrs + camera → PLY file.""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SharpImageAttrsToPLY", + display_name="SHARP Image + Attrs to PLY", + category="SHARP", + description=( + "Take a 14-channel MULTIBAND_IMAGE of per-pixel gaussian " + "attributes (e.g. from SharpPredictGaussianAttrs's " + "first_layer_gaussian_attrs output) + the matching camera " + "extrinsics/intrinsics and write a 3D Gaussian Splatting " + "PLY file. The multiband stores positions in NDC; this " + "node unprojects to world coords via SHARP's " + "unproject_gaussians before saving." + ), + is_output_node=True, + inputs=[ + io.Custom("MULTIBAND_IMAGE").Input( + "gaussian_attrs", + tooltip="14-channel multiband (samples [B, 14, H, W]) " + "with channel order position_xyz, scale_xyz, " + "quat_wxyz, color_rgb, opacity. Typically the " + "first_layer_gaussian_attrs or " + "second_layer_gaussian_attrs output of " + "SharpPredictGaussianAttrs."), + io.Custom("EXTRINSICS").Input( + "extrinsics", + tooltip="Per-batch extrinsics matching the multiband " + "batch (4×4 each). From " + "SharpPredictGaussianAttrs.extrinsics."), + io.Custom("INTRINSICS").Input( + "intrinsics", + tooltip="Per-batch intrinsics rescaled to the gaussian " + "grid (3×3 or 4×4 pixel-K for H×W). From " + "SharpPredictGaussianAttrs.intrinsics."), + io.String.Input( + "output_prefix", default="sharp_attrs", + optional=True, + tooltip="Basename for the output PLY (single) or batch " + "folder (N>1)."), + io.Float.Input( + "focal_length_px", default=0.0, min=0.0, max=8192.0, + step=1.0, optional=True, + tooltip="Override focal length in pixels saved into the " + "PLY metadata. 0 = use intrinsics[0, 0] directly."), + ], + outputs=[ + io.String.Output(display_name="ply_path"), + io.Custom("EXTRINSICS").Output(display_name="extrinsics"), + io.Custom("INTRINSICS").Output(display_name="intrinsics"), + ], + ) + + @classmethod + @torch.no_grad() + def execute( + cls, + gaussian_attrs: dict, + extrinsics: torch.Tensor, + intrinsics: torch.Tensor, + output_prefix: str = "sharp_attrs", + focal_length_px: float = 0.0, + ): + import comfy.utils + from .sharp.gaussians import Gaussians3D, save_ply, unproject_gaussians + + # ---- unpack the multiband --------------------------------------- + if not isinstance(gaussian_attrs, dict) or "samples" not in gaussian_attrs: + raise ValueError( + f"SharpImageAttrsToPLY: gaussian_attrs must be a MULTIBAND_IMAGE " + f"dict with key 'samples', got {type(gaussian_attrs).__name__}" + ) + samples = gaussian_attrs["samples"] # [B, C, H, W] + if samples.dim() != 4 or samples.shape[1] < 14: + raise ValueError( + f"SharpImageAttrsToPLY: samples must be [B, 14+, H, W], " + f"got {tuple(samples.shape)}" + ) + B, C, H, W = samples.shape + if C < 14: + raise ValueError(f"need at least 14 channels, got {C}") + + # ---- normalize camera inputs ------------------------------------ + ext = extrinsics.detach().float() if isinstance(extrinsics, torch.Tensor) \ + else torch.as_tensor(np.asarray(extrinsics)).float() + intr = intrinsics.detach().float() if isinstance(intrinsics, torch.Tensor) \ + else torch.as_tensor(np.asarray(intrinsics)).float() + if ext.dim() == 2: + ext = ext.unsqueeze(0) + if intr.dim() == 2: + intr = intr.unsqueeze(0) + if ext.shape[0] != B: + raise ValueError( + f"extrinsics batch {ext.shape[0]} != multiband batch {B}" + ) + if intr.shape[0] != B: + raise ValueError( + f"intrinsics batch {intr.shape[0]} != multiband batch {B}" + ) + + device = samples.device if samples.is_cuda else torch.device( + "cuda" if torch.cuda.is_available() else "cpu" + ) + samples = samples.to(device).float() + ext = ext.to(device) + intr = intr.to(device) + + # ---- output path convention (matches SharpPredict) -------------- + os.makedirs(OUTPUT_DIR, exist_ok=True) + timestamp = int(time.time() * 1000) + if B == 1: + output_path = os.path.join(OUTPUT_DIR, f"{output_prefix}_{timestamp}.ply") + output_folder = None + is_batch = False + else: + output_folder = os.path.join(OUTPUT_DIR, f"{output_prefix}_{timestamp}") + os.makedirs(output_folder, exist_ok=True) + output_path = output_folder + is_batch = True + + all_ply_paths = [] + all_extrinsics = [] + all_intrinsics = [] + + pbar = comfy.utils.ProgressBar(B) + t_start = time.time() + + for b in range(B): + # ---- extract per-pixel attrs → Gaussians3D in NDC ------------ + s = samples[b] # [14, H, W] + n_pixels = H * W + + # Multiband stores [C, H, W]; flatten spatial dims → [C, N], then + # transpose to [N, C] for the per-channel slices below. + flat = s.reshape(C, n_pixels).T # [N, C] + mean_ndc = flat[:, POS].contiguous() # [N, 3] + scales = flat[:, SCALE].contiguous() # [N, 3] + quats = flat[:, QUAT].contiguous() # [N, 4] + colors = flat[:, COLOR].contiguous() # [N, 3] + opacs = flat[:, OPACITY].contiguous() # [N] + + # Gaussians3D expects [B=1, N, C] for mean/scale/quat/color, + # [B=1, N] for opacity. + gaussians_ndc = Gaussians3D( + mean_vectors=mean_ndc.unsqueeze(0), + singular_values=scales.unsqueeze(0), + quaternions=quats.unsqueeze(0), + colors=colors.unsqueeze(0), + opacities=opacs.unsqueeze(0), + ) + + # ---- camera matrices for unprojection ------------------------ + ext_b = ext[b] # [4, 4] + K_b = intr[b].clone() # [3, 3] or [4, 4] + if K_b.shape == (3, 3): + K4 = torch.eye(4, device=device, dtype=K_b.dtype) + K4[:3, :3] = K_b + K_b = K4 + + # SHARP's internal convention: scale K to internal_shape + # (the resolution used during decode). The multiband H/W + # already match that internal grid (768 for the gaussian + # decoder head), so no further rescale is needed — but + # `unproject_gaussians` wants image_shape=(H, W) matching K's + # native resolution. The intrinsics input is already pixel-K + # for (H, W) (per the rescale done in SharpPredictGaussianAttrs). + image_shape = (H, W) + + # ---- NDC → world --------------------------------------------- + gaussians_world = unproject_gaussians( + gaussians_ndc, ext_b, K_b, image_shape, + ) + + # ---- save PLY ------------------------------------------------ + if is_batch: + ply_path = os.path.join(output_folder, f"{b+1:03d}.ply") + else: + ply_path = output_path + + f_px = float(focal_length_px) if focal_length_px > 0 \ + else float(K_b[0, 0].item()) + _, metadata = save_ply( + gaussians_world, f_px, image_shape, Path(ply_path), + ) + + all_ply_paths.append(ply_path) + all_extrinsics.append(metadata["extrinsic"]) + all_intrinsics.append(metadata["intrinsic"]) + pbar.update(1) + + elapsed = time.time() - t_start + loc = output_folder if is_batch else output_path + total_n = B * H * W + _p( + f"{B} multiband(s) {H}×{W} → {total_n/1e6:.2f}M gaussians; " + f"saved to {loc}; {elapsed:.1f}s" + ) + + return io.NodeOutput( + output_path, all_extrinsics[0], all_intrinsics[0], + ) + + +NODE_CLASS_MAPPINGS = { + "SharpImageAttrsToPLY": SharpImageAttrsToPLY, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "SharpImageAttrsToPLY": "SHARP Image + Attrs to PLY", +} diff --git a/nodes/merge_gaussians.py b/nodes/merge_gaussians.py index 328e790..1cac321 100644 --- a/nodes/merge_gaussians.py +++ b/nodes/merge_gaussians.py @@ -5,6 +5,7 @@ Merges multiple PLY files (from panorama samples) into a single unified Gaussian import logging import os +import sys import time from pathlib import Path @@ -16,6 +17,11 @@ from comfy_api.latest import io log = logging.getLogger("sharp") +def _p(msg: str) -> None: + """Print to stderr so the line shows up in ComfyUI's worker log.""" + print(f"[MergeGaussians] {msg}", file=sys.stderr, flush=True) + + def load_ply_simple(path: str) -> dict: """Load Gaussian data from PLY file. @@ -134,7 +140,18 @@ class MergeGaussians(io.ComfyNode): tooltip="Filter out Gaussians with opacity below this (0 = no filter)"), ], outputs=[ - io.String.Output(display_name="ply_path"), + io.String.Output( + display_name="ply_paths", + tooltip="Passthrough of the input ply_folder. Wire to " + "downstream nodes that want to operate on the " + "original per-face PLYs (e.g. SharpDedupGaussiansEquirect " + "on the merged_ply_path while still keeping access " + "to the originals)."), + io.String.Output( + display_name="merged_ply_path", + tooltip="Path to the merged output PLY file (a single " + "PLY containing every gaussian from every input " + "PLY, post-filter)."), io.Int.Output(display_name="num_gaussians"), ], ) @@ -148,72 +165,79 @@ class MergeGaussians(io.ComfyNode): min_opacity: float = 0.0, ): """Merge all PLY files in folder.""" + import comfy.utils - # Find all PLY files + t_start = time.time() + + ply_folder_in = ply_folder ply_folder = Path(ply_folder) if not ply_folder.exists(): - raise ValueError(f"PLY folder does not exist: {ply_folder}") + raise ValueError(f"PLY folder does not exist: {ply_folder_in!r}") ply_files = sorted(ply_folder.glob("*.ply")) if not ply_files: raise ValueError(f"No PLY files found in: {ply_folder}") - log.info(f"Found {len(ply_files)} PLY files to merge") + filt_str = [] + if max_depth > 0: + filt_str.append(f"max_depth={max_depth:.2f}m") + if min_opacity > 0: + filt_str.append(f"min_opacity={min_opacity:.3f}") + filt_msg = (" + filters " + ", ".join(filt_str)) if filt_str else " (no filters)" + _p(f"merging {len(ply_files)} PLY file(s) from {ply_folder}{filt_msg}") - # Load all PLY files all_positions = [] all_colors = [] all_scales = [] all_rotations = [] all_opacities = [] + n_in_total = 0 + n_filtered_total = 0 + pbar = comfy.utils.ProgressBar(len(ply_files)) for i, ply_path in enumerate(ply_files): - log.info(f"Loading {ply_path.name} ({i+1}/{len(ply_files)})") data = load_ply_simple(str(ply_path)) - positions = data['positions'] colors = data['colors'] scales = data['scales'] rotations = data['rotations'] opacities = data['opacities'] - # Apply filters - mask = np.ones(len(positions), dtype=bool) + n_before = len(positions) + n_in_total += n_before + mask = np.ones(n_before, dtype=bool) if max_depth > 0: - # Filter by depth (distance from origin) depths = np.linalg.norm(positions, axis=-1) mask &= depths <= max_depth - if min_opacity > 0: mask &= opacities >= min_opacity - - filtered_count = (~mask).sum() - if filtered_count > 0: - log.info(f" Filtered out {filtered_count:,} Gaussians") + filtered_count = int((~mask).sum()) + n_filtered_total += filtered_count all_positions.append(positions[mask]) all_colors.append(colors[mask]) all_scales.append(scales[mask]) all_rotations.append(rotations[mask]) all_opacities.append(opacities[mask]) + pbar.update(1) - # Concatenate all merged_positions = np.concatenate(all_positions, axis=0) merged_colors = np.concatenate(all_colors, axis=0) merged_scales = np.concatenate(all_scales, axis=0) merged_rotations = np.concatenate(all_rotations, axis=0) merged_opacities = np.concatenate(all_opacities, axis=0) - num_gaussians = len(merged_positions) - log.info(f"Total Gaussians after merge: {num_gaussians:,}") - # Save merged PLY + if n_filtered_total > 0: + pct = 100.0 * n_filtered_total / max(n_in_total, 1) + _p(f"filtered {n_filtered_total:,} / {n_in_total:,} gaussians ({pct:.1f}%)") + timestamp = int(time.time() * 1000) output_filename = f"{output_prefix}_{timestamp}.ply" output_path = ply_folder.parent / output_filename - log.info(f"Saving to {output_path}") + t_save = time.time() save_merged_ply( merged_positions, merged_colors, @@ -222,10 +246,22 @@ class MergeGaussians(io.ComfyNode): merged_opacities, str(output_path), ) + save_elapsed = time.time() - t_save - log.info(f"Done! Merged {len(ply_files)} files into {num_gaussians:,} Gaussians") + try: + size_mb = output_path.stat().st_size / (1024 * 1024) + size_str = f", {size_mb:.1f} MB" + except Exception: + size_str = "" - return io.NodeOutput(str(output_path), num_gaussians) + elapsed = time.time() - t_start + _p( + f"merged {len(ply_files)} file(s) → {num_gaussians/1e6:.2f}M gaussians " + f"→ {output_path}{size_str}; " + f"save {save_elapsed:.1f}s, total {elapsed:.1f}s" + ) + + return io.NodeOutput(str(ply_folder_in), str(output_path), num_gaussians) NODE_CLASS_MAPPINGS = { diff --git a/nodes/predict_foreground.py b/nodes/predict_foreground.py new file mode 100644 index 0000000..7d6e548 --- /dev/null +++ b/nodes/predict_foreground.py @@ -0,0 +1,218 @@ +"""SharpPredictForeground node for ComfyUI-Sharp. + +Same as SharpPredict, but emits ONLY layer 0 (visible/front surface) — drops +layer 1 (back/occluded surface). Halves the per-image gaussian count +(768²×1 instead of 768²×2). Use when you don't need backside detail and want +smaller PLYs / faster downstream rendering. + +The gaussian_composer's flatten order is (layer, h, w), so the first +H_grid × W_grid entries of the returned Gaussians3D are layer 0; we slice +to that range before save_ply. +""" + +import logging +import os +import time +from pathlib import Path + +import torch + +from comfy_api.latest import io + +from .predict import _predict_image_cached +from .utils.image import comfy_to_numpy_rgb, convert_focallength + +log = logging.getLogger("sharp") + +try: + import folder_paths + OUTPUT_DIR = folder_paths.get_output_directory() +except ImportError: + OUTPUT_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "output") + + +class SharpPredictForeground(io.ComfyNode): + """Run SHARP inference and save PLY containing ONLY layer-0 gaussians. + + Identical wiring to SharpPredict (same inputs, same outputs). The only + difference is the foreground-only slice applied before save_ply. + """ + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SharpPredictForeground", + display_name="SHARP Predict Foreground (Image to PLY)", + category="SHARP", + description=( + "Generate 3D Gaussian Splatting PLY file(s) from image(s) " + "using SHARP, keeping only layer-0 (visible/front surface) " + "gaussians and dropping layer-1 (back/occluded). Halves the " + "per-image gaussian count vs SharpPredict. Batch input creates " + "a folder with numbered PLY files." + ), + is_output_node=True, + inputs=[ + io.Custom("SHARP_MODEL_CONFIG").Input("model"), + io.Image.Input("image"), + io.Float.Input( + "focal_length_mm", default=30.0, min=0.0, max=500.0, + step=0.1, optional=True, + tooltip="Focal length in mm (35mm equiv). 0 = auto (30mm). " + "Ignored if intrinsics provided."), + io.String.Input( + "output_prefix", default="sharp_fg", optional=True, + tooltip="Prefix for output PLY filename or batch folder."), + io.Custom("EXTRINSICS").Input( + "extrinsics", optional=True, + tooltip="Camera extrinsics (from SamplePanorama). If " + "batched, must match image batch size."), + io.Custom("INTRINSICS").Input( + "intrinsics", optional=True, + tooltip="Camera intrinsics. Overrides focal_length_mm."), + ], + outputs=[ + io.String.Output(display_name="ply_path"), + io.Custom("EXTRINSICS").Output(display_name="extrinsics"), + io.Custom("INTRINSICS").Output(display_name="intrinsics"), + ], + ) + + @classmethod + @torch.no_grad() + def execute( + cls, + model, + image: torch.Tensor, + focal_length_mm: float = 0.0, + output_prefix: str = "sharp_fg", + extrinsics: torch.Tensor = None, + intrinsics: torch.Tensor = None, + ): + import comfy.model_management + import comfy.utils + from .load_model import _load_sharp_model + from .sharp.gaussians import save_ply + + patcher = _load_sharp_model(model) + predictor = patcher.model + device = patcher.load_device + + if image.dim() == 3: + image = image.unsqueeze(0) + batch_size = image.shape[0] + + # Layer-0 count = output_resolution². + H_grid = W_grid = int(predictor.output_resolution) + n_layer0 = H_grid * W_grid + + has_camera_params = extrinsics is not None and intrinsics is not None + if has_camera_params: + if extrinsics.dim() == 2: + extrinsics = extrinsics.unsqueeze(0) + if extrinsics.shape[0] != batch_size: + raise ValueError( + f"Extrinsics batch size ({extrinsics.shape[0]}) must match " + f"image batch size ({batch_size})" + ) + log.info( + f"Processing {batch_size} image(s) with provided camera " + f"parameters (foreground-only, layer 0 = {n_layer0} gaussians/face)" + ) + else: + log.info( + f"Processing {batch_size} image(s) (foreground-only, " + f"layer 0 = {n_layer0} gaussians/face)" + ) + + os.makedirs(OUTPUT_DIR, exist_ok=True) + timestamp = int(time.time() * 1000) + + if batch_size == 1: + output_filename = f"{output_prefix}_{timestamp}.ply" + output_path = os.path.join(OUTPUT_DIR, output_filename) + output_folder = None + is_batch = False + else: + folder_name = f"{output_prefix}_{timestamp}" + output_folder = os.path.join(OUTPUT_DIR, folder_name) + os.makedirs(output_folder, exist_ok=True) + output_path = output_folder + is_batch = True + + all_ply_paths = [] + all_extrinsics = [] + all_intrinsics = [] + + inference_start = time.time() + pbar = comfy.utils.ProgressBar(batch_size) + + for i in range(batch_size): + comfy.model_management.throw_exception_if_processing_interrupted() + single_image = image[i:i+1] + image_np = comfy_to_numpy_rgb(single_image) + height, width = image_np.shape[:2] + + if i == 0: + log.info(f"Image size: {width}x{height}") + + if has_camera_params: + img_intrinsics = intrinsics.to(device) + img_extrinsics = extrinsics[i].to(device) + f_px = img_intrinsics[0, 0].item() + else: + if focal_length_mm > 0: + f_px = convert_focallength(width, height, focal_length_mm) + else: + f_px = convert_focallength(width, height, 30.0) + img_extrinsics = None + img_intrinsics = None + + log.info(f"Running inference on image {i+1}/{batch_size}...") + gaussians = _predict_image_cached( + patcher, predictor, image_np, f_px, device, + extrinsics=img_extrinsics, + intrinsics=img_intrinsics, + ) + + # Foreground-only slice: drop layer 1 (occluded surface). + # Composer flatten order is (layer, h, w), so [0:n_layer0] = layer 0. + gaussians = gaussians._replace( + mean_vectors=gaussians.mean_vectors[:, :n_layer0], + singular_values=gaussians.singular_values[:, :n_layer0], + quaternions=gaussians.quaternions[:, :n_layer0], + colors=gaussians.colors[:, :n_layer0], + opacities=gaussians.opacities[:, :n_layer0], + ) + + if is_batch: + ply_filename = f"{i+1:03d}.ply" + ply_path = os.path.join(output_folder, ply_filename) + else: + ply_path = output_path + + _, metadata = save_ply(gaussians, f_px, (height, width), Path(ply_path)) + + all_ply_paths.append(ply_path) + all_extrinsics.append(metadata["extrinsic"]) + all_intrinsics.append(metadata["intrinsic"]) + + log.info(f"Saved: {ply_path} ({metadata['num_gaussians']:,} gaussians, layer 0)") + pbar.update(1) + + inference_time = time.time() - inference_start + log.info( + f"Total inference time: {inference_time:.2f}s " + f"({inference_time/batch_size:.2f}s per image)" + ) + + return io.NodeOutput(output_path, all_extrinsics[0], all_intrinsics[0]) + + +NODE_CLASS_MAPPINGS = { + "SharpPredictForeground": SharpPredictForeground, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "SharpPredictForeground": "SHARP Predict Foreground (Image to PLY)", +} diff --git a/nodes/predict_gaussian_attrs.py b/nodes/predict_gaussian_attrs.py index a92e826..33d81d4 100644 --- a/nodes/predict_gaussian_attrs.py +++ b/nodes/predict_gaussian_attrs.py @@ -57,6 +57,7 @@ from __future__ import annotations import hashlib import logging +import sys import time import numpy as np @@ -65,6 +66,13 @@ import torch.nn.functional as F from comfy_api.latest import io + +def _p(msg: str) -> None: + """Print to stderr so the line shows up in ComfyUI's worker log + (`log.info`/`log.debug` on the 'sharp' logger don't always get routed + through the comfy-env worker stderr pipe — direct print does).""" + print(f"[SharpPredictGaussianAttrs] {msg}", file=sys.stderr, flush=True) + log = logging.getLogger("sharp") @@ -224,10 +232,37 @@ class SharpPredictGaussianAttrs(io.ComfyNode): "output."), ], outputs=[ + io.Image.Output( + display_name="metric_depth", + tooltip="[B, 1536, 1536, 3] per-pixel metric depth from " + "SHARP's disparity head at NATIVE 1536² resolution " + "(`disparity_factor / monodepth_output.disparity`). " + "Unlike the layer_depth_refined outputs (gaussian z " + "with sub-pixel xy drift), this is a true per-pixel " + "depth map — wire with `extrinsics_mdepth + " + "intrinsics_mdepth` into SharpDepthMerge for a " + "seam-free LSMR merge that matches MoGe2-style " + "smoothness. Memory: 42 faces × 1536² fp32 ≈ 395 MB."), + io.Custom("EXTRINSICS").Output( + display_name="extrinsics_mdepth", + tooltip="Pass-through of input extrinsics. Extrinsics are " + "resolution-independent (world-to-camera transform); " + "this is just the same matrix paired with metric_depth " + "so workflows can wire the metric-depth path as a " + "self-contained triplet."), + io.Custom("INTRINSICS").Output( + display_name="intrinsics_mdepth", + tooltip="Intrinsics rescaled to the 1536² metric_depth grid " + "(pixel-K). Pairs with `metric_depth` so the " + "downstream SharpDepthMerge's K/depth-shape " + "invariant holds without conversion."), io.Image.Output( display_name="first_layer_depth_refined", tooltip="[B, H, W, 3] depth broadcast across 3 channels. " - "Layer-0 (visible surface) post-decoder z."), + "Layer-0 (visible surface) post-decoder z. NOTE: " + "this is the gaussian's z position; the gaussian's " + "xy may have drifted from the source pixel grid. " + "Use `metric_depth` instead for clean per-pixel depth."), io.Custom("MULTIBAND_IMAGE").Output( display_name="first_layer_gaussian_attrs", tooltip="[B, 14, H, W] all layer-0 gaussian attributes. " @@ -279,6 +314,7 @@ class SharpPredictGaussianAttrs(io.ComfyNode): first_attrs = [] second_depths = [] second_attrs = [] + metric_depths = [] # disparity-head depth, downsampled to gaussian grid # Load model to GPU with the budget the existing predict path uses. internal_shape = (1536, 1536) @@ -334,6 +370,20 @@ class SharpPredictGaussianAttrs(io.ComfyNode): f_px = (width / 36.0) * max(0.1, float(focal_length_mm or 30.0)) disparity_factor = torch.tensor([f_px / width]).float().to(device) + + # Disparity-head metric depth at SHARP's native 1536². Matches + # predict_depth.py's convention: `disparity_factor / + # monodepth_disparity`. Emitted at native res rather than + # downsampled to the gaussian grid (768) so callers get the + # full disparity-head detail; the paired `intrinsics_mdepth` + # output below carries the matching pixel-K for 1536. + monodepth_disparity = monodepth_output.disparity # [1, 1, 1536, 1536] + metric_depth_full = ( + disparity_factor.view(1, 1, 1, 1) + / monodepth_disparity.clamp(min=1e-4) + ) # [1, 1, 1536, 1536] + metric_depths.append(metric_depth_full[0, 0].cpu()) # [1536, 1536] + t1 = time.time() gaussians_ndc = predictor.decode(monodepth_output, image_resized_pt, disparity_factor) log.debug(f" [{b}] decode time: {time.time() - t1:.2f}s") @@ -358,54 +408,78 @@ class SharpPredictGaussianAttrs(io.ComfyNode): d1_batch = torch.cat(second_depths, dim=0) attrs0_batch = torch.cat(first_attrs, dim=0) # [B, H, W, 14] attrs1_batch = torch.cat(second_attrs, dim=0) + metric_batch = torch.stack(metric_depths, dim=0) # [B, H, W] # IMAGE format: [B, H, W, 3] depth broadcast over channels. d0_img = d0_batch.unsqueeze(-1).expand(-1, -1, -1, 3).contiguous() d1_img = d1_batch.unsqueeze(-1).expand(-1, -1, -1, 3).contiguous() + metric_img = metric_batch.unsqueeze(-1).expand(-1, -1, -1, 3).contiguous() # MULTIBAND_IMAGE: dict with samples [B, 14, H, W] + channel_names. l0_mb = _build_multiband(attrs0_batch) l1_mb = _build_multiband(attrs1_batch) - # Rescale intrinsics so their pixel-space matches the emitted depth grid - # (H_grid x W_grid), not the input face shape. Without this the - # downstream merger normalizes K by the wrong W and ends up with the - # wrong effective FOV per face -> visible seams in the merged equirect. - if intrinsics is not None: - sx = float(W_grid) / float(width) - sy = float(H_grid) / float(height) - intr = intrinsics.detach().clone().float() if isinstance(intrinsics, torch.Tensor) \ - else torch.as_tensor(np.asarray(intrinsics)).float().clone() - if intr.dim() == 2: - intr[0, :] *= sx - intr[1, :] *= sy - k_dbg = intr - elif intr.dim() == 3: - intr[:, 0, :] *= sx - intr[:, 1, :] *= sy - k_dbg = intr[0] + # Rescale intrinsics so pixel-K matches the emitted depth grid for + # each output pair. The downstream merger normalizes K by the depth + # tensor's W; if K's native res doesn't match that W, the effective + # FOV is wrong → visible seams. We emit TWO K versions: + # - intrinsics : K rescaled to (H_grid, W_grid) = 768², + # pairs with gaussian-z outputs. + # - intrinsics_mdepth: K rescaled to (1536, 1536), + # pairs with metric_depth. + # extrinsics are resolution-independent — same tensor for both pairs. + def _rescale_K(intr_in, target_w, target_h, src_w, src_h): + sx = float(target_w) / float(src_w) + sy = float(target_h) / float(src_h) + k = intr_in.detach().clone().float() if isinstance(intr_in, torch.Tensor) \ + else torch.as_tensor(np.asarray(intr_in)).float().clone() + if k.dim() == 2: + k[0, :] *= sx + k[1, :] *= sy + k_dbg = k + elif k.dim() == 3: + k[:, 0, :] *= sx + k[:, 1, :] *= sy + k_dbg = k[0] else: - raise ValueError(f"intrinsics must be (3,3) or (N,3,3), got {tuple(intr.shape)}") - intrinsics_out = intr - k_fx_dbg = float(k_dbg[0, 0]) + raise ValueError(f"intrinsics must be (3,3) or (N,3,3), got {tuple(k.shape)}") + return k, float(k_dbg[0, 0]) + + if intrinsics is not None: + # Gaussian-pair K (768²) + intrinsics_out, k_fx_dbg = _rescale_K( + intrinsics, W_grid, H_grid, width, height, + ) + # Metric-depth-pair K (1536²) + intrinsics_mdepth_out, k_fx_md_dbg = _rescale_K( + intrinsics, internal_shape[1], internal_shape[0], width, height, + ) else: intrinsics_out = None + intrinsics_mdepth_out = None k_fx_dbg = None + k_fx_md_dbg = None + extrinsics_mdepth_out = extrinsics # resolution-independent pass-through n_gaussians_total = B * num_layers * H_grid * W_grid elapsed = time.time() - t_start d0_med = float(d0_batch.median()) d1_med = float(d1_batch.median()) - k_str = f", K fx→{k_fx_dbg:.1f}" if k_fx_dbg is not None else "" - log.info( - f"[SharpPredictGaussianAttrs] {B} face(s) → " - f"{n_gaussians_total/1e6:.2f}M gaussians " + m_med = float(metric_batch.median()) + k_str = ( + f", K_{H_grid} fx→{k_fx_dbg:.1f} K_{internal_shape[0]} fx→{k_fx_md_dbg:.1f}" + if k_fx_dbg is not None else "" + ) + _p( + f"{B} face(s) → {n_gaussians_total/1e6:.2f}M gaussians " f"({B}×{num_layers}×{H_grid}²); " - f"layer0/1 depth median={d0_med:.2f}m/{d1_med:.2f}m" + f"metric/layer0/layer1 depth median={m_med:.2f}m/{d0_med:.2f}m/{d1_med:.2f}m" f"{k_str}; {elapsed:.1f}s" ) return io.NodeOutput( + metric_img, + extrinsics_mdepth_out, intrinsics_mdepth_out, d0_img, l0_mb, d1_img, l1_mb, extrinsics, intrinsics_out, ) diff --git a/nodes/predict_gaussians_from_depth.py b/nodes/predict_gaussians_from_depth.py new file mode 100644 index 0000000..0161f64 --- /dev/null +++ b/nodes/predict_gaussians_from_depth.py @@ -0,0 +1,342 @@ +"""SharpPredictGaussiansFromMetricDepth — image + external depth → PLY paths. + +Like SharpPredict, but feeds an externally-provided metric depth into +`predictor.decode(..., depth=...)`. The decoder's depth_alignment module +(`model.py:2023`) blends the raw monodepth with the external depth before +init_model initializes gaussian base values, so the resulting gaussians +share a consistent geometric scaffold across multiple views — fixes the +per-face seams caused by per-view monocular scale ambiguity. + +Output convention matches SharpPredict: + - Single image → single PLY file at OUTPUT_DIR/{prefix}_{ts}.ply + - Batch (>1) → folder OUTPUT_DIR/{prefix}_{ts}/ with NNN.ply files + +Wire the batch folder straight into `MergeGaussians (PLY Files)` for a +single merged scene PLY. + +Typical pipeline: + 1. SharpPanoramaIcosahedronSplit → face images + 2. SharpPredictMetricDepth (per face) → raw per-face metric depths + 3. SharpDepthMerge → seam-free equirect depth (LSMR) + 4. SharpPanoramaIcosahedronSplit (on the merged depth IMAGE) + → re-cropped CONSISTENT face depths + 5. SharpPredictGaussiansFromMetricDepth(face_images, consistent_face_depths) + → folder of seam-aligned PLYs + 6. MergeGaussians → final unified PLY +""" + +from __future__ import annotations + +import os +import sys +import time +from pathlib import Path + +import numpy as np +import torch +import torch.nn.functional as F +from comfy_api.latest import io + +from .predict_gaussian_attrs import ( + _compute_image_hash, _monodepth_to, _encode_cache, +) + +try: + import folder_paths + OUTPUT_DIR = folder_paths.get_output_directory() +except ImportError: + OUTPUT_DIR = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "output", + ) + + +def _p(msg: str) -> None: + print(f"[SharpPredictGaussiansFromMetricDepth] {msg}", + file=sys.stderr, flush=True) + + +class SharpPredictGaussiansFromMetricDepth(io.ComfyNode): + """SHARP gaussian decoder driven by external depth → PLY paths.""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SharpPredictGaussiansFromMetricDepth", + display_name="SHARP Predict Gaussians (Image + Metric Depth)", + category="SHARP", + description=( + "Run SHARP's gaussian decoder using an externally-provided " + "metric depth as the alignment target. The external depth " + "becomes the geometric scaffold for `init_model`'s gaussian " + "base values, so gaussians share a consistent surface across " + "multiple views — fixes per-face seams.\n\n" + "Saves PLYs (single image → one .ply; batch → folder of " + "NNN.ply). Pipe into MergeGaussians to combine into one PLY." + ), + is_output_node=True, + inputs=[ + io.Custom("SHARP_MODEL_CONFIG").Input("model"), + io.Image.Input("image"), + io.Image.Input( + "metric_depth", + tooltip="External per-face metric depth (any resolution; " + "auto-resized to SHARP's internal 1536² before " + "alignment). Typically: SharpPredictMetricDepth " + "→ SharpDepthMerge → SharpPanoramaIcosahedronSplit " + "(depth as IMAGE)."), + io.Float.Input( + "focal_length_mm", default=30.0, min=0.0, max=500.0, + step=0.1, optional=True, + tooltip="Focal length in mm (35mm equiv). Ignored if " + "intrinsics provided."), + io.String.Input( + "output_prefix", default="sharp_aligned", optional=True, + tooltip="Prefix for PLY filename or batch folder name."), + io.Boolean.Input( + "save_background_layer", default=True, optional=True, + tooltip=( + "SHARP emits 2 gaussian layers per pixel: layer 0 " + "(visible/front surface) and layer 1 (back/occluded). " + "True (default): save both layers — 2× the gaussian " + "count, captures hidden surfaces revealed by parallax. " + "False: save only layer 0 — halves the gaussian count " + "(768² per face instead of 768²×2). Use False when " + "you'll voxel-dedup downstream anyway, or when the " + "scene has minimal occlusion (open outdoor)." + )), + io.Custom("EXTRINSICS").Input( + "extrinsics", optional=True, + tooltip="Per-face extrinsics from " + "SharpPanoramaIcosahedronSplit. Must match " + "image batch size if batched."), + io.Custom("INTRINSICS").Input( + "intrinsics", optional=True, + tooltip="Per-face intrinsics from " + "SharpPanoramaIcosahedronSplit (pixel-K for the " + "ORIGINAL face image resolution, not the merged " + "depth resolution). If absent, focal_length_mm " + "is used."), + ], + outputs=[ + io.String.Output( + display_name="ply_path", + tooltip="Single image: path to {prefix}_{ts}.ply. " + "Batch: path to the folder (identical to " + "ply_folder output)."), + io.String.Output( + display_name="ply_folder", + tooltip="Always the containing folder of the PLY(s): " + "OUTPUT_DIR/{prefix}_{ts}/ for batch, or the " + "parent dir for single-image runs. Wire into " + "MergeGaussians.ply_folder regardless of batch size."), + io.Custom("EXTRINSICS").Output(display_name="extrinsics"), + io.Custom("INTRINSICS").Output(display_name="intrinsics"), + ], + ) + + @classmethod + @torch.no_grad() + def execute( + cls, + model, + image: torch.Tensor, + metric_depth: torch.Tensor, + focal_length_mm: float = 30.0, + output_prefix: str = "sharp_aligned", + save_background_layer: bool = True, + extrinsics: torch.Tensor | None = None, + intrinsics: torch.Tensor | None = None, + ): + global _encode_cache + import comfy.model_management + import comfy.utils + from .load_model import _load_sharp_model + from .sharp.gaussians import save_ply, unproject_gaussians + + patcher = _load_sharp_model(model) + predictor = patcher.model + device = patcher.load_device + + if image.dim() == 3: + image = image.unsqueeze(0) + B = image.shape[0] + + if metric_depth.dim() == 3: + metric_depth = metric_depth.unsqueeze(0) + if metric_depth.shape[0] != B: + raise ValueError( + f"metric_depth batch {metric_depth.shape[0]} != image batch {B}" + ) + + # Sanity-check the alignment is wired through the model. + scale_map_estimator = predictor.depth_alignment.scale_map_estimator + if scale_map_estimator is None: + _p("WARN: scale_map_estimator not in model; depth_alignment may " + "no-op and gaussians may not benefit from the external depth.") + + internal_shape = (1536, 1536) + H_grid = W_grid = int(predictor.output_resolution) # typ. 768 + input_shape = [1, 3, internal_shape[0], internal_shape[1]] + memory_required = patcher.memory_required(input_shape) + comfy.model_management.load_models_gpu( + [patcher], memory_required=memory_required, + ) + + # Output path(s) — same convention as SharpPredict. + os.makedirs(OUTPUT_DIR, exist_ok=True) + timestamp = int(time.time() * 1000) + if B == 1: + output_path = os.path.join(OUTPUT_DIR, f"{output_prefix}_{timestamp}.ply") + output_folder = None + is_batch = False + else: + output_folder = os.path.join(OUTPUT_DIR, f"{output_prefix}_{timestamp}") + os.makedirs(output_folder, exist_ok=True) + output_path = output_folder + is_batch = True + + all_ply_paths = [] + all_extrinsics = [] + all_intrinsics = [] + + pbar = comfy.utils.ProgressBar(B) + t_start = time.time() + n_gaussians_total = 0 + + for b in range(B): + comfy.model_management.throw_exception_if_processing_interrupted() + + img_np = image[b].cpu().numpy() if isinstance(image, torch.Tensor) else np.asarray(image[b]) + if img_np.dtype != np.uint8: + img_np = (np.clip(img_np, 0, 1) * 255 + 0.5).astype(np.uint8) + height, width = img_np.shape[:2] + image_hash = _compute_image_hash(img_np) + + # Encode (shared cache). + if _encode_cache["image_hash"] == image_hash: + monodepth_output = _monodepth_to(_encode_cache["monodepth_output"], device) + image_resized_pt = _encode_cache["image_resized"].to(device) + else: + _encode_cache["image_hash"] = None + image_pt = ( + torch.from_numpy(img_np.copy()).float().to(device).permute(2, 0, 1) / 255.0 + ) + image_resized_pt = F.interpolate( + image_pt[None], + size=(internal_shape[1], internal_shape[0]), + mode="bilinear", align_corners=True, + ) + monodepth_output, _ = predictor.encode(image_resized_pt) + _encode_cache["image_hash"] = image_hash + _encode_cache["monodepth_output"] = _monodepth_to(monodepth_output, "cpu") + _encode_cache["image_resized"] = image_resized_pt.cpu() + _encode_cache["original_shape"] = (height, width) + comfy.model_management.soft_empty_cache() + + # Focal-length resolution. intrinsics is pixel-K for `width` (the + # face image resolution). disparity_factor matches SharpPredict. + if intrinsics is not None: + intr_b = intrinsics[b] if intrinsics.dim() == 3 else intrinsics + f_px = float(intr_b[0, 0]) * (internal_shape[0] / width) + else: + f_px = (width / 36.0) * max(0.1, float(focal_length_mm or 30.0)) + disparity_factor = torch.tensor([f_px / width]).float().to(device) + + # External depth → 1536² (alignment target). + md_b = metric_depth[b] + if md_b.dim() == 3: + md_b = md_b[..., 0] + md_b = md_b.to(device).float().unsqueeze(0).unsqueeze(0) + external_depth = F.interpolate( + md_b, size=internal_shape, mode="bilinear", align_corners=True, + ) + + # Decode WITH depth alignment. + gaussians_ndc = predictor.decode( + monodepth_output, image_resized_pt, disparity_factor, + depth=external_depth, + ) + + # Build extrinsics + intrinsics_resized for unprojection. + if extrinsics is not None: + ext_b = extrinsics[b] if extrinsics.dim() == 3 else extrinsics + unproj_extrinsics = ext_b.to(device).float() + else: + unproj_extrinsics = torch.eye(4, device=device) + + if intrinsics is not None: + intr_b = intrinsics[b] if intrinsics.dim() == 3 else intrinsics + K = intr_b.to(device).float().clone() + # Promote to 4x4 if it came as 3x3 (unproject_gaussians wants 4x4). + if K.shape == (3, 3): + K4 = torch.eye(4, device=device, dtype=K.dtype) + K4[:3, :3] = K + K = K4 + else: + K = torch.tensor( + [ + [f_px, 0, width / 2, 0], + [0, f_px, height / 2, 0], + [0, 0, 1, 0], + [0, 0, 0, 1], + ], + dtype=torch.float32, device=device, + ) + intrinsics_resized = K.clone() + intrinsics_resized[0] *= internal_shape[0] / width + intrinsics_resized[1] *= internal_shape[1] / height + + gaussians = unproject_gaussians( + gaussians_ndc, unproj_extrinsics, intrinsics_resized, internal_shape, + ) + + # Optionally drop layer 1 (back/occluded surface). + # The gaussian_composer's flatten order is (layer, h, w), so the + # first H_grid*W_grid entries are layer 0, the second are layer 1. + if not save_background_layer: + n_layer0 = H_grid * W_grid + gaussians = gaussians._replace( + mean_vectors=gaussians.mean_vectors[:, :n_layer0], + singular_values=gaussians.singular_values[:, :n_layer0], + quaternions=gaussians.quaternions[:, :n_layer0], + colors=gaussians.colors[:, :n_layer0], + opacities=gaussians.opacities[:, :n_layer0], + ) + + # Save PLY. + if is_batch: + ply_path = os.path.join(output_folder, f"{b+1:03d}.ply") + else: + ply_path = output_path + _, metadata = save_ply(gaussians, f_px, (height, width), Path(ply_path)) + + all_ply_paths.append(ply_path) + all_extrinsics.append(metadata["extrinsic"]) + all_intrinsics.append(metadata["intrinsic"]) + n_gaussians_total += int(metadata["num_gaussians"]) + pbar.update(1) + + elapsed = time.time() - t_start + loc = output_folder if is_batch else output_path + layer_str = "layer0 only" if not save_background_layer else "both layers" + _p( + f"{B} face(s) → {n_gaussians_total/1e6:.2f}M gaussians total " + f"({layer_str}, depth-aligned); saved to {loc}; {elapsed:.1f}s" + ) + + # ply_folder is always the directory holding the PLYs (regardless + # of single/batch), so downstream MergeGaussians can wire to it + # without caring about batch size. + ply_folder_out = output_folder if is_batch else os.path.dirname(output_path) + + return io.NodeOutput( + output_path, ply_folder_out, all_extrinsics[0], all_intrinsics[0], + ) + + +NODE_CLASS_MAPPINGS = { + "SharpPredictGaussiansFromMetricDepth": SharpPredictGaussiansFromMetricDepth, +} +NODE_DISPLAY_NAME_MAPPINGS = { + "SharpPredictGaussiansFromMetricDepth": "SHARP Predict Gaussians (Image + Metric Depth)", +} diff --git a/nodes/predict_metric_depth.py b/nodes/predict_metric_depth.py new file mode 100644 index 0000000..3739e7b --- /dev/null +++ b/nodes/predict_metric_depth.py @@ -0,0 +1,203 @@ +"""SharpPredictMetricDepth — image → disparity-head metric depth only. + +Tap SHARP's encoder + disparity head; skip the gaussian decoder entirely. +Fastest path to a clean per-pixel depth map at SHARP's native 1536² +resolution, paired with extrinsics_mdepth (pass-through) and +intrinsics_mdepth (pixel-K rescaled to 1536). The triplet wires straight +into SharpDepthMerge with no K/depth-shape mismatch. + +Why separate from SharpPredictGaussianAttrs: + - ~2× faster per face (no init_model + feature_model + prediction_head + forward passes). + - Lower VRAM (skip decoder allocs). + - Cleaner workflows where you only want depth (mesh build, navmesh, + downstream depth-aligned gaussian prediction, etc.). + +Shares `_encode_cache` with SharpPredictGaussianAttrs so encoding the +same image across both nodes is a single forward pass. +""" + +from __future__ import annotations + +import sys +import time + +import numpy as np +import torch +import torch.nn.functional as F +from comfy_api.latest import io + +from .predict_gaussian_attrs import ( + _compute_image_hash, _monodepth_to, _encode_cache, +) + + +def _p(msg: str) -> None: + print(f"[SharpPredictMetricDepth] {msg}", file=sys.stderr, flush=True) + + +def _rescale_K(intr_in, target_w, target_h, src_w, src_h): + sx = float(target_w) / float(src_w) + sy = float(target_h) / float(src_h) + k = intr_in.detach().clone().float() if isinstance(intr_in, torch.Tensor) \ + else torch.as_tensor(np.asarray(intr_in)).float().clone() + if k.dim() == 2: + k[0, :] *= sx + k[1, :] *= sy + k_dbg = k + elif k.dim() == 3: + k[:, 0, :] *= sx + k[:, 1, :] *= sy + k_dbg = k[0] + else: + raise ValueError(f"intrinsics must be (3,3) or (N,3,3), got {tuple(k.shape)}") + return k, float(k_dbg[0, 0]) + + +class SharpPredictMetricDepth(io.ComfyNode): + """SHARP encoder + disparity head only → per-pixel metric depth at 1536².""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SharpPredictMetricDepth", + display_name="SHARP Predict Metric Depth", + category="SHARP", + description=( + "Run only SHARP's encoder + disparity head (skip the gaussian " + "decoder). Outputs per-pixel metric depth at native 1536² + " + "paired extrinsics/intrinsics. ~2× faster than " + "SharpPredictGaussianAttrs when you only want depth. Wire " + "into SharpDepthMerge for clean equirect depth merge." + ), + inputs=[ + io.Custom("SHARP_MODEL_CONFIG").Input("model"), + io.Image.Input("image"), + io.Float.Input( + "focal_length_mm", default=30.0, min=0.0, max=500.0, + step=0.1, optional=True, + tooltip="Focal length in mm (35mm equiv). Ignored if " + "intrinsics provided."), + io.Custom("EXTRINSICS").Input( + "extrinsics", optional=True, + tooltip="Pass-through to extrinsics_mdepth output."), + io.Custom("INTRINSICS").Input( + "intrinsics", optional=True, + tooltip="If provided, overrides focal_length_mm. " + "Re-emitted on intrinsics_mdepth rescaled to 1536²."), + ], + outputs=[ + io.Image.Output( + display_name="metric_depth", + tooltip="[B, 1536, 1536, 3] disparity-head metric depth " + "at SHARP's native resolution."), + io.Custom("EXTRINSICS").Output( + display_name="extrinsics_mdepth", + tooltip="Pass-through of input extrinsics (resolution-" + "independent)."), + io.Custom("INTRINSICS").Output( + display_name="intrinsics_mdepth", + tooltip="Intrinsics rescaled to the 1536² metric_depth grid."), + ], + ) + + @classmethod + @torch.no_grad() + def execute( + cls, + model, + image: torch.Tensor, + focal_length_mm: float = 30.0, + extrinsics: torch.Tensor | None = None, + intrinsics: torch.Tensor | None = None, + ): + global _encode_cache + import comfy.model_management + from .load_model import _load_sharp_model + + patcher = _load_sharp_model(model) + predictor = patcher.model + device = patcher.load_device + + if image.dim() == 3: + image = image.unsqueeze(0) + B = image.shape[0] + t_start = time.time() + + internal_shape = (1536, 1536) + input_shape = [1, 3, internal_shape[0], internal_shape[1]] + memory_required = patcher.memory_required(input_shape) + comfy.model_management.load_models_gpu( + [patcher], memory_required=memory_required, + ) + + metric_depths = [] + last_width = last_height = None + for b in range(B): + img_np = image[b].cpu().numpy() if isinstance(image, torch.Tensor) else np.asarray(image[b]) + if img_np.dtype != np.uint8: + img_np = (np.clip(img_np, 0, 1) * 255 + 0.5).astype(np.uint8) + height, width = img_np.shape[:2] + last_width, last_height = width, height + image_hash = _compute_image_hash(img_np) + + if _encode_cache["image_hash"] == image_hash: + monodepth_output = _monodepth_to(_encode_cache["monodepth_output"], device) + else: + _encode_cache["image_hash"] = None + image_pt = ( + torch.from_numpy(img_np.copy()).float().to(device).permute(2, 0, 1) / 255.0 + ) + image_resized_pt = F.interpolate( + image_pt[None], + size=(internal_shape[1], internal_shape[0]), + mode="bilinear", align_corners=True, + ) + monodepth_output, _ = predictor.encode(image_resized_pt) + _encode_cache["image_hash"] = image_hash + _encode_cache["monodepth_output"] = _monodepth_to(monodepth_output, "cpu") + _encode_cache["image_resized"] = image_resized_pt.cpu() + _encode_cache["original_shape"] = (height, width) + comfy.model_management.soft_empty_cache() + + if intrinsics is not None: + intr_b = intrinsics[b] if intrinsics.dim() == 3 else intrinsics + f_px = float(intr_b[0, 0]) * (internal_shape[0] / width) + else: + f_px = (width / 36.0) * max(0.1, float(focal_length_mm or 30.0)) + + disparity_factor_scalar = f_px / width + monodepth_disparity = monodepth_output.disparity # [1, 1, 1536, 1536] + metric_depth_full = ( + disparity_factor_scalar / monodepth_disparity.clamp(min=1e-4) + ) # [1, 1, 1536, 1536] + metric_depths.append(metric_depth_full[0, 0].cpu()) # [1536, 1536] + + metric_batch = torch.stack(metric_depths, dim=0) # [B, 1536, 1536] + metric_img = metric_batch.unsqueeze(-1).expand(-1, -1, -1, 3).contiguous() + + if intrinsics is not None: + intrinsics_mdepth_out, k_fx_md_dbg = _rescale_K( + intrinsics, internal_shape[1], internal_shape[0], + last_width, last_height, + ) + else: + intrinsics_mdepth_out = None + k_fx_md_dbg = None + extrinsics_mdepth_out = extrinsics + + m_med = float(metric_batch.median()) + elapsed = time.time() - t_start + k_str = f", K_{internal_shape[0]} fx→{k_fx_md_dbg:.1f}" if k_fx_md_dbg is not None else "" + _p( + f"{B} face(s) @ {internal_shape[0]}²; " + f"metric depth median={m_med:.2f}m{k_str}; {elapsed:.1f}s" + ) + + return io.NodeOutput( + metric_img, extrinsics_mdepth_out, intrinsics_mdepth_out, + ) + + +NODE_CLASS_MAPPINGS = {"SharpPredictMetricDepth": SharpPredictMetricDepth} +NODE_DISPLAY_NAME_MAPPINGS = {"SharpPredictMetricDepth": "SHARP Predict Metric Depth"} diff --git a/workflows/pano.json b/workflows/pano.json index a073a57..554e0eb 100644 --- a/workflows/pano.json +++ b/workflows/pano.json @@ -1 +1 @@ -{"id":"sharp-basic-workflow","revision":0,"last_node_id":34,"last_link_id":83,"nodes":[{"id":9,"type":"LoadSharpModel","pos":[-333.4312128826269,-989.2139440177309],"size":[270,82],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"precision","name":"precision","shape":7,"type":"COMBO","widget":{"name":"precision"},"link":null}],"outputs":[{"localized_name":"model_config","name":"model_config","type":"SHARP_MODEL_CONFIG","links":[59]}],"properties":{"cnr_id":"comfyui-sharp","ver":"51939f0d3bdcd9f4965978445ad0fa31a6222ef0","Node name for S&R":"LoadSharpModel"},"widgets_values":["auto"]},{"id":25,"type":"PreviewImage","pos":[740.1385628634665,-806.6059769599477],"size":[636,398],"flags":{},"order":6,"mode":0,"inputs":[{"localized_name":"images","name":"images","type":"IMAGE","link":66}],"outputs":[],"properties":{"cnr_id":"comfy-core","ver":"0.12.2","Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":31,"type":"GetImageSize","pos":[-171.20477315771052,-496.670417063211],"size":[140,124],"flags":{},"order":3,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"IMAGE","link":82}],"outputs":[{"localized_name":"width","name":"width","type":"INT","links":[69]},{"localized_name":"height","name":"height","type":"INT","links":[70]},{"localized_name":"batch_size","name":"batch_size","type":"INT","links":null}],"properties":{"cnr_id":"comfy-core","ver":"0.22.0","Node name for S&R":"GetImageSize"},"widgets_values":[]},{"id":28,"type":"SharpPredictGaussianAttrs","pos":[-11.318504206652932,-796.1153966433557],"size":[364.5328125,158],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"model","name":"model","type":"SHARP_MODEL_CONFIG","link":59},{"localized_name":"image","name":"image","type":"IMAGE","link":58},{"localized_name":"extrinsics","name":"extrinsics","shape":7,"type":"EXTRINSICS","link":73},{"localized_name":"intrinsics","name":"intrinsics","shape":7,"type":"INTRINSICS","link":74},{"localized_name":"focal_length_mm","name":"focal_length_mm","shape":7,"type":"FLOAT","widget":{"name":"focal_length_mm"},"link":null}],"outputs":[{"localized_name":"first_layer_depth_refined","name":"first_layer_depth_refined","type":"IMAGE","links":[65]},{"localized_name":"first_layer_gaussian_attrs","name":"first_layer_gaussian_attrs","type":"MULTIBAND_IMAGE","links":null},{"localized_name":"second_layer_depth_refined","name":"second_layer_depth_refined","type":"IMAGE","links":null},{"localized_name":"second_layer_gaussian_attrs","name":"second_layer_gaussian_attrs","type":"MULTIBAND_IMAGE","links":null},{"localized_name":"extrinsics","name":"extrinsics","type":"EXTRINSICS","links":[]},{"localized_name":"intrinsics","name":"intrinsics","type":"INTRINSICS","links":[]}],"properties":{"cnr_id":"comfyui-sharp","ver":"344051f3e2eb0527d83ba2c2c0746d46bbf8565b","Node name for S&R":"SharpPredictGaussianAttrs"},"widgets_values":[30]},{"id":29,"type":"SharpDepthMerge","pos":[396.9468475836463,-804.7254960264037],"size":[270,190],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"face_depths","name":"face_depths","type":"IMAGE","link":65},{"localized_name":"extrinsics","name":"extrinsics","type":"EXTRINSICS","link":75},{"localized_name":"intrinsics","name":"intrinsics","type":"INTRINSICS","link":76},{"localized_name":"face_valid_masks","name":"face_valid_masks","shape":7,"type":"MASK","link":null},{"localized_name":"out_width","name":"out_width","type":"INT","widget":{"name":"out_width"},"link":69},{"localized_name":"out_height","name":"out_height","type":"INT","widget":{"name":"out_height"},"link":70},{"localized_name":"use_gpu","name":"use_gpu","type":"BOOLEAN","widget":{"name":"use_gpu"},"link":null},{"localized_name":"strategy","name":"strategy","type":"COMBO","widget":{"name":"strategy"},"link":null}],"outputs":[{"localized_name":"depth","name":"depth","type":"IMAGE","links":[66,78]},{"localized_name":"valid_mask","name":"valid_mask","type":"MASK","links":null}],"properties":{"cnr_id":"comfyui-sharp","ver":"344051f3e2eb0527d83ba2c2c0746d46bbf8565b","Node name for S&R":"SharpDepthMerge"},"widgets_values":[1920,960,true,"auto"]},{"id":32,"type":"SharpBuildMesh","pos":[855.9169305893066,-1094.7384144194202],"size":[287.853515625,166],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"panorama","name":"panorama","type":"IMAGE","link":83},{"localized_name":"depth","name":"depth","type":"IMAGE","link":78},{"localized_name":"sky_mask","name":"sky_mask","shape":7,"type":"MASK","link":null},{"localized_name":"valid_mask","name":"valid_mask","shape":7,"type":"MASK","link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null},{"localized_name":"scene_cap_depth","name":"scene_cap_depth","shape":7,"type":"FLOAT","widget":{"name":"scene_cap_depth"},"link":null}],"outputs":[{"localized_name":"mesh","name":"mesh","type":"TRIMESH","links":[79]},{"localized_name":"mesh_preview","name":"mesh_preview","type":"IMAGE","links":null},{"localized_name":"global_median_depth","name":"global_median_depth","type":"FLOAT","links":null}],"properties":{"cnr_id":"comfyui-sharp","ver":"344051f3e2eb0527d83ba2c2c0746d46bbf8565b","Node name for S&R":"SharpBuildMesh"},"widgets_values":[1024,"randomize",8]},{"id":33,"type":"GeomPackRemesh","pos":[1198.7700133013057,-1101.11603401942],"size":[358.068359375,158],"flags":{},"order":8,"mode":0,"inputs":[{"localized_name":"trimesh","name":"trimesh","type":"TRIMESH","link":79},{"localized_name":"backend","name":"backend","type":"COMFY_DYNAMICCOMBO_V3","widget":{"name":"backend"},"link":null},{"localized_name":"backend.gpu_target_face_count","name":"backend.gpu_target_face_count","type":"INT","widget":{"name":"backend.gpu_target_face_count"},"link":null},{"localized_name":"backend.remesh_band","name":"backend.remesh_band","type":"FLOAT","widget":{"name":"backend.remesh_band"},"link":null}],"outputs":[{"localized_name":"remeshed_mesh","name":"remeshed_mesh","type":"TRIMESH","links":[80]},{"localized_name":"info","name":"info","type":"STRING","links":null}],"properties":{"cnr_id":"comfyui-geometrypack","ver":"c67199de05705642258e727fa118f412877b4ebf","Node name for S&R":"GeomPackRemesh"},"widgets_values":["gpu_cumesh",500000,1,"false"]},{"id":34,"type":"GeomPackPreviewMeshVTK","pos":[1631.2914299017073,-1122.95447277733],"size":[512,714],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"trimesh","name":"trimesh","shape":7,"type":"TRIMESH","link":80},{"localized_name":"voxelgrid","name":"voxelgrid","shape":7,"type":"VOXELGRID","link":null},{"localized_name":"mode","name":"mode","type":"COMBO","widget":{"name":"mode"},"link":null}],"outputs":[],"properties":{"cnr_id":"comfyui-geometrypack","ver":"c67199de05705642258e727fa118f412877b4ebf","Node name for S&R":"GeomPackPreviewMeshVTK"},"widgets_values":["fields","{\"show_edges\":false,\"camera_state\":\"\",\"selected_field\":\"\",\"selected_channel\":\"magnitude\",\"selected_colormap\":\"erdc_rainbow_bright\"}"]},{"id":26,"type":"SharpPanoramaIcosahedronSplit","pos":[-387.32488905316114,-797.8930522427164],"size":[328.6578125,186],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"panorama","name":"panorama","type":"IMAGE","link":81},{"localized_name":"resolution","name":"resolution","type":"INT","widget":{"name":"resolution"},"link":null},{"localized_name":"subdivision","name":"subdivision","type":"COMBO","widget":{"name":"subdivision"},"link":null},{"localized_name":"use_gpu","name":"use_gpu","shape":7,"type":"BOOLEAN","widget":{"name":"use_gpu"},"link":null}],"outputs":[{"localized_name":"face_images","name":"face_images","type":"IMAGE","links":[58]},{"localized_name":"extrinsics","name":"extrinsics","type":"EXTRINSICS","links":[73,75]},{"localized_name":"intrinsics","name":"intrinsics","type":"INTRINSICS","links":[74,76]},{"localized_name":"fov_x_deg","name":"fov_x_deg","type":"FLOAT","links":null},{"localized_name":"debug_image","name":"debug_image","type":"IMAGE","links":null}],"properties":{"cnr_id":"comfyui-sharp","ver":"344051f3e2eb0527d83ba2c2c0746d46bbf8565b","Node name for S&R":"SharpPanoramaIcosahedronSplit"},"widgets_values":[1536,"icosahedron_42",true]},{"id":7,"type":"LoadImage","pos":[-1163.7622907189734,-982.6675026789543],"size":[662.2282800153919,367.6329862374266],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[81,82,83]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"cnr_id":"comfy-core","ver":"0.12.2","Node name for S&R":"LoadImage"},"widgets_values":["pano.png","image"]}],"links":[[58,26,0,28,1,"IMAGE"],[59,9,0,28,0,"SHARP_MODEL_CONFIG"],[65,28,0,29,0,"IMAGE"],[66,29,0,25,0,"IMAGE"],[69,31,0,29,4,"INT"],[70,31,1,29,5,"INT"],[73,26,1,28,2,"EXTRINSICS"],[74,26,2,28,3,"INTRINSICS"],[75,26,1,29,1,"EXTRINSICS"],[76,26,2,29,2,"INTRINSICS"],[78,29,0,32,1,"IMAGE"],[79,32,0,33,0,"TRIMESH"],[80,33,0,34,0,"TRIMESH"],[81,7,0,26,0,"IMAGE"],[82,7,0,31,0,"IMAGE"],[83,7,0,32,0,"IMAGE"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.5644739300537777,"offset":[1080.1378849470495,1307.8547200141966]},"workflowRendererVersion":"LG","frontendVersion":"1.34.9"},"version":0.4} \ No newline at end of file +{"id":"sharp-basic-workflow","revision":0,"last_node_id":54,"last_link_id":135,"nodes":[{"id":7,"type":"LoadImage","pos":[-1163.7622907189734,-982.6675026789543],"size":[662.2282800153919,367.6329862374266],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[81,134]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"cnr_id":"comfy-core","ver":"0.12.2","Node name for S&R":"LoadImage"},"widgets_values":["pano.png","image"]},{"id":47,"type":"SharpPredictMetricDepth","pos":[-108.2206905670124,-932.7836063312919],"size":[287.273046875,118],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"model","name":"model","type":"SHARP_MODEL_CONFIG","link":114},{"localized_name":"image","name":"image","type":"IMAGE","link":115},{"localized_name":"extrinsics","name":"extrinsics","shape":7,"type":"EXTRINSICS","link":116},{"localized_name":"intrinsics","name":"intrinsics","shape":7,"type":"INTRINSICS","link":117},{"localized_name":"focal_length_mm","name":"focal_length_mm","shape":7,"type":"FLOAT","widget":{"name":"focal_length_mm"},"link":null}],"outputs":[{"localized_name":"metric_depth","name":"metric_depth","type":"IMAGE","links":[118]},{"localized_name":"extrinsics_mdepth","name":"extrinsics_mdepth","type":"EXTRINSICS","links":[119]},{"localized_name":"intrinsics_mdepth","name":"intrinsics_mdepth","type":"INTRINSICS","links":[120]}],"properties":{"cnr_id":"comfyui-sharp","ver":"9ae300958498a99946acd046599f14a13b2e9175","Node name for S&R":"SharpPredictMetricDepth"},"widgets_values":[30]},{"id":26,"type":"SharpPanoramaIcosahedronSplit","pos":[-477.5134982020259,-917.0707430938513],"size":[328.6578125,186],"flags":{},"order":3,"mode":0,"inputs":[{"localized_name":"panorama","name":"panorama","type":"IMAGE","link":81},{"localized_name":"resolution","name":"resolution","type":"INT","widget":{"name":"resolution"},"link":null},{"localized_name":"subdivision","name":"subdivision","type":"COMBO","widget":{"name":"subdivision"},"link":null},{"localized_name":"use_gpu","name":"use_gpu","shape":7,"type":"BOOLEAN","widget":{"name":"use_gpu"},"link":null}],"outputs":[{"localized_name":"face_images","name":"face_images","type":"IMAGE","links":[115]},{"localized_name":"extrinsics","name":"extrinsics","type":"EXTRINSICS","links":[116]},{"localized_name":"intrinsics","name":"intrinsics","type":"INTRINSICS","links":[117]},{"localized_name":"fov_x_deg","name":"fov_x_deg","type":"FLOAT","links":[]},{"localized_name":"debug_image","name":"debug_image","type":"IMAGE","links":[]}],"properties":{"cnr_id":"comfyui-sharp","ver":"344051f3e2eb0527d83ba2c2c0746d46bbf8565b","Node name for S&R":"SharpPanoramaIcosahedronSplit","aux_id":"PozzettiAndrea/ComfyUI-Sharp"},"widgets_values":[1536,"icosahedron_42",true]},{"id":9,"type":"LoadSharpModel","pos":[-777.9320220314913,-1113.7600179184876],"size":[270,82],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"precision","name":"precision","shape":7,"type":"COMBO","widget":{"name":"precision"},"link":null}],"outputs":[{"localized_name":"model_config","name":"model_config","type":"SHARP_MODEL_CONFIG","links":[114,130]}],"properties":{"cnr_id":"comfyui-sharp","ver":"51939f0d3bdcd9f4965978445ad0fa31a6222ef0","Node name for S&R":"LoadSharpModel","aux_id":"PozzettiAndrea/ComfyUI-Sharp"},"widgets_values":["auto"]},{"id":29,"type":"SharpDepthMerge","pos":[219.994246421537,-931.0619782102407],"size":[270,190],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"face_depths","name":"face_depths","type":"IMAGE","link":118},{"localized_name":"extrinsics","name":"extrinsics","type":"EXTRINSICS","link":119},{"localized_name":"intrinsics","name":"intrinsics","type":"INTRINSICS","link":120},{"localized_name":"face_valid_masks","name":"face_valid_masks","shape":7,"type":"MASK","link":null},{"localized_name":"out_width","name":"out_width","type":"INT","widget":{"name":"out_width"},"link":null},{"localized_name":"out_height","name":"out_height","type":"INT","widget":{"name":"out_height"},"link":null},{"localized_name":"use_gpu","name":"use_gpu","type":"BOOLEAN","widget":{"name":"use_gpu"},"link":null},{"localized_name":"strategy","name":"strategy","type":"COMBO","widget":{"name":"strategy"},"link":null}],"outputs":[{"localized_name":"depth","name":"depth","type":"IMAGE","links":[122]},{"localized_name":"valid_mask","name":"valid_mask","type":"MASK","links":null}],"properties":{"cnr_id":"comfyui-sharp","ver":"344051f3e2eb0527d83ba2c2c0746d46bbf8565b","Node name for S&R":"SharpDepthMerge","aux_id":"PozzettiAndrea/ComfyUI-Sharp"},"widgets_values":[1920,960,true,"auto"]},{"id":54,"type":"MergeGaussians","pos":[1357.0884506944033,-971.4686490745111],"size":[289.2828125,150],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"ply_folder","name":"ply_folder","type":"STRING","widget":{"name":"ply_folder"},"link":135},{"localized_name":"output_prefix","name":"output_prefix","shape":7,"type":"STRING","widget":{"name":"output_prefix"},"link":null},{"localized_name":"max_depth","name":"max_depth","shape":7,"type":"FLOAT","widget":{"name":"max_depth"},"link":null},{"localized_name":"min_opacity","name":"min_opacity","shape":7,"type":"FLOAT","widget":{"name":"min_opacity"},"link":null}],"outputs":[{"localized_name":"ply_path","name":"ply_path","type":"STRING","links":null},{"localized_name":"num_gaussians","name":"num_gaussians","type":"INT","links":null}],"properties":{"cnr_id":"comfyui-sharp","ver":"9ae300958498a99946acd046599f14a13b2e9175","Node name for S&R":"MergeGaussians"},"widgets_values":["","merged",0,0]},{"id":48,"type":"SharpPanoramaIcosahedronSplit","pos":[522.9564535947337,-927.745953268992],"size":[328.6578125,186],"flags":{},"order":6,"mode":0,"inputs":[{"localized_name":"panorama","name":"panorama","type":"IMAGE","link":122},{"localized_name":"resolution","name":"resolution","type":"INT","widget":{"name":"resolution"},"link":null},{"localized_name":"subdivision","name":"subdivision","type":"COMBO","widget":{"name":"subdivision"},"link":null},{"localized_name":"use_gpu","name":"use_gpu","shape":7,"type":"BOOLEAN","widget":{"name":"use_gpu"},"link":null}],"outputs":[{"localized_name":"face_images","name":"face_images","type":"IMAGE","links":[131]},{"localized_name":"extrinsics","name":"extrinsics","type":"EXTRINSICS","links":[132]},{"localized_name":"intrinsics","name":"intrinsics","type":"INTRINSICS","links":[133]},{"localized_name":"fov_x_deg","name":"fov_x_deg","type":"FLOAT","links":[]},{"localized_name":"debug_image","name":"debug_image","type":"IMAGE","links":[]}],"properties":{"cnr_id":"comfyui-sharp","ver":"344051f3e2eb0527d83ba2c2c0746d46bbf8565b","Node name for S&R":"SharpPanoramaIcosahedronSplit","aux_id":"PozzettiAndrea/ComfyUI-Sharp"},"widgets_values":[1536,"icosahedron_42",true]},{"id":53,"type":"SharpPredictGaussiansFromMetricDepth","pos":[881.3445942022132,-963.6372998105013],"size":[427.4712890625,162],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"model","name":"model","type":"SHARP_MODEL_CONFIG","link":130},{"localized_name":"image","name":"image","type":"IMAGE","link":134},{"localized_name":"metric_depth","name":"metric_depth","type":"IMAGE","link":131},{"localized_name":"extrinsics","name":"extrinsics","shape":7,"type":"EXTRINSICS","link":132},{"localized_name":"intrinsics","name":"intrinsics","shape":7,"type":"INTRINSICS","link":133},{"localized_name":"focal_length_mm","name":"focal_length_mm","shape":7,"type":"FLOAT","widget":{"name":"focal_length_mm"},"link":null},{"localized_name":"output_prefix","name":"output_prefix","shape":7,"type":"STRING","widget":{"name":"output_prefix"},"link":null}],"outputs":[{"localized_name":"ply_path","name":"ply_path","type":"STRING","links":[135]},{"localized_name":"extrinsics","name":"extrinsics","type":"EXTRINSICS","links":null},{"localized_name":"intrinsics","name":"intrinsics","type":"INTRINSICS","links":null}],"properties":{"cnr_id":"comfyui-sharp","ver":"9ae300958498a99946acd046599f14a13b2e9175","Node name for S&R":"SharpPredictGaussiansFromMetricDepth"},"widgets_values":[30,"sharp_aligned"]}],"links":[[81,7,0,26,0,"IMAGE"],[114,9,0,47,0,"SHARP_MODEL_CONFIG"],[115,26,0,47,1,"IMAGE"],[116,26,1,47,2,"EXTRINSICS"],[117,26,2,47,3,"INTRINSICS"],[118,47,0,29,0,"IMAGE"],[119,47,1,29,1,"EXTRINSICS"],[120,47,2,29,2,"INTRINSICS"],[122,29,0,48,0,"IMAGE"],[130,9,0,53,0,"SHARP_MODEL_CONFIG"],[131,48,0,53,2,"IMAGE"],[132,48,1,53,3,"EXTRINSICS"],[133,48,2,53,4,"INTRINSICS"],[134,7,0,53,1,"IMAGE"],[135,53,0,54,0,"STRING"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.6830134553650713,"offset":[-45.43829346936729,1250.3750455283682]},"workflowRendererVersion":"LG","frontendVersion":"1.34.9"},"version":0.4} \ No newline at end of file