diff --git a/README.md b/README.md index 2047454..c37646d 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,82 @@
-# Interior Design for Comfyui +# 🛋️ Interior Design for ComfyUI v2.0
-This node adapts [StableDesign: 2nd place solution for the Generative Interior Design 2024 competition](https://huggingface.co/spaces/MykolaL/StableDesign) to use directly on ComfyUI. +This project adapts the original pipeline from +[StableDesign: 2nd place solution for the Generative Interior Design 2024 competition](https://huggingface.co/spaces/MykolaL/StableDesign) +to work **natively and fully inside ComfyUI**, leveraging modern custom nodes and a modular architecture. -More about this pipeline works, see this post on [Medium](https://medium.com/@melgor89/generative-interior-design-challenge-2024-2nd-place-solution-6338f19f6fe3) -# -The pipeline automatically downloads trained models of `controlnet_depth` and `own_controlnet` to the local script of ComfyUI runs in folder `./custom_models`. Other models are downloaded to the `.cache` folder of Hugging Face. +Learn more about the original method in this [Medium article](https://medium.com/@melgor89/generative-interior-design-challenge-2024-2nd-place-solution-6338f19f6fe3). -This node is available on [ComfyUI Manager](https://github.com/ltdrdata/ComfyUI-Manager) -![image](https://github.com/user-attachments/assets/44e3520c-1b42-41a2-8996-47e5fb37d9a1) +--- + +## ✅ Key Features + +- **Fully restructured for ComfyUI compatibility** +- Includes 3 custom nodes developed specifically for this repository: + - `"Interior Design Segmentator"` + - `"Image Normalize"` + - `"Control Items"` +- Uses additional advanced custom nodes: + - [Depth Anything V2](https://github.com/DepthAnything/Depth-Anything-V2) + - [ComfyUI_IPAdapter_plus](https://github.com/cubiq/ComfyUI_IPAdapter_plus) +- Reimplements only the **necessary components** from the original pipeline, such as: + - **Semantic segmenter** using [`openmmlab/upernet-convnext-small`](https://huggingface.co/openmmlab/upernet-convnext-small) +- Supports standard ComfyUI workflows with **ControlNet and IP Adapter integration** +- Compatible with other workflows and custom models +- Inference is significantly faster than the original pipeline (test the workflow and compare). + +--- + +## 🧱 Pipeline Overview + +- The idea was to use as many nodes from the ComfyUI core as possible. However, for certain tasks, I had to rely on custom nodes or adapt my own nodes to stay as faithful as possible to the original pipeline, or alternatively, use dependencies from other custom node repositories. +- Uses the **same base models** as the original StableDesign pipeline +- Includes **custom ControlNet models**: + - `controlnet_depth` + - `own_controlnet` (segmentation) +- These ControlNets, along with the depth map and segmentation map, are passed into the **UNet** of the diffusion model +- Final step includes **refinement with latent noise injection and up/down-scaling**(because of the size limitation of the original pipeline) + +--- + +## 📁 Example Workflow + +An example workflow is included in this repository: +[`stable-design-for-comfyui.json`](./workflow/stable-design-for-comfyui.json) + +You can load it directly into ComfyUI to test or extend the pipeline. + +![Preview](./assets/image.png) + +--- + +## 📦 Installation & Dependencies + +This node is available on [ComfyUI Manager](https://github.com/ltdrdata/ComfyUI-Manager) +You must also install the following custom nodes manually if not yet available: + +- [Depth Anything V2](https://github.com/DepthAnything/Depth-Anything-V2) +- [ComfyUI_IPAdapter_plus](https://github.com/cubiq/ComfyUI_IPAdapter_plus) *(note: unmaintained)* +- [ComfyMath](https://github.com/evanspearman/ComfyMath) + +--- + +## 🔧 Upcoming Features + +- **Expansion of Control Items**: Add support for more semantic regions and control inputs to increase flexibility and detail in generation. +- **Evaluation of CLIP Conditioning**: Reassess the combination of positive and negative CLIP embeddings in ControlNet to verify if results match the original pipeline's behavior. +- **Custom IP Adapter Node**: Implement a native IP Adapter node to replace the current dependency on `ComfyUI_IPAdapter_plus`, which is currently unmaintained. + +--- + +## 🧪 License & Acknowledgements + +Original pipeline by MykolaL ([StableDesign on Hugging Face](https://huggingface.co/spaces/MykolaL/StableDesign)) +This adaptation reuses only essential logic, converted to full ComfyUI-compatible workflow format. + +--- + +feel free to open issues and PRs diff --git a/__init__.py b/__init__.py index 1d50455..7013287 100644 --- a/__init__.py +++ b/__init__.py @@ -1,13 +1,19 @@ from .node.node import InteriorDesignNode +from .node.node import ImageNormalize +from .node.node import ControlItems -# Registra o nó no ComfyUI NODE_CLASS_MAPPINGS = { - "interior-design-for-comfyui": InteriorDesignNode, + "Interior Design Segmentator": InteriorDesignNode, + "Image Normalize": ImageNormalize, + "Control Items": ControlItems, + } NODE_DISPLAY_NAME_MAPPINGS = { - "interior-design-for-comfyui": "Interior Design for ComfyUI", + "Interior Design Segmentator": "Interior Design Segmentator", + "Image Normalize": "Image Resize Normalizer", + "Control Items": "Control Items Selector", } diff --git a/assets/image.png b/assets/image.png new file mode 100644 index 0000000..228bcf4 Binary files /dev/null and b/assets/image.png differ diff --git a/node/node.py b/node/node.py index 4d9ef0d..01abd3a 100644 --- a/node/node.py +++ b/node/node.py @@ -1,84 +1,137 @@ +from typing import Literal import torch -import torch.nn.functional as F from comfy.model_management import get_torch_device -from .pipeline import ControlNetDepthDesignModelMulti # Seu pipeline original -from torchvision import transforms -from torchvision.transforms import ToPILImage +from .pipeline import SimplifiedSegmenter +import torch.nn.functional as F class InteriorDesignNode: def __init__(self): self.device = get_torch_device() - self.dtype = torch.float16 if self.device == "cuda" else torch.float32 - self.pipeline = ControlNetDepthDesignModelMulti() # Seu pipeline original + self.pipeline = SimplifiedSegmenter( + device=self.device + ) @classmethod def INPUT_TYPES(cls): return { "required": { "image": ("IMAGE",), - "positive": ("STRING",{"forceInput": True}), # Prompt de descrição do design - "guidance_scale": ("FLOAT", {"default": 10, "min": 1.0, "max": 20.0, "step": 0.1}), - "num_steps": ("INT", {"default": 50, "min": 1, "max": 100}), - "strength": ("FLOAT", {"default": 0.9, "min": 0.1, "max": 1.0}), - "seed": ("INT", {"default": 40, "min": 0, "max": 0xffffffffffffffff}), - "img_size": ("INT", {"default": 768, "min": 256, "max": 768, "step": 8}), - "img_dept_threshold": ("FLOAT", {"default": 0.5, "min": 0.1, "max": 0.99, "step": 0.01}), - "img_seg_threshold": ("FLOAT", {"default": 0.5, "min": 0.1, "max": 0.99, "step": 0.01}), - "ip_adapter_scale": ("FLOAT", {"default": 0.40, "min": 0.01, "max": 1.0, "step": 0.01}), - "ip_adapter_model":(["ip-adapter_sd15.bin","ip-adapter-plus_sd15.bin","ip-adapter_sd15_light.bin"], {}) + "control_items": ("CONTROL_ITEMS",), }, - "optional": { - "negative": ("STRING",{"forceInput": True}), # Prompt de descrição do design - } } - RETURN_TYPES = ("IMAGE",) # Retorna a imagem gerada - RETURN_NAMES = ("IMAGE",) # Nome do retorno + RETURN_TYPES = ("IMAGE", "MASK") + RETURN_NAMES = ("IMAGE", "MASK") + FUNCTION = "process" + CATEGORY = "segmentation" - FUNCTION = "process" # Função que será executada - CATEGORY = "interior_design" # Categoria para o ComfyUI + def process(self, image, control_items): + image = image.permute(0, 3, 1, 2).to(self.device) - def process(self, image, positive, guidance_scale, num_steps, strength, seed, img_size, img_dept_threshold, img_seg_threshold, ip_adapter_scale, ip_adapter_model, negative=None): - # Pré-processamento da imagem - self.pipeline.ip_adapter_scale = ip_adapter_scale - self.pipeline.ip_adapter_model = ip_adapter_model - self.pipeline.seed = seed - if negative is not None: - self.pipeline.neg_prompt = negative - image = tensor_to_pil(image) - - # Chama o pipeline para gerar o design de interiores - design_image = self.pipeline.generate_design( - empty_room_image=image, - prompt=positive, - guidance_scale=guidance_scale, - num_steps=num_steps, - strength=strength, - img_size=img_size, - img_dept_threshold=img_dept_threshold, - img_seg_threshold=img_seg_threshold, + seg_cond, mask = self.pipeline(images=image,control_items=control_items) + seg_cond = seg_cond.permute(0, 2, 3, 1) + mask = mask.permute(0, 2, 3, 1) + mask = mask.squeeze(-1) + + #seg_cond = seg_cond.float() / 255.0 (its already normalized in the pipeline) + + return (seg_cond, mask,) + + +class ImageNormalize: + def __init__(self): + self.device = get_torch_device() + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "images": ("IMAGE",), + "target_size": ("INT", { + "default": 512, + "min": 2, + "max": 4096, + "step": 1, + "display": "number", + }), + "multiple": ("INT", { + "default": 8, + "min": 2, + "max": 128, + "step": 1, + "display": "number", + }), + "mode": (["bilinear", "nearest", "bicubic", "trilinear","area","nearest-exact"],), + } + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES = ("IMAGE",) + FUNCTION = "resize_tensor_to_target" + CATEGORY = "image" + + + + + def resize_tensor_to_target( + self, + images: torch.Tensor, + target_size: int, + multiple: int, + mode: Literal[ + "nearest", "linear", "bilinear", "bicubic", "trilinear", "area", "nearest-exact" + ] = "bilinear" + ) -> torch.Tensor: + + if images.dim() == 4 and images.shape[-1] == 3: + images = images.permute(0, 3, 1, 2) + + b, c, h, w = images.shape + + max_dim = max(h, w) + scale = target_size / max_dim if max_dim > target_size else 1.0 + + new_h = int(h * scale) + new_w = int(w * scale) + + new_h = ((new_h + multiple - 1) // multiple) * multiple + new_w = ((new_w + multiple - 1) // multiple) * multiple + + resized = F.interpolate( + images, size=(new_h, new_w), mode=mode, + align_corners=False if mode in ['linear', 'bilinear', 'bicubic', 'trilinear'] else None ) - design_image = pil_to_tensor(design_image, self.device, self.dtype) - return (design_image,) -def tensor_to_pil(image_tensor): - print(f"Tensor: {image_tensor.shape},{image_tensor.dtype}") - if image_tensor.ndimension() == 4: - image_tensor = image_tensor.squeeze(0) # Remove a dimensão do batch - image_tensor = image_tensor.permute(2, 0, 1) # Troca de [altura, largura, canais] -> [canais, altura, largura] - if image_tensor.shape[0] == 4: - image_tensor = image_tensor[:3] - to_pil = ToPILImage() - pil_image = to_pil(image_tensor) - print(f"Imagem PIL: {pil_image.size},{pil_image.mode}") - return pil_image + resized = resized.permute(0, 2, 3, 1) -def pil_to_tensor(image_pil, device, dtype): - print(f"Imagem PIL: {image_pil.size},{image_pil.mode}") - to_tensor = transforms.ToTensor() - tensor = to_tensor(image_pil) - tensor = tensor.permute(1, 2, 0) # Troca de [canais, altura, largura] -> [altura, largura, canais] - tensor = tensor.unsqueeze(0) # Adiciona a dimensão do batch - tensor = tensor.to(device, dtype=dtype) - print(f"Tensor: {tensor.shape},{tensor.dtype}") - return tensor \ No newline at end of file + return (resized,) + + +class ControlItems: + def __init__(self): + self.device = get_torch_device() + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "window": ("BOOLEAN", {"default": True}), + "door": ("BOOLEAN", {"default": True}), + "staircase": ("BOOLEAN", {"default": False}), + }, + } + + RETURN_TYPES = ("CONTROL_ITEMS",) + RETURN_NAMES = ("CONTROL_ITEMS",) + FUNCTION = "process" + CATEGORY = "control" + + def process(self, window: bool, door: bool, staircase: bool): + control_items = [] + if window: + control_items.append("windowpane;window") + if door: + control_items.append("door;double;door") + if staircase: + control_items.append("stairway;staircase") + return (control_items,) diff --git a/node/pipeline.py b/node/pipeline.py index bbb87e5..3aec9c3 100644 --- a/node/pipeline.py +++ b/node/pipeline.py @@ -1,319 +1,126 @@ -from typing import Tuple, Union, List - +from typing import Tuple,List, Union import numpy as np from PIL import Image - -import torch -from diffusers.pipelines.controlnet import StableDiffusionControlNetInpaintPipeline -from diffusers import ControlNetModel, UniPCMultistepScheduler, AutoPipelineForText2Image from transformers import AutoImageProcessor, UperNetForSemanticSegmentation, AutoModelForDepthEstimation from .colors import ade_palette from .utils import map_colors_rgb -from diffusers import StableDiffusionXLPipeline -import gc -import os -from huggingface_hub import hf_hub_download -import folder_paths +import torch +from torch import Tensor +from transformers import AutoImageProcessor, UperNetForSemanticSegmentation +# ------------------- +# Classe principal +# ------------------- -repo_id = "MykolaL/StableDesign" -file_name = "diffusion_pytorch_model.safetensors" -models = ["controlnet_depth", "own_controlnet"] -rootfolder = folder_paths.folder_names_and_paths["controlnet"][0][0] +class SimplifiedSegmenter: + def __init__(self, device: str = "cuda"): + self.device = device -def check_and_download(model_name,rootfolder): - hf_hub_download( - repo_id=repo_id, - filename=file_name, - subfolder= model_name, - local_dir=rootfolder, - repo_type="space" - ) - hf_hub_download( - repo_id=repo_id, - filename="config.json", - subfolder= model_name, - local_dir=rootfolder, - repo_type="space" - ) -for model_name in models: - model_path = os.path.join(rootfolder, model_name) - check_and_download(model_name,rootfolder) - - -device = "cuda" -dtype = torch.float16 - - -def filter_items( - colors_list: Union[List, np.ndarray], - items_list: Union[List, np.ndarray], - items_to_remove: Union[List, np.ndarray] -) -> Tuple[Union[List, np.ndarray], Union[List, np.ndarray]]: - """ - Filters items and their corresponding colors from given lists, excluding - specified items. - Args: - colors_list: A list or numpy array of colors corresponding to items. - items_list: A list or numpy array of items. - items_to_remove: A list or numpy array of items to be removed. - Returns: - A tuple of two lists or numpy arrays: filtered colors and filtered - items. - """ - filtered_colors = [] - filtered_items = [] - for color, item in zip(colors_list, items_list): - if item not in items_to_remove: - filtered_colors.append(color) - filtered_items.append(item) - return filtered_colors, filtered_items - -def get_segmentation_pipeline( -) -> Tuple[AutoImageProcessor, UperNetForSemanticSegmentation]: - """Method to load the segmentation pipeline - Returns: - Tuple[AutoImageProcessor, UperNetForSemanticSegmentation]: segmentation pipeline - """ - image_processor = AutoImageProcessor.from_pretrained( - "openmmlab/upernet-convnext-small" - ) - image_segmentor = UperNetForSemanticSegmentation.from_pretrained( - "openmmlab/upernet-convnext-small" - ) - return image_processor, image_segmentor - - -def segment_image( - image: Image, - image_processor: AutoImageProcessor, - image_segmentor: UperNetForSemanticSegmentation -) -> Image: - """ - Segments an image using a semantic segmentation model. - Args: - image (Image): The input image to be segmented. - image_processor (AutoImageProcessor): The processor to prepare the - image for segmentation. - image_segmentor (UperNetForSemanticSegmentation): The semantic - segmentation model used to identify different segments in the image. - Returns: - Image: The segmented image with each segment colored differently based - on its identified class. - """ - # image_processor, image_segmentor = get_segmentation_pipeline() - pixel_values = image_processor(image, return_tensors="pt").pixel_values - with torch.no_grad(): - outputs = image_segmentor(pixel_values) - - seg = image_processor.post_process_semantic_segmentation( - outputs, target_sizes=[image.size[::-1]])[0] - color_seg = np.zeros((seg.shape[0], seg.shape[1], 3), dtype=np.uint8) - palette = np.array(ade_palette()) - for label, color in enumerate(palette): - color_seg[seg == label, :] = color - color_seg = color_seg.astype(np.uint8) - seg_image = Image.fromarray(color_seg).convert('RGB') - return seg_image - - -def get_depth_pipeline(): - feature_extractor = AutoImageProcessor.from_pretrained("LiheYoung/depth-anything-large-hf", - torch_dtype=dtype) - depth_estimator = AutoModelForDepthEstimation.from_pretrained("LiheYoung/depth-anything-large-hf", - torch_dtype=dtype) - return feature_extractor, depth_estimator - - -def get_depth_image( - image: Image, - feature_extractor: AutoImageProcessor, - depth_estimator: AutoModelForDepthEstimation -) -> Image: - image_to_depth = feature_extractor(images=image, return_tensors="pt").to(device) - with torch.no_grad(): - depth_map = depth_estimator(**image_to_depth).predicted_depth - - width, height = image.size - depth_map = torch.nn.functional.interpolate( - depth_map.unsqueeze(1).float(), - size=(height, width), - mode="bicubic", - align_corners=False, - ) - depth_min = torch.amin(depth_map, dim=[1, 2, 3], keepdim=True) - depth_max = torch.amax(depth_map, dim=[1, 2, 3], keepdim=True) - depth_map = (depth_map - depth_min) / (depth_max - depth_min) - image = torch.cat([depth_map] * 3, dim=1) - - image = image.permute(0, 2, 3, 1).cpu().numpy()[0] - image = Image.fromarray((image * 255.0).clip(0, 255).astype(np.uint8)) - return image - - -def resize_dimensions(dimensions, target_size): - """ - Resize PIL to target size while maintaining aspect ratio - If smaller than target size leave it as is - """ - width, height = dimensions - - # Check if both dimensions are smaller than the target size - if width < target_size and height < target_size: - return dimensions - - # Determine the larger side - if width > height: - # Calculate the aspect ratio - aspect_ratio = height / width - # Resize dimensions - return (target_size, int(target_size * aspect_ratio)) - else: - # Calculate the aspect ratio - aspect_ratio = width / height - # Resize dimensions - return (int(target_size * aspect_ratio), target_size) - - -def flush(): - gc.collect() - torch.cuda.empty_cache() - - -class ControlNetDepthDesignModelMulti: - """ Produces random noise images """ - - def __init__(self): - """ Initialize your model(s) here """ - #os.environ['HF_HUB_OFFLINE'] = "True" - self.ip_adapter_scale = 0.4 - self.ip_adapter_model = "ip-adapter_sd15.bin" - self.seed = 323*111 - self.neg_prompt = "window, door, low resolution, banner, logo, watermark, text, deformed, blurry, out of focus, surreal, ugly, beginner" - self.control_items = ["windowpane;window", "door;double;door"] - self.additional_quality_suffix = "interior design, 4K, high resolution, photorealistic" - - - - self.controlnet_depth = ControlNetModel.from_pretrained( - rootfolder,subfolder="controlnet_depth", - torch_dtype=dtype, - use_safetensors=True + print("🔄 Loading segmentator...") + self.image_processor = AutoImageProcessor.from_pretrained( + "openmmlab/upernet-convnext-small" ) - - self.controlnet_seg = ControlNetModel.from_pretrained( - rootfolder,subfolder="own_controlnet", - torch_dtype=dtype, - use_safetensors=True + self.image_segmentor = UperNetForSemanticSegmentation.from_pretrained( + "openmmlab/upernet-convnext-small" ) + self.image_segmentor.eval() - self.pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained( - "SG161222/Realistic_Vision_V5.1_noVAE", - controlnet=[self.controlnet_depth, self.controlnet_seg], - safety_checker=None, - torch_dtype=dtype, - ) - + def tensor_to_pil(self, tensor: Tensor) -> Image.Image: + if tensor.max() > 1.0: + arr = tensor.cpu().permute(1, 2, 0).numpy().astype(np.uint8) + else: + arr = (tensor.cpu().permute(1, 2, 0).numpy() * 255).astype(np.uint8) + return Image.fromarray(arr) - self.pipe.load_ip_adapter("h94/IP-Adapter", subfolder="models", - weight_name=self.ip_adapter_model,) - self.pipe.set_ip_adapter_scale(self.ip_adapter_scale) - self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config) - self.pipe = self.pipe.to(device) - - self.guide_pipe = StableDiffusionXLPipeline.from_pretrained( - "segmind/SSD-1B", - torch_dtype=dtype, - use_safetensors=True, - variant="fp16" - ) - self.guide_pipe = self.guide_pipe.to(device) - - self.seg_image_processor, self.image_segmentor = get_segmentation_pipeline() - self.depth_feature_extractor, self.depth_estimator = get_depth_pipeline() - self.depth_estimator = self.depth_estimator.to(device) - - def generate_design(self, empty_room_image: Image, prompt: str, guidance_scale: int = 10, num_steps: int = 50, strength: float =0.9, img_size: int = 640, img_dept_threshold=0.5, img_seg_threshold=0.5 ) -> Image: + def segment_image(self, + image: Image, + ) -> Image: """ - Given an image of an empty room and a prompt - generate the designed room according to the prompt - Inputs - - empty_room_image - An RGB PIL Image of the empty room - prompt - Text describing the target design elements of the room - Returns - - design_image - PIL Image of the same size as the empty room image - If the size is not the same the submission will fail. + Segments an image using a semantic segmentation model. + Args: + image (Image): The input image to be segmented. + image_processor (AutoImageProcessor): The processor to prepare the + image for segmentation. + image_segmentor (UperNetForSemanticSegmentation): The semantic + segmentation model used to identify different segments in the image. + Returns: + Image: The segmented image with each segment colored differently based + on its identified class. """ - print(f"Positive Prompt: {prompt}") - print(f"Negative Prompt: {self.neg_prompt}") - print(f"Seed: {self.seed}") - print(f"IP Adapter Scale: {self.ip_adapter_scale}") + # image_processor, image_segmentor = get_segmentation_pipeline() + pixel_values = self.image_processor(image, return_tensors="pt").pixel_values + image_size = image.size[::-1] + print(f"Image size: {image_size}, pixel_values shape: {pixel_values.shape}") + with torch.no_grad(): + outputs = self.image_segmentor(pixel_values) + + seg = self.image_processor.post_process_semantic_segmentation( + outputs, target_sizes=[image_size])[0] + color_seg = np.zeros((seg.shape[0], seg.shape[1], 3), dtype=np.uint8) + palette = np.array(ade_palette()) + for label, color in enumerate(palette): + color_seg[seg == label, :] = color + color_seg = color_seg.astype(np.uint8) + seg_image = Image.fromarray(color_seg).convert('RGB') + return seg_image + + @staticmethod + def filter_items( + colors_list: Union[List, np.ndarray], + items_list: Union[List, np.ndarray], + items_to_remove: Union[List, np.ndarray] + ) -> Tuple[Union[List, np.ndarray], Union[List, np.ndarray]]: + """ + Filters items and their corresponding colors from given lists, excluding + specified items. + Args: + colors_list: A list or numpy array of colors corresponding to items. + items_list: A list or numpy array of items. + items_to_remove: A list or numpy array of items to be removed. + Returns: + A tuple of two lists or numpy arrays: filtered colors and filtered + items. + """ + filtered_colors = [] + filtered_items = [] + for color, item in zip(colors_list, items_list): + if item not in items_to_remove: + filtered_colors.append(color) + filtered_items.append(item) + return filtered_colors, filtered_items + - flush() - self.generator = torch.Generator(device=device).manual_seed(self.seed) - - pos_prompt = prompt + f', {self.additional_quality_suffix}' - - orig_w, orig_h = empty_room_image.size - new_width, new_height = resize_dimensions(empty_room_image.size, img_size) - input_image = empty_room_image.resize((new_width, new_height)) - real_seg = np.array(segment_image(input_image, - self.seg_image_processor, - self.image_segmentor)) + def __call__(self, images: Tensor, control_items: List) -> Tuple[Tensor, Tensor]: + """ + images: Tensor (B,C,H,W) ou (C,H,W) + Returns: + seg_cond_tensor: (B,3,H,W) uint8 + mask_tensor: (B,1,H,W) float 0/1 + """ + print(f'Control items: {control_items}') + if images.dim() == 4: + input_image = self.tensor_to_pil(images[0]) + else: + input_image = self.tensor_to_pil(images) + real_seg = np.array(self.segment_image(input_image)) unique_colors = np.unique(real_seg.reshape(-1, real_seg.shape[2]), axis=0) unique_colors = [tuple(color) for color in unique_colors] segment_items = [map_colors_rgb(i) for i in unique_colors] - chosen_colors, segment_items = filter_items( + chosen_colors, segment_items = self.filter_items( colors_list=unique_colors, items_list=segment_items, - items_to_remove=self.control_items + items_to_remove=control_items ) - mask = np.zeros_like(real_seg) + mask = np.zeros((real_seg.shape[0], real_seg.shape[1]), dtype=np.uint8) for color in chosen_colors: color_matches = (real_seg == color).all(axis=2) mask[color_matches] = 1 - image_np = np.array(input_image) - image = Image.fromarray(image_np).convert("RGB") - mask_image = Image.fromarray((mask * 255).astype(np.uint8)).convert("RGB") - segmentation_cond_image = Image.fromarray(real_seg).convert("RGB") - - image_depth = get_depth_image(image, self.depth_feature_extractor, self.depth_estimator) - - # generate image that would be used as IP-adapter - flush() - new_width_ip = int(new_width / 8) * 8 - new_height_ip = int(new_height / 8) * 8 - ip_image = self.guide_pipe(pos_prompt, - num_inference_steps=num_steps, - negative_prompt=self.neg_prompt, - height=new_height_ip, - width=new_width_ip, - generator=[self.generator]).images[0] - - flush() - generated_image = self.pipe( - prompt=pos_prompt, - negative_prompt=self.neg_prompt, - num_inference_steps=num_steps, - strength=strength, - guidance_scale=guidance_scale, - generator=[self.generator], - image=image, - mask_image=mask_image, - ip_adapter_image=ip_image, - control_image=[image_depth, segmentation_cond_image], - controlnet_conditioning_scale=[img_dept_threshold, img_seg_threshold] - ).images[0] - - flush() - design_image = generated_image.resize( - (orig_w, orig_h), Image.Resampling.LANCZOS - ) - - return design_image - - - + seg_cond_image = Image.fromarray(real_seg).convert("RGB") + seg_cond_tensor = torch.from_numpy(np.array(seg_cond_image)).permute(2,0,1).unsqueeze(0).to(self.device).float() / 255.0 # shape (1,3,H,W) + mask_tensor = torch.from_numpy(mask).unsqueeze(0).unsqueeze(0).float() # shape (1,1,H,W) + mask_tensor = mask_tensor.to('cpu') # move to cpu (avoid bug when use the mask on IP Adapter?) + print(f'Shape of seg_cond_tensor: {seg_cond_tensor.shape}, mask_tensor: {mask_tensor.shape}') + return seg_cond_tensor, mask_tensor diff --git a/node/utils.py b/node/utils.py index 6c8ee73..fa787a6 100644 --- a/node/utils.py +++ b/node/utils.py @@ -1,5 +1,4 @@ import gc - import numpy as np from PIL import Image import torch diff --git a/pyproject.toml b/pyproject.toml index e4cefeb..3fda407 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "interior-design-for-comfyui" description = "This node is based on MykolaL/StableDesign" -version = "1.1.5" +version = "2.0.0" license = {file = "LICENSE"} [project.urls] diff --git a/requirements.txt b/requirements.txt index a19199d..30b93b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,5 @@ -transformers==4.48.0 -huggingface_hub==0.30.2 -accelerate==1.6.0 -altair==5.5.0 -diffusers==0.33.1 -numpy==1.26.4 -opencv-python==4.11.0.86 -pillow==11.2.1 -scipy==1.15.2 -torchvision -transformers==4.48.0 -xformers==0.0.29.post3 +torch +transformers +Pillow +numpy +scipy \ No newline at end of file diff --git a/workflow/example.json b/workflow/example.json deleted file mode 100644 index 35ff0e0..0000000 --- a/workflow/example.json +++ /dev/null @@ -1,217 +0,0 @@ -{ - "last_node_id": 30, - "last_link_id": 44, - "nodes": [ - { - "id": 14, - "type": "PreviewImage", - "pos": [ - 936.3425903320312, - -25.60385513305664 - ], - "size": [ - 210, - 246 - ], - "flags": { - "collapsed": false - }, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 43 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "PreviewImage" - }, - "widgets_values": [] - }, - { - "id": 5, - "type": "LoadImage", - "pos": [ - 14.436566352844238, - -146.12979125976562 - ], - "size": [ - 315, - 314 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 42 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "bedroom_1.jpg", - "image" - ] - }, - { - "id": 26, - "type": "interior-design-for-comfyui", - "pos": [ - 487.18255615234375, - -8.370706558227539 - ], - "size": [ - 340.20001220703125, - 274 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 42 - }, - { - "name": "positive", - "type": "STRING", - "link": 44, - "widget": { - "name": "positive" - } - }, - { - "name": "negative", - "type": "STRING", - "link": null, - "widget": { - "name": "negative" - }, - "shape": 7 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 43 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "interior-design-for-comfyui" - }, - "widgets_values": [ - "", - 10, - 50, - 0.9, - 814014690245587, - "randomize", - 768, - 0.5, - 0.5, - "" - ] - }, - { - "id": 29, - "type": "StringFunction|pysssss", - "pos": [ - -133.7979736328125, - 223.61289978027344 - ], - "size": [ - 577.1492309570312, - 354.80126953125 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "STRING", - "type": "STRING", - "links": [ - 44 - ] - } - ], - "properties": { - "Node name for S&R": "StringFunction|pysssss" - }, - "widgets_values": [ - "append", - "yes", - "An elegantly appointed bedroom in the Art Deco style, featuring a grand king-size bed with geometric bedding, a luxurious velvet armchair, and a mirrored nightstand that reflects the room's opulence. Art Deco-inspired artwork adds a touch of glamour", - "", - "", - "An elegantly appointed bedroom in the Art Deco style, featuring a grand king-size bed with geometric bedding, a luxurious velvet armchair, and a mirrored nightstand that reflects the room's opulence. Art Deco-inspired artwork adds a touch of glamour" - ] - } - ], - "links": [ - [ - 42, - 5, - 0, - 26, - 0, - "IMAGE" - ], - [ - 43, - 26, - 0, - 14, - 0, - "IMAGE" - ], - [ - 44, - 29, - 0, - 26, - 1, - "STRING" - ] - ], - "groups": [], - "config": {}, - "extra": { - "ds": { - "scale": 0.5706456420278478, - "offset": [ - 625.0543518197444, - 269.4681234254746 - ] - }, - "ue_links": [], - "node_versions": { - "comfy-core": "v0.3.7-10-g23827ca", - "StableDesign-for-ComfyUI": "d469e19f1196cce71737197957efd604c7d45c4f", - "comfyui-custom-scripts": "a53ef9b617ed1331640d7a2cd97644995908dc00" - } - }, - "version": 0.4 -} \ No newline at end of file diff --git a/workflow/stable-desing-for-comfyui.json b/workflow/stable-desing-for-comfyui.json new file mode 100644 index 0000000..4caf963 --- /dev/null +++ b/workflow/stable-desing-for-comfyui.json @@ -0,0 +1,3068 @@ +{ + "id": "2e2763d1-cbdd-47c4-bbc1-155355544f13", + "revision": 0, + "last_node_id": 161, + "last_link_id": 346, + "nodes": [ + { + "id": 6, + "type": "VAEDecode", + "pos": [ + 412.9100341796875, + -726.0072631835938 + ], + "size": [ + 140, + 46 + ], + "flags": {}, + "order": 42, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 20 + }, + { + "name": "vae", + "type": "VAE", + "link": 3 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 56, + 92 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [] + }, + { + "id": 7, + "type": "KSampler", + "pos": [ + 90.69685363769531, + -736.1925048828125 + ], + "size": [ + 270, + 262 + ], + "flags": {}, + "order": 39, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 12 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 80 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 81 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 6 + }, + { + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": 204 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 20 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 42, + "fixed", + 25, + 9, + "euler", + "normal", + 0.9000000000000001 + ] + }, + { + "id": 22, + "type": "ControlNetLoader", + "pos": [ + -702.7457885742188, + 414.4010009765625 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CONTROL_NET", + "type": "CONTROL_NET", + "links": [ + 34 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "ControlNetLoader" + }, + "widgets_values": [ + "depth\\controlnet_depth(StableDesign).safetensors" + ] + }, + { + "id": 33, + "type": "PrimitiveStringMultiline", + "pos": [ + -1395.5089111328125, + -123.28988647460938 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 207 + ] + } + ], + "title": "Negative Prompt (Optional)", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "PrimitiveStringMultiline" + }, + "widgets_values": [ + " " + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 81, + "type": "StringConcatenate", + "pos": [ + -887.3472290039062, + -397.93585205078125 + ], + "size": [ + 368.97589111328125, + 166 + ], + "flags": { + "collapsed": true + }, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "string_a", + "type": "STRING", + "widget": { + "name": "string_a" + }, + "link": 210 + } + ], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 203, + 209 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "StringConcatenate" + }, + "widgets_values": [ + "", + "interior design, 4K, high resolution, photorealistic", + "," + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 23, + "type": "ControlNetLoader", + "pos": [ + -524.0086669921875, + 81.8063735961914 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CONTROL_NET", + "type": "CONTROL_NET", + "links": [ + 258 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "ControlNetLoader" + }, + "widgets_values": [ + "seg\\own_controlnet(StableDesign).safetensors" + ] + }, + { + "id": 3, + "type": "CheckpointLoaderSimple", + "pos": [ + -536.0324096679688, + -609.9761352539062 + ], + "size": [ + 270, + 98 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 12 + ] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 8, + 11 + ] + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 3 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "SSD-1B.safetensors" + ] + }, + { + "id": 25, + "type": "SetUnionControlNetType", + "pos": [ + -172.53834533691406, + 117.04817199707031 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "control_net", + "type": "CONTROL_NET", + "link": 258 + } + ], + "outputs": [ + { + "name": "CONTROL_NET", + "type": "CONTROL_NET", + "links": [ + 259 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "SetUnionControlNetType" + }, + "widgets_values": [ + "segment" + ] + }, + { + "id": 32, + "type": "PrimitiveStringMultiline", + "pos": [ + -1397.5028076171875, + -365.8805236816406 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 210 + ] + } + ], + "title": "Positive Prompt", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "PrimitiveStringMultiline" + }, + "widgets_values": [ + "A dining room that epitomizes contemporary elegance, anchored by a sleek, minimalist dining table paired with stylish modern chairs. Artistic lighting fixtures create a focal point above, while the surrounding minimalist decor ensures the space feels open, airy, and utterly modern" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 76, + "type": "VAEEncodeForInpaint", + "pos": [ + 621.3834228515625, + 41.49498748779297 + ], + "size": [ + 272.4048767089844, + 98 + ], + "flags": {}, + "order": 38, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 277 + }, + { + "name": "vae", + "type": "VAE", + "link": 144 + }, + { + "name": "mask", + "type": "MASK", + "link": 346 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 146 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "VAEEncodeForInpaint" + }, + "widgets_values": [ + 1 + ] + }, + { + "id": 121, + "type": "MaskToImage", + "pos": [ + -328.9177551269531, + 211.1032257080078 + ], + "size": [ + 184.62362670898438, + 26 + ], + "flags": {}, + "order": 37, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 342 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 247 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "MaskToImage" + }, + "widgets_values": [] + }, + { + "id": 4, + "type": "CLIPTextEncode", + "pos": [ + -198.57489013671875, + -651.8577880859375 + ], + "size": [ + 210.29940795898438, + 88 + ], + "flags": { + "collapsed": true + }, + "order": 27, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 8 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 203 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 80 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 5, + "type": "CLIPTextEncode", + "pos": [ + -206.19314575195312, + -582.1829833984375 + ], + "size": [ + 210.29940795898438, + 88 + ], + "flags": { + "collapsed": true + }, + "order": 25, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 11 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 208 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 81 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 132, + "type": "MarkdownNote", + "pos": [ + -530.7215576171875, + -816.4869995117188 + ], + "size": [ + 260.4696960449219, + 162.07650756835938 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "Generates a guide image to be used as input for the IP Adapter.\n\n- Can be generated using the `segmind/SSD-1B` model: \n [https://huggingface.co/segmind/SSD-1B](https://huggingface.co/segmind/SSD-1B/blob/main/SSD-1B.safetensors)\n- The model should be stored under `models/checkpoints` folder. \n- Alternatively, the user may provide a different model or a reference image.\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 111, + "type": "ControlNetApplyAdvanced", + "pos": [ + -140.79214477539062, + -115.4814453125 + ], + "size": [ + 270, + 186 + ], + "flags": {}, + "order": 36, + "mode": 0, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 218 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 217 + }, + { + "name": "control_net", + "type": "CONTROL_NET", + "link": 259 + }, + { + "name": "image", + "type": "IMAGE", + "link": 345 + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": 219 + } + ], + "outputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 298 + ] + }, + { + "name": "negative", + "type": "CONDITIONING", + "links": [ + 301 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "ControlNetApplyAdvanced" + }, + "widgets_values": [ + 0.5000000000000001, + 0, + 1 + ] + }, + { + "id": 137, + "type": "MarkdownNote", + "pos": [ + 795.6456909179688, + -578.12646484375 + ], + "size": [ + 396.82476806640625, + 282.08331298828125 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "Uses the guide image with the [ComfyUI_IPAdapter_plus](https://github.com/cubiq/ComfyUI_IPAdapter_plus) custom node.\n\n## Required Models\n\n- **IP Adapter SD15** \n Download from: \n [https://huggingface.co/h94/IP-Adapter/resolve/main/models/ip-adapter_sd15.safetensors](https://huggingface.co/h94/IP-Adapter/models) \n Save to (maybe you need to create the folder): \n `models/ipadapter`\n\n- **CLIP ViT-H Encoder** \n Download from: \n [https://huggingface.co/h94/IP-Adapter/resolve/main/models/image_encoder/model.safetensors](https://huggingface.co/h94/IP-Adapter/models/image_encoder) \n Save to: \n `models/clip_vision/` \n ⚠️ Must be renamed to `CLIP-ViT-H-14-laion2B-s32B-b79K.safetensors` for the node to work, since advanced model loading is no longer supported and the repo is unmaintained.\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 139, + "type": "MarkdownNote", + "pos": [ + 147.14968872070312, + 24.772274017333984 + ], + "size": [ + 298.49114990234375, + 282.06671142578125 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "This step uses a segmentation map to guide image generation via ControlNet.\n\n## ControlNet Model\n\n- Download the segmentation ControlNet model from: \n [https://huggingface.co/spaces/MykolaL/StableDesign/blob/main/own_controlnet](https://huggingface.co/spaces/MykolaL/StableDesign/blob/main/own_controlnet/diffusion_pytorch_model.safetensors) \n- Save it to: \n `models/controlnet` \n ⚠️ Rename the file to avoid confusion, as it comes with the generic name `diffusion_pytorch_model.safetensors`.\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 13, + "type": "EmptyLatentImage", + "pos": [ + -222.43978881835938, + -480.87078857421875 + ], + "size": [ + 210, + 106 + ], + "flags": { + "collapsed": false + }, + "order": 34, + "mode": 0, + "inputs": [ + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 304 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 305 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 6 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 56, + 512, + 1 + ] + }, + { + "id": 24, + "type": "SetUnionControlNetType", + "pos": [ + -407.49163818359375, + 416.0394592285156 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "control_net", + "type": "CONTROL_NET", + "link": 34 + } + ], + "outputs": [ + { + "name": "CONTROL_NET", + "type": "CONTROL_NET", + "links": [ + 169 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "SetUnionControlNetType" + }, + "widgets_values": [ + "depth" + ] + }, + { + "id": 45, + "type": "DownloadAndLoadDepthAnythingV2Model", + "pos": [ + -740.7487182617188, + 528.8541259765625 + ], + "size": [ + 365.3121032714844, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "da_v2_model", + "type": "DAMODEL", + "links": [ + 82 + ] + } + ], + "properties": { + "cnr_id": "comfyui-depthanythingv2", + "ver": "d505cbca99803fc63327b8305618a23e59a18b42", + "Node name for S&R": "DownloadAndLoadDepthAnythingV2Model", + "aux_id": "kijai/ComfyUI-DepthAnythingV2" + }, + "widgets_values": [ + "depth_anything_v2_vitl_fp32.safetensors", + "auto" + ] + }, + { + "id": 46, + "type": "DepthAnything_V2", + "pos": [ + -717.4320678710938, + 672.8815307617188 + ], + "size": [ + 312.734130859375, + 77.45999908447266 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "name": "da_model", + "type": "DAMODEL", + "link": 82 + }, + { + "name": "images", + "type": "IMAGE", + "link": 276 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 98, + 171 + ] + } + ], + "properties": { + "cnr_id": "comfyui-depthanythingv2", + "ver": "d505cbca99803fc63327b8305618a23e59a18b42", + "Node name for S&R": "DepthAnything_V2" + }, + "widgets_values": [] + }, + { + "id": 79, + "type": "ControlNetApplyAdvanced", + "pos": [ + -343.232421875, + 528.325439453125 + ], + "size": [ + 270, + 186 + ], + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 215 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 216 + }, + { + "name": "control_net", + "type": "CONTROL_NET", + "link": 169 + }, + { + "name": "image", + "type": "IMAGE", + "link": 171 + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": 170 + } + ], + "outputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 297 + ] + }, + { + "name": "negative", + "type": "CONDITIONING", + "links": [ + 300 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "ControlNetApplyAdvanced" + }, + "widgets_values": [ + 0.5000000000000001, + 0, + 1 + ] + }, + { + "id": 138, + "type": "MarkdownNote", + "pos": [ + 118.04875946044922, + 421.34381103515625 + ], + "size": [ + 344.6339111328125, + 304.2916259765625 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "This step uses depth information to guide image generation via ControlNet.\n\n## Depth Estimation\n\n- Uses the custom node [Depth Anything V2](https://github.com/DepthAnything/Depth-Anything-V2)\n- This replaces the older Depth Anything model, offering improved results with no major changes required to the pipeline.\n\n## ControlNet Model\n\n- Download the original custom depth ControlNet model from: \n [https://huggingface.co/spaces/MykolaL/StableDesign/blob/main/controlnet_depth](https://huggingface.co/spaces/MykolaL/StableDesign/blob/main/controlnet_depth/diffusion_pytorch_model.safetensors) \n- Save it to: \n `models/controlnet` \n ⚠️ It is recommended to rename the file, as it uses the default name from the repository (`diffusion_pytorch_model.safetensors`).\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 63, + "type": "IPAdapterUnifiedLoader", + "pos": [ + 610.3306884765625, + -812.8778076171875 + ], + "size": [ + 270, + 78 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 111 + }, + { + "name": "ipadapter", + "shape": 7, + "type": "IPADAPTER", + "link": 115 + } + ], + "outputs": [ + { + "name": "model", + "type": "MODEL", + "links": [ + 112 + ] + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "links": [ + 114 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ipadapter_plus", + "ver": "a0f451a5113cf9becb0847b92884cb10cbdec0ef", + "Node name for S&R": "IPAdapterUnifiedLoader" + }, + "widgets_values": [ + "STANDARD (medium strength)" + ] + }, + { + "id": 64, + "type": "IPAdapterModelLoader", + "pos": [ + 618.4702758789062, + -688.3241577148438 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IPADAPTER", + "type": "IPADAPTER", + "links": [ + 115 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ipadapter_plus", + "ver": "a0f451a5113cf9becb0847b92884cb10cbdec0ef", + "Node name for S&R": "IPAdapterModelLoader" + }, + "widgets_values": [ + "ip-adapter_sd15.safetensors" + ] + }, + { + "id": 122, + "type": "PreviewImage", + "pos": [ + 156.95230102539062, + -262.51239013671875 + ], + "size": [ + 140, + 246.00001525878906 + ], + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 247 + } + ], + "outputs": [], + "title": "Mask", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 120, + "type": "PreviewImage", + "pos": [ + 313.9969177246094, + -260.78179931640625 + ], + "size": [ + 140, + 246 + ], + "flags": {}, + "order": 35, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 344 + } + ], + "outputs": [], + "title": "Segmentation", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 34, + "type": "PreviewImage", + "pos": [ + 429.91259765625, + -612.2884521484375 + ], + "size": [ + 140, + 246 + ], + "flags": {}, + "order": 43, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 56 + } + ], + "outputs": [], + "title": "Style Guidance", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 59, + "type": "PreviewImage", + "pos": [ + -52.13082504272461, + 422.2935791015625 + ], + "size": [ + 140, + 246.00003051757812 + ], + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 98 + } + ], + "outputs": [], + "title": "Depth", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 142, + "type": "CM_NearestSDXLResolution", + "pos": [ + -471.7952575683594, + -430.25872802734375 + ], + "size": [ + 191.64413452148438, + 46 + ], + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 306 + } + ], + "outputs": [ + { + "name": "width", + "type": "INT", + "links": [ + 304 + ] + }, + { + "name": "height", + "type": "INT", + "links": [ + 305 + ] + } + ], + "properties": { + "cnr_id": "ComfyMath", + "ver": "c01177221c31b8e5fbc062778fc8254aeb541638", + "Node name for S&R": "CM_NearestSDXLResolution" + }, + "widgets_values": [] + }, + { + "id": 9, + "type": "LoadImage", + "pos": [ + -945.4544677734375, + -361.9350891113281 + ], + "size": [ + 270, + 314 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 273, + 311 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [] + } + ], + "title": "Load Input Image", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "dinning_room_3.jpg", + "image" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 130, + "type": "Image Normalize", + "pos": [ + -1169.78759765625, + -506.8669128417969 + ], + "size": [ + 227.74464416503906, + 106 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 273 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 276, + 277, + 306, + 340 + ] + } + ], + "properties": { + "cnr_id": "interior-design-for-comfyui", + "ver": "9443c413e17048827197402d64823bf279780479", + "Node name for S&R": "Image Normalize" + }, + "widgets_values": [ + 512, + 8, + "bilinear" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 38, + "type": "CheckpointLoaderSimple", + "pos": [ + 608.1879272460938, + -139.8812255859375 + ], + "size": [ + 319.5867919921875, + 108.5184097290039 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 111 + ] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 61, + 62 + ] + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 79, + 144, + 170, + 219, + 329, + 336 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "Realistic_Vision_V5.1-inpainting.safetensors" + ] + }, + { + "id": 107, + "type": "PrimitiveInt", + "pos": [ + -1404.6439208984375, + -502.1680603027344 + ], + "size": [ + 210, + 82 + ], + "flags": { + "collapsed": false + }, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 204, + 205, + 337 + ] + } + ], + "title": "Seed", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "PrimitiveInt" + }, + "widgets_values": [ + 143668900, + "fixed" + ] + }, + { + "id": 35, + "type": "CLIPTextEncode", + "pos": [ + 982.7101440429688, + -133.51083374023438 + ], + "size": [ + 210.29940795898438, + 88 + ], + "flags": { + "collapsed": true + }, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 61 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 214 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 216, + 217, + 332 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 36, + "type": "CLIPTextEncode", + "pos": [ + 979.80615234375, + -68.59585571289062 + ], + "size": [ + 210.29940795898438, + 88 + ], + "flags": { + "collapsed": true + }, + "order": 28, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 62 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 209 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 215, + 218, + 331 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 126, + "type": "PairConditioningCombine", + "pos": [ + 667.4053344726562, + 209.75762939453125 + ], + "size": [ + 162.07186889648438, + 86 + ], + "flags": {}, + "order": 40, + "mode": 0, + "inputs": [ + { + "name": "positive_A", + "type": "CONDITIONING", + "link": 297 + }, + { + "name": "negative_A", + "type": "CONDITIONING", + "link": 300 + }, + { + "name": "positive_B", + "type": "CONDITIONING", + "link": 298 + }, + { + "name": "negative_B", + "type": "CONDITIONING", + "link": 301 + } + ], + "outputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 302 + ] + }, + { + "name": "negative", + "type": "CONDITIONING", + "links": [ + 303 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "PairConditioningCombine" + }, + "widgets_values": [] + }, + { + "id": 82, + "type": "StringConcatenate", + "pos": [ + -871.6675415039062, + 3.361741065979004 + ], + "size": [ + 210, + 150 + ], + "flags": { + "collapsed": true + }, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "string_b", + "type": "STRING", + "widget": { + "name": "string_b" + }, + "link": 207 + } + ], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 208, + 214 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "StringConcatenate" + }, + "widgets_values": [ + "window, door, low resolution, banner, logo, watermark, text, deformed, blurry, out of focus, surreal, ugly, beginner", + "", + "," + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 42, + "type": "VAEDecode", + "pos": [ + 1220.304931640625, + -161.55670166015625 + ], + "size": [ + 140, + 46 + ], + "flags": {}, + "order": 46, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 77 + }, + { + "name": "vae", + "type": "VAE", + "link": 79 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 308, + 309 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [] + }, + { + "id": 41, + "type": "KSampler", + "pos": [ + 862.2103881835938, + 192.63446044921875 + ], + "size": [ + 270, + 262 + ], + "flags": {}, + "order": 45, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 93 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 302 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 303 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 146 + }, + { + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": 205 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 77 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 42, + "fixed", + 50, + 10, + "euler", + "normal", + 0.9000000000000001 + ] + }, + { + "id": 145, + "type": "PreviewImage", + "pos": [ + 1400.232421875, + -162.7487030029297 + ], + "size": [ + 140, + 246 + ], + "flags": {}, + "order": 48, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 309 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 141, + "type": "MarkdownNote", + "pos": [ + 1168.7044677734375, + 137.92733764648438 + ], + "size": [ + 467.87884521484375, + 416.0183410644531 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "\nThis step generates the final image using **Stable Diffusion 1.5 with inpainting**, guided by both depth and segmentation ControlNets.\n\n## Base Model\n\n- **Model:** [Realistic Vision V5.1 (Inpainting)](https://huggingface.co/SG161222/Realistic_Vision_V5.1_noVAE/blob/main/Realistic_Vision_V5.1-inpainting.safetensors) \n- Save to: \n `models/checkpoints`\n\n> ⚠️ A newer version exists, but no major improvements were observed.\n\n## ControlNet Combination\n\n- ControlNets are combined in this order:\n 1. Depth\n 2. Segmentation\n\n> 🔁 Changing the order had any impact on results. Other approaches might help better match the original pipeline.\n\n\nThe original pipeline explicitly replaced the scheduler with `UniPCMultistepScheduler`:\n\n```python\nfrom diffusers import UniPCMultistepScheduler\n\nself.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pip" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 146, + "type": "ImageScale", + "pos": [ + 1876.01171875, + -161.27243041992188 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 49, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 310 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 312 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 313 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 328 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "ImageScale" + }, + "widgets_values": [ + "bilinear", + 512, + 512, + "disabled" + ] + }, + { + "id": 143, + "type": "UpscaleModelLoader", + "pos": [ + 2166.35302734375, + -153.9991912841797 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "UPSCALE_MODEL", + "type": "UPSCALE_MODEL", + "links": [ + 307 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "UpscaleModelLoader" + }, + "widgets_values": [ + "4x-ESRGAN.pth" + ] + }, + { + "id": 144, + "type": "ImageUpscaleWithModel", + "pos": [ + 2163.442626953125, + -42.37593460083008 + ], + "size": [ + 221.98202514648438, + 46 + ], + "flags": {}, + "order": 47, + "mode": 0, + "inputs": [ + { + "name": "upscale_model", + "type": "UPSCALE_MODEL", + "link": 307 + }, + { + "name": "image", + "type": "IMAGE", + "link": 308 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 310 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "ImageUpscaleWithModel" + }, + "widgets_values": [] + }, + { + "id": 157, + "type": "VAEDecode", + "pos": [ + 1983.02294921875, + 17.500192642211914 + ], + "size": [ + 140, + 46 + ], + "flags": {}, + "order": 52, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 334 + }, + { + "name": "vae", + "type": "VAE", + "link": 336 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 338 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [] + }, + { + "id": 159, + "type": "SaveImage", + "pos": [ + 2150.8388671875, + 55.758201599121094 + ], + "size": [ + 270, + 270 + ], + "flags": {}, + "order": 53, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 338 + } + ], + "outputs": [], + "title": "Output", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41" + }, + "widgets_values": [ + "ComfyUI" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 155, + "type": "KSampler", + "pos": [ + 1840.90625, + 117.30351257324219 + ], + "size": [ + 270, + 262 + ], + "flags": {}, + "order": 51, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 339 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 331 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 332 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 333 + }, + { + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": 337 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 334 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 42, + "fixed", + 100, + 10, + "uni_pc_bh2", + "normal", + 0.5000000000000001 + ] + }, + { + "id": 156, + "type": "VAEEncode", + "pos": [ + 1827.583984375, + 18.867908477783203 + ], + "size": [ + 140, + 46 + ], + "flags": {}, + "order": 50, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 328 + }, + { + "name": "vae", + "type": "VAE", + "link": 329 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 333 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "VAEEncode" + }, + "widgets_values": [] + }, + { + "id": 147, + "type": "GetImageSize", + "pos": [ + 1727.86376953125, + -149.89578247070312 + ], + "size": [ + 140, + 124 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 311 + } + ], + "outputs": [ + { + "name": "width", + "type": "INT", + "links": [ + 312 + ] + }, + { + "name": "height", + "type": "INT", + "links": [ + 313 + ] + }, + { + "name": "batch_size", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "GetImageSize" + }, + "widgets_values": [] + }, + { + "id": 160, + "type": "MarkdownNote", + "pos": [ + 2463.355712890625, + -168.02671813964844 + ], + "size": [ + 422.3630676269531, + 312.2231750488281 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "This step performs a final refinement pass using the diffusion model, aimed at improving image coherence while preserving original details.\n\n## Process Overview\n\n1. **Upscaling:** \n - The image from Step is upscaled using **ESRGAN x4**.\n\n2. **Downscaling:** \n - The upscaled image is resized back to its original resolution.\n\n3. **Latent Noise Injection:** \n - The latent representation of the resized image is injected with **50% noise** before reprocessing.\n\n4. **Diffusion Pass with IP Adapter Only:** \n - The image is passed again through the **IP Adapter**, using both **positive and negative CLIP embeddings**, but **without ControlNet**, to avoid overwriting fine details.\n - The same **inpainting model** is reused (`Realistic_Vision_V5.1-inpainting.safetensors`), but with **double the number of steps** for improved refinement.\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 57, + "type": "IPAdapter", + "pos": [ + 922.7847290039062, + -812.4465942382812 + ], + "size": [ + 270, + 190 + ], + "flags": {}, + "order": 44, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 112 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 114 + }, + { + "name": "image", + "type": "IMAGE", + "link": 92 + }, + { + "name": "attn_mask", + "shape": 7, + "type": "MASK", + "link": 343 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 93, + 339 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ipadapter_plus", + "ver": "a0f451a5113cf9becb0847b92884cb10cbdec0ef", + "Node name for S&R": "IPAdapter" + }, + "widgets_values": [ + 1.0000000000000002, + 0, + 1, + "standard" + ] + }, + { + "id": 131, + "type": "Control Items", + "pos": [ + -503.3625183105469, + -102.65502166748047 + ], + "size": [ + 270, + 106 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CONTROL_ITEMS", + "type": "CONTROL_ITEMS", + "links": [ + 341 + ] + } + ], + "properties": { + "cnr_id": "interior-design-for-comfyui", + "ver": "9443c413e17048827197402d64823bf279780479", + "Node name for S&R": "Control Items" + }, + "widgets_values": [ + true, + true, + false + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 161, + "type": "Interior Design Segmentator", + "pos": [ + -461.2010498046875, + -224.7035675048828 + ], + "size": [ + 214.98886108398438, + 46 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 340 + }, + { + "name": "control_items", + "type": "CONTROL_ITEMS", + "link": 341 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 344, + 345 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 342, + 343, + 346 + ] + } + ], + "properties": { + "cnr_id": "interior-design-for-comfyui", + "ver": "9443c413e17048827197402d64823bf279780479", + "Node name for S&R": "Interior Design Segmentator" + }, + "color": "#233", + "bgcolor": "#355" + } + ], + "links": [ + [ + 3, + 3, + 2, + 6, + 1, + "VAE" + ], + [ + 6, + 13, + 0, + 7, + 3, + "LATENT" + ], + [ + 8, + 3, + 1, + 4, + 0, + "CLIP" + ], + [ + 11, + 3, + 1, + 5, + 0, + "CLIP" + ], + [ + 12, + 3, + 0, + 7, + 0, + "MODEL" + ], + [ + 20, + 7, + 0, + 6, + 0, + "LATENT" + ], + [ + 34, + 22, + 0, + 24, + 0, + "CONTROL_NET" + ], + [ + 56, + 6, + 0, + 34, + 0, + "IMAGE" + ], + [ + 61, + 38, + 1, + 35, + 0, + "CLIP" + ], + [ + 62, + 38, + 1, + 36, + 0, + "CLIP" + ], + [ + 77, + 41, + 0, + 42, + 0, + "LATENT" + ], + [ + 79, + 38, + 2, + 42, + 1, + "VAE" + ], + [ + 80, + 4, + 0, + 7, + 1, + "CONDITIONING" + ], + [ + 81, + 5, + 0, + 7, + 2, + "CONDITIONING" + ], + [ + 82, + 45, + 0, + 46, + 0, + "DAMODEL" + ], + [ + 92, + 6, + 0, + 57, + 2, + "IMAGE" + ], + [ + 93, + 57, + 0, + 41, + 0, + "MODEL" + ], + [ + 98, + 46, + 0, + 59, + 0, + "IMAGE" + ], + [ + 111, + 38, + 0, + 63, + 0, + "MODEL" + ], + [ + 112, + 63, + 0, + 57, + 0, + "MODEL" + ], + [ + 114, + 63, + 1, + 57, + 1, + "IPADAPTER" + ], + [ + 115, + 64, + 0, + 63, + 1, + "IPADAPTER" + ], + [ + 144, + 38, + 2, + 76, + 1, + "VAE" + ], + [ + 146, + 76, + 0, + 41, + 3, + "LATENT" + ], + [ + 169, + 24, + 0, + 79, + 2, + "CONTROL_NET" + ], + [ + 170, + 38, + 2, + 79, + 4, + "VAE" + ], + [ + 171, + 46, + 0, + 79, + 3, + "IMAGE" + ], + [ + 203, + 81, + 0, + 4, + 1, + "STRING" + ], + [ + 204, + 107, + 0, + 7, + 4, + "INT" + ], + [ + 205, + 107, + 0, + 41, + 4, + "INT" + ], + [ + 207, + 33, + 0, + 82, + 0, + "STRING" + ], + [ + 208, + 82, + 0, + 5, + 1, + "STRING" + ], + [ + 209, + 81, + 0, + 36, + 1, + "STRING" + ], + [ + 210, + 32, + 0, + 81, + 0, + "STRING" + ], + [ + 214, + 82, + 0, + 35, + 1, + "STRING" + ], + [ + 215, + 36, + 0, + 79, + 0, + "CONDITIONING" + ], + [ + 216, + 35, + 0, + 79, + 1, + "CONDITIONING" + ], + [ + 217, + 35, + 0, + 111, + 1, + "CONDITIONING" + ], + [ + 218, + 36, + 0, + 111, + 0, + "CONDITIONING" + ], + [ + 219, + 38, + 2, + 111, + 4, + "VAE" + ], + [ + 247, + 121, + 0, + 122, + 0, + "IMAGE" + ], + [ + 258, + 23, + 0, + 25, + 0, + "CONTROL_NET" + ], + [ + 259, + 25, + 0, + 111, + 2, + "CONTROL_NET" + ], + [ + 273, + 9, + 0, + 130, + 0, + "IMAGE" + ], + [ + 276, + 130, + 0, + 46, + 1, + "IMAGE" + ], + [ + 277, + 130, + 0, + 76, + 0, + "IMAGE" + ], + [ + 297, + 79, + 0, + 126, + 0, + "CONDITIONING" + ], + [ + 298, + 111, + 0, + 126, + 2, + "CONDITIONING" + ], + [ + 300, + 79, + 1, + 126, + 1, + "CONDITIONING" + ], + [ + 301, + 111, + 1, + 126, + 3, + "CONDITIONING" + ], + [ + 302, + 126, + 0, + 41, + 1, + "CONDITIONING" + ], + [ + 303, + 126, + 1, + 41, + 2, + "CONDITIONING" + ], + [ + 304, + 142, + 0, + 13, + 0, + "INT" + ], + [ + 305, + 142, + 1, + 13, + 1, + "INT" + ], + [ + 306, + 130, + 0, + 142, + 0, + "IMAGE" + ], + [ + 307, + 143, + 0, + 144, + 0, + "UPSCALE_MODEL" + ], + [ + 308, + 42, + 0, + 144, + 1, + "IMAGE" + ], + [ + 309, + 42, + 0, + 145, + 0, + "IMAGE" + ], + [ + 310, + 144, + 0, + 146, + 0, + "IMAGE" + ], + [ + 311, + 9, + 0, + 147, + 0, + "IMAGE" + ], + [ + 312, + 147, + 0, + 146, + 1, + "INT" + ], + [ + 313, + 147, + 1, + 146, + 2, + "INT" + ], + [ + 328, + 146, + 0, + 156, + 0, + "IMAGE" + ], + [ + 329, + 38, + 2, + 156, + 1, + "VAE" + ], + [ + 331, + 36, + 0, + 155, + 1, + "CONDITIONING" + ], + [ + 332, + 35, + 0, + 155, + 2, + "CONDITIONING" + ], + [ + 333, + 156, + 0, + 155, + 3, + "LATENT" + ], + [ + 334, + 155, + 0, + 157, + 0, + "LATENT" + ], + [ + 336, + 38, + 2, + 157, + 1, + "VAE" + ], + [ + 337, + 107, + 0, + 155, + 4, + "INT" + ], + [ + 338, + 157, + 0, + 159, + 0, + "IMAGE" + ], + [ + 339, + 57, + 0, + 155, + 0, + "MODEL" + ], + [ + 340, + 130, + 0, + 161, + 0, + "IMAGE" + ], + [ + 341, + 131, + 0, + 161, + 1, + "CONTROL_ITEMS" + ], + [ + 342, + 161, + 1, + 121, + 0, + "MASK" + ], + [ + 343, + 161, + 1, + 57, + 3, + "MASK" + ], + [ + 344, + 161, + 0, + 120, + 0, + "IMAGE" + ], + [ + 345, + 161, + 0, + 111, + 3, + "IMAGE" + ], + [ + 346, + 161, + 1, + 76, + 2, + "MASK" + ] + ], + "groups": [ + { + "id": 1, + "title": "IP Adapter Image Guidance", + "bounding": [ + -546.0324096679688, + -890.0869750976562, + 1125.945068359375, + 533.7985229492188 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Input", + "bounding": [ + -1414.6439208984375, + -579.0248413085938, + 749.1898193359375, + 665.7351684570312 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Depth Controlnet", + "bounding": [ + -750.7487182617188, + 340.8009948730469, + 1223.431396484375, + 419.5405578613281 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Diffusion Step", + "bounding": [ + 598.1879272460938, + -236.3487091064453, + 1048.395263671875, + 800.2943725585938 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 5, + "title": "IP Adapter", + "bounding": [ + 600.3306884765625, + -886.477783203125, + 602.1399536132812, + 600.4346313476562 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 6, + "title": "Segment Controlnet", + "bounding": [ + -534.0086669921875, + -336.11248779296875, + 998.0056762695312, + 652.951416015625 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 7, + "title": "Output", + "bounding": [ + 1717.86376953125, + -241.62672424316406, + 1177.85498046875, + 630.9302368164062 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6934334949442131, + "offset": [ + 1331.5040977754481, + 706.8026326130054 + ] + }, + "groupNodes": {}, + "frontendVersion": "1.21.7" + }, + "version": 0.4 +} \ No newline at end of file