fix: apply differential diffusion before move to device.

This patch will prevent OOM from DifferentialDiffusion

README.md

@@ -228,11 +228,14 @@ This custom node helps to conveniently enhance images through Detector, Detailer
   
 
 ### Batch/List Util
-  * `Image batch To Image List` - Convert Image batch to Image List
+  * `Image Batch to Image List` - Convert Image batch to Image List
     - You can use images generated in a multi batch to handle them
+  * `Image List to Image Batch` - Convert Image List to Image Batch 
   * `Make Image List` - Convert multiple images into a single image list
   * `Make Image Batch` - Convert multiple images into a single image batch
     - The input of images can be scaled up as needed
+  * `Masks to Mask List`, `Mask List to Masks`, `Make Mask List`, `Make Mask Batch` - It has the same functionality as the nodes above, but uses mask as input instead of image.
+  * `Flatten Mask Batch` - Flattens a Mask Batch into a single Mask. Normal operation is not guaranteed for non-binary masks. 
 
 
 ### Logics (experimental) 

__init__.py

@@ -161,6 +161,7 @@ NODE_CLASS_MAPPINGS = {
     "ImpactSegsAndMask": SegsBitwiseAndMask,
     "ImpactSegsAndMaskForEach": SegsBitwiseAndMaskForEach,
     "EmptySegs": EmptySEGS,
+    "ImpactFlattenMask": FlattenMask,
 
     "MediaPipeFaceMeshToSEGS": MediaPipeFaceMeshToSEGS,
     "MaskToSEGS": MaskToSEGS,
@@ -249,6 +250,8 @@ NODE_CLASS_MAPPINGS = {
     "ImpactImageBatchToImageList": ImageBatchToImageList,
     "ImpactMakeImageList": MakeImageList,
     "ImpactMakeImageBatch": MakeImageBatch,
+    "ImpactMakeMaskList": MakeMaskList,
+    "ImpactMakeMaskBatch": MakeMaskBatch,
 
     "RegionalSampler": RegionalSampler,
     "RegionalSamplerAdvanced": RegionalSamplerAdvanced,
@@ -332,6 +335,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "BitwiseAndMask": "Pixelwise(MASK & MASK)",
     "SubtractMask": "Pixelwise(MASK - MASK)",
     "AddMask": "Pixelwise(MASK + MASK)",
+    "ImpactFlattenMask": "Flatten Mask Batch",
     "DetailerForEach": "Detailer (SEGS)",
     "DetailerForEachPipe": "Detailer (SEGS/pipe)",
     "DetailerForEachDebug": "DetailerDebug (SEGS)",
@@ -400,12 +404,16 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "ImpactInversedSwitch": "Inversed Switch (Any)",
     "ImpactExecutionOrderController": "Execution Order Controller",
 
-    "MasksToMaskList": "Masks to Mask List",
-    "MaskListToMaskBatch": "Mask List to Masks",
-    "ImpactImageBatchToImageList": "Image batch to Image List",
+    "MasksToMaskList": "Mask Batch to Mask List",
+    "MaskListToMaskBatch": "Mask List to Mask Batch",
+    "ImpactImageBatchToImageList": "Image Batch to Image List",
     "ImageListToImageBatch": "Image List to Image Batch",
+
     "ImpactMakeImageList": "Make Image List",
     "ImpactMakeImageBatch": "Make Image Batch",
+    "ImpactMakeMaskList": "Make Mask List",
+    "ImpactMakeMaskBatch": "Make Mask Batch",
+
     "ImpactStringSelector": "String Selector",
     "StringListToString": "String List to String",
     "WildcardPromptFromString": "Wildcard Prompt from String",

js/impact-pack.js

@@ -237,7 +237,7 @@ app.registerExtension({
 		if(nodeData.name == "ImpactControlBridge") {
 			const onConnectionsChange = nodeType.prototype.onConnectionsChange;
 			nodeType.prototype.onConnectionsChange = function (type, index, connected, link_info) {
-				if(!link_info || this.inputs[0].type != '*')
+				if(index != 0 || !link_info || this.inputs[0].type != '*')
 					return;
 
 				// assign type
@@ -393,6 +393,7 @@ app.registerExtension({
 		}
 
 		if (nodeData.name === 'ImpactMakeImageList' || nodeData.name === 'ImpactMakeImageBatch' ||
+		    nodeData.name === 'ImpactMakeMaskList' || nodeData.name === 'ImpactMakeMaskBatch' ||
 			nodeData.name === 'CombineRegionalPrompts' ||
 			nodeData.name === 'ImpactCombineConditionings' || nodeData.name === 'ImpactConcatConditionings' ||
 			nodeData.name === 'ImpactSEGSConcat' ||
@@ -405,6 +406,11 @@ app.registerExtension({
 				input_name = "image";
 				break;
 
+			case 'ImpactMakeMaskList':
+			case 'ImpactMakeMaskBatch':
+				input_name = "mask";
+				break;
+
 			case 'ImpactSEGSConcat':
 				input_name = "segs";
 				break;

js/impact-wildcard.js

@@ -6,11 +6,15 @@ let refresh_btn2 = document.querySelector('button[title="Refresh widgets in node
 
 let orig = refresh_btn.onclick;
 
-refresh_btn.onclick = function() {
-	orig();
-	api.fetchApi('/impact/wildcards/refresh');
-};
+if(refresh_btn) {
+    refresh_btn.onclick = function() {
+        orig();
+        api.fetchApi('/impact/wildcards/refresh');
+    };
+}
 
-refresh_btn2.addEventListener('click', function() {
-	api.fetchApi('/impact/wildcards/refresh');
-});
+if(refresh_btn2) {
+    refresh_btn2?.addEventListener('click', function() {
+        api.fetchApi('/impact/wildcards/refresh');
+    });
+}

modules/impact/animatediff_nodes.py

@@ -5,6 +5,13 @@ from impact.core import SEG
 from impact.segs_nodes import SEGSPaste
 
 
+try:
+    from comfy_extras import nodes_differential_diffusion
+except Exception:
+    print(f"\n#############################################\n[Impact Pack] ComfyUI is an outdated version.\n#############################################\n")
+    raise Exception("[Impact Pack] ComfyUI is an outdated version.")
+
+
 class SEGSDetailerForAnimateDiff:
     @classmethod
     def INPUT_TYPES(cls):
@@ -53,6 +60,9 @@ class SEGSDetailerForAnimateDiff:
         new_segs = []
         cnet_image_list = []
 
+        if noise_mask_feather > 0 and 'denoise_mask_function' not in model.model_options:
+            model = nodes_differential_diffusion.DifferentialDiffusion().apply(model)[0]
+
         for seg in segs[1]:
             cropped_image_frames = None
 

modules/impact/bridge_nodes.py

@@ -19,6 +19,9 @@ class PreviewBridge:
                     "images": ("IMAGE",),
                     "image": ("STRING", {"default": ""}),
                     },
+                "optional": {
+                    "block": ("BOOLEAN", {"default": False, "label_on": "if_empty_mask", "label_off": "never", "tooltip": "is_empty_mask: If the mask is empty, the execution is stopped.\nnever: The execution is never stopped."})
+                    },
                 "hidden": {"unique_id": "UNIQUE_ID", "extra_pnginfo": "EXTRA_PNGINFO"},
                 }
 
@@ -30,6 +33,8 @@ class PreviewBridge:
 
     CATEGORY = "ImpactPack/Util"
 
+    DESCRIPTION = "This is a feature that allows you to edit and send a Mask over a image.\nIf the block is set to 'is_empty_mask', the execution is stopped when the mask is empty."
+
     def __init__(self):
         super().__init__()
         self.output_dir = folder_paths.get_temp_directory()
@@ -70,7 +75,7 @@ class PreviewBridge:
 
         return image, mask.unsqueeze(0), ui_item
 
-    def doit(self, images, image, unique_id, prompt=None, extra_pnginfo=None):
+    def doit(self, images, image, unique_id, block=False, prompt=None, extra_pnginfo=None):
         need_refresh = False
 
         if unique_id not in core.preview_bridge_cache:
@@ -96,9 +101,20 @@ class PreviewBridge:
 
             image = image2
 
+        is_empty_mask = torch.all(mask == 0)
+
+        if block and is_empty_mask and core.is_execution_model_version_supported:
+            from comfy_execution.graph import ExecutionBlocker
+            result = ExecutionBlocker(None), ExecutionBlocker(None)
+        elif block and is_empty_mask:
+            print(f"[Impact Pack] PreviewBridge: ComfyUI is outdated - blocking feature is disabled.")
+            result = pixels, mask
+        else:
+            result = pixels, mask
+
         return {
             "ui": {"images": image},
-            "result": (pixels, mask, ),
+            "result": result,
         }
 
 
@@ -179,7 +195,8 @@ class PreviewBridgeLatent:
                                         "TAEF1", "TAESDXL", "TAESD15", "TAESD3"],),
                     },
                 "optional": {
-                    "vae_opt": ("VAE", )
+                    "vae_opt": ("VAE", ),
+                    "block": ("BOOLEAN", {"default": False, "label_on": "if_empty_mask", "label_off": "never", "tooltip": "is_empty_mask: If the mask is empty, the execution is stopped.\nnever: The execution is never stopped. Instead, it returns a white mask."})
                 },
                 "hidden": {"unique_id": "UNIQUE_ID", "prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"},
                 }
@@ -192,6 +209,8 @@ class PreviewBridgeLatent:
 
     CATEGORY = "ImpactPack/Util"
 
+    DESCRIPTION = "This is a feature that allows you to edit and send a Mask over a latent image.\nIf the block is set to 'is_empty_mask', the execution is stopped when the mask is empty."
+
     def __init__(self):
         super().__init__()
         self.output_dir = folder_paths.get_temp_directory()
@@ -233,7 +252,7 @@ class PreviewBridgeLatent:
 
         return image, mask, ui_item
 
-    def doit(self, latent, image, preview_method, vae_opt=None, unique_id=None, prompt=None, extra_pnginfo=None):
+    def doit(self, latent, image, preview_method, vae_opt=None, block=False, unique_id=None, prompt=None, extra_pnginfo=None):
         latent_channels = latent['samples'].shape[1]
         preview_method_channels = 16 if 'SD3' in preview_method or 'SC-Prior' in preview_method or 'FLUX.1' in preview_method or 'TAEF1' == preview_method else 4
 
@@ -262,10 +281,14 @@ class PreviewBridgeLatent:
                     del res_latent['noise_mask']
                 else:
                     res_latent = latent
+
+                is_empty_mask = True
             else:
                 res_latent = latent.copy()
                 res_latent['noise_mask'] = mask
 
+                is_empty_mask = torch.all(mask == 1)
+
             res_image = [path_item]
         else:
             decoded_image = decode_latent(latent, preview_method, vae_opt)
@@ -287,11 +310,15 @@ class PreviewBridgeLatent:
                                 'subfolder': 'PreviewBridge',
                                 'type': 'temp',
                             }]
+
+                is_empty_mask = torch.all(mask == 1)
             else:
                 mask = torch.ones(latent['samples'].shape[2:], dtype=torch.float32, device="cpu").unsqueeze(0)
                 res = nodes.PreviewImage().save_images(decoded_image, filename_prefix="PreviewBridge/PBL-", prompt=prompt, extra_pnginfo=extra_pnginfo)
                 res_image = res['ui']['images']
 
+                is_empty_mask = True
+
             path = os.path.join(folder_paths.get_temp_directory(), 'PreviewBridge', res_image[0]['filename'])
             core.set_previewbridge_image(unique_id, path, res_image[0])
             core.preview_bridge_image_id_map[image] = (path, res_image[0])
@@ -300,7 +327,16 @@ class PreviewBridgeLatent:
 
             res_latent = latent
 
+        if block and is_empty_mask and core.is_execution_model_version_supported:
+            from comfy_execution.graph import ExecutionBlocker
+            result = ExecutionBlocker(None), ExecutionBlocker(None)
+        elif block and is_empty_mask:
+            print(f"[Impact Pack] PreviewBridgeLatent: ComfyUI is outdated - blocking feature is disabled.")
+            result = res_latent, mask
+        else:
+            result = res_latent, mask
+
         return {
             "ui": {"images": res_image},
-            "result": (res_latent, mask, ),
+            "result": result,
         }

modules/impact/config.py

@@ -1,7 +1,7 @@
 import configparser
 import os
 
-version_code = [7, 3, 1]
+version_code = [7, 5, 1]
 version = f"V{version_code[0]}.{version_code[1]}" + (f'.{version_code[2]}' if len(version_code) > 2 else '')
 
 dependency_version = 22

modules/impact/core.py

@@ -237,7 +237,7 @@ def enhance_detail(image, model, clip, vae, guide_size, guide_size_for_bbox, max
         noise_mask = utils.tensor_gaussian_blur_mask(noise_mask, noise_mask_feather)
         noise_mask = noise_mask.squeeze(3)
 
-        if noise_mask_feather > 0:
+        if noise_mask_feather > 0 and 'denoise_mask_function' not in model.model_options:
             model = nodes_differential_diffusion.DifferentialDiffusion().apply(model)[0]
 
     if wildcard_opt is not None and wildcard_opt != "":
@@ -383,7 +383,7 @@ def enhance_detail_for_animatediff(image_frames, model, clip, vae, guide_size, g
         noise_mask = utils.tensor_gaussian_blur_mask(noise_mask, noise_mask_feather)
         noise_mask = noise_mask.squeeze(3)
 
-    if noise_mask_feather > 0:
+    if noise_mask_feather > 0 and 'denoise_mask_function' not in model.model_options:
         model = nodes_differential_diffusion.DifferentialDiffusion().apply(model)[0]
 
     if wildcard_opt is not None and wildcard_opt != "":

modules/impact/hook_nodes.py

@@ -78,6 +78,8 @@ class PreviewDetailerHookProvider:
 
     CATEGORY = "ImpactPack/Util"
 
+    NOT_IDEMPOTENT = True
+
     def doit(self, quality, unique_id):
         hook = hooks.PreviewDetailerHook(unique_id, quality)
-        return (hook, hook)
+        return hook, hook

modules/impact/impact_pack.py

@@ -29,6 +29,14 @@ import impact.wildcards as wildcards
 from . import hooks
 from . import utils
 
+
+try:
+    from comfy_extras import nodes_differential_diffusion
+except Exception:
+    print(f"\n#############################################\n[Impact Pack] ComfyUI is an outdated version.\n#############################################\n")
+    raise Exception("[Impact Pack] ComfyUI is an outdated version.")
+
+
 warnings.filterwarnings('ignore', category=UserWarning, message='TypedStorage is deprecated')
 
 model_path = folder_paths.models_dir
@@ -62,10 +70,10 @@ class CLIPSegDetectorProvider:
     @classmethod
     def INPUT_TYPES(s):
         return {"required": {
-                        "text": ("STRING", {"multiline": False}),
-                        "blur": ("FLOAT", {"min": 0, "max": 15, "step": 0.1, "default": 7}),
-                        "threshold": ("FLOAT", {"min": 0, "max": 1, "step": 0.05, "default": 0.4}),
-                        "dilation_factor": ("INT", {"min": 0, "max": 10, "step": 1, "default": 4}),
+                        "text": ("STRING", {"multiline": False, "tooltip": "Enter the targets to be detected, separated by commas"}),
+                        "blur": ("FLOAT", {"min": 0, "max": 15, "step": 0.1, "default": 7, "tooltip": "Blurs the detected mask"}),
+                        "threshold": ("FLOAT", {"min": 0, "max": 1, "step": 0.05, "default": 0.4, "tooltip": "Detects only areas that are certain above the threshold."}),
+                        "dilation_factor": ("INT", {"min": 0, "max": 10, "step": 1, "default": 4, "tooltip": "Dilates the detected mask."}),
                     }
                 }
 
@@ -74,6 +82,8 @@ class CLIPSegDetectorProvider:
 
     CATEGORY = "ImpactPack/Util"
 
+    DESCRIPTION = "Provides a detection function using CLIPSeg, which generates masks based on text prompts.\nTo use this node, the CLIPSeg custom node must be installed."
+
     def doit(self, text, blur, threshold, dilation_factor):
         if "CLIPSeg" in nodes.NODE_CLASS_MAPPINGS:
             return (core.BBoxDetectorBasedOnCLIPSeg(text, blur, threshold, dilation_factor), )
@@ -87,8 +97,10 @@ class SAMLoader:
         models = [x for x in folder_paths.get_filename_list("sams") if 'hq' not in x]
         return {
             "required": {
-                "model_name": (models + ['ESAM'], ),
-                "device_mode": (["AUTO", "Prefer GPU", "CPU"],),
+                "model_name": (models + ['ESAM'], {"tooltip": "The detection accuracy varies depending on the SAM model. ESAM can only be used if ComfyUI-YoloWorld-EfficientSAM is installed."}),
+                "device_mode": (["AUTO", "Prefer GPU", "CPU"], {"tooltip": "AUTO: Only applicable when a GPU is available. It temporarily loads the SAM_MODEL into VRAM only when the detection function is used.\n"
+                                                                           "Prefer GPU: Tries to keep the SAM_MODEL on the GPU whenever possible. This can be used when there is sufficient VRAM available.\n"
+                                                                           "CPU: Always loads only on the CPU."}),
             }
         }
 
@@ -97,6 +109,8 @@ class SAMLoader:
 
     CATEGORY = "ImpactPack"
 
+    DESCRIPTION = "Load the SAM (Segment Anything) model. This can be used in places that utilize SAM detection functionality, such as SAMDetector or SimpleDetector.\nThe SAM detection functionality in Impact Pack must use the SAM_MODEL loaded through this node."
+
     def load_model(self, model_name, device_mode="auto"):
         if model_name == 'ESAM':
             if 'ESAM_ModelLoader_Zho' not in nodes.NODE_CLASS_MAPPINGS:
@@ -251,6 +265,9 @@ class DetailerForEach:
         else:
             ordered_segs = segs[1]
 
+        if noise_mask_feather > 0 and 'denoise_mask_function' not in model.model_options:
+            model = nodes_differential_diffusion.DifferentialDiffusion().apply(model)[0]
+
         for i, seg in enumerate(ordered_segs):
             cropped_image = crop_ndarray4(image.cpu().numpy(), seg.crop_region)  # Never use seg.cropped_image to handle overlapping area
             cropped_image = to_tensor(cropped_image)
@@ -1640,6 +1657,8 @@ class BitwiseAndMaskForEach:
 
     CATEGORY = "ImpactPack/Operation"
 
+    DESCRIPTION = "Retains only the overlapping areas between the masks included in base_segs and the mask regions of mask_segs. SEGS with no overlapping mask areas are filtered out."
+
     def doit(self, base_segs, mask_segs):
         mask = core.segs_to_combined_mask(mask_segs)
         mask = make_3d_mask(mask)
@@ -1661,6 +1680,8 @@ class SubtractMaskForEach:
 
     CATEGORY = "ImpactPack/Operation"
 
+    DESCRIPTION = "Removes only the overlapping areas between the masks included in base_segs and the mask regions of mask_segs. SEGS with no overlapping mask areas are filtered out."
+
     def doit(self, base_segs, mask_segs):
         mask = core.segs_to_combined_mask(mask_segs)
         mask = make_3d_mask(mask)
@@ -1686,6 +1707,25 @@ class ToBinaryMask:
         return (mask,)
 
 
+class FlattenMask:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {
+                        "masks": ("MASK",),
+                    }
+                }
+
+    RETURN_TYPES = ("MASK",)
+    FUNCTION = "doit"
+
+    CATEGORY = "ImpactPack/Operation"
+
+    def doit(self, masks):
+        masks = utils.make_3d_mask(masks)
+        masks = utils.flatten_mask(masks)
+        return (masks,)
+
+
 class BitwiseAndMask:
     @classmethod
     def INPUT_TYPES(s):

modules/impact/segs_nodes.py

@@ -14,6 +14,13 @@ from comfy.cli_args import args
 import math
 
 
+try:
+    from comfy_extras import nodes_differential_diffusion
+except Exception:
+    print(f"\n#############################################\n[Impact Pack] ComfyUI is an outdated version.\n#############################################\n")
+    raise Exception("[Impact Pack] ComfyUI is an outdated version.")
+
+
 class SEGSDetailer:
     @classmethod
     def INPUT_TYPES(s):
@@ -69,6 +76,9 @@ class SEGSDetailer:
         new_segs = []
         cnet_pil_list = []
 
+        if noise_mask_feather > 0 and 'denoise_mask_function' not in model.model_options:
+            model = nodes_differential_diffusion.DifferentialDiffusion().apply(model)[0]
+
         for i in range(batch_size):
             seed += 1
             for seg in segs[1]:

modules/impact/segs_upscaler.py

@@ -98,7 +98,7 @@ def img2img_segs(image, model, clip, vae, seed, steps, cfg, sampler_name, schedu
         noise_mask = tensor_gaussian_blur_mask(noise_mask, noise_mask_feather)
         noise_mask = noise_mask.squeeze(3)
 
-        if noise_mask_feather > 0:
+        if noise_mask_feather > 0 and 'denoise_mask_function' not in model.model_options:
             model = nodes_differential_diffusion.DifferentialDiffusion().apply(model)[0]
 
     if control_net_wrapper is not None:

modules/impact/util_nodes.py

@@ -392,6 +392,26 @@ class ImageBatchToImageList:
         return (images, )
 
 
+class MakeMaskList:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {"mask1": ("MASK",), }}
+
+    RETURN_TYPES = ("MASK",)
+    OUTPUT_IS_LIST = (True,)
+    FUNCTION = "doit"
+
+    CATEGORY = "ImpactPack/Util"
+
+    def doit(self, **kwargs):
+        masks = []
+
+        for k, v in kwargs.items():
+            masks.append(v)
+
+        return (masks, )
+
+
 class MakeImageList:
     @classmethod
     def INPUT_TYPES(s):
@@ -437,6 +457,31 @@ class MakeImageBatch:
             return (image1,)
 
 
+class MakeMaskBatch:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {"mask1": ("MASK",), }}
+
+    RETURN_TYPES = ("MASK",)
+    FUNCTION = "doit"
+
+    CATEGORY = "ImpactPack/Util"
+
+    def doit(self, **kwargs):
+        mask1 = kwargs['mask1']
+        del kwargs['mask1']
+        masks = [utils.make_3d_mask(value) for value in kwargs.values()]
+
+        if len(masks) == 0:
+            return (mask1,)
+        else:
+            for mask2 in masks:
+                if mask1.shape[1:] != mask2.shape[1:]:
+                    mask2 = comfy.utils.common_upscale(mask2.movedim(-1, 1), mask1.shape[2], mask1.shape[1], "lanczos", "center").movedim(1, -1)
+                mask1 = torch.cat((mask1, mask2), dim=0)
+            return (mask1,)
+
+
 class ReencodeLatent:
     @classmethod
     def INPUT_TYPES(s):

modules/impact/utils.py

@@ -5,8 +5,7 @@ import numpy as np
 import folder_paths
 import nodes
 from . import config
-from PIL import Image, ImageFilter
-from scipy.ndimage import zoom
+from PIL import Image
 import comfy
 
 
@@ -579,6 +578,14 @@ def apply_mask_alpha_to_pil(decoded_pil, mask):
     return decoded_rgba
 
 
+def flatten_mask(all_masks):
+    merged_mask = (all_masks[0] * 255).to(torch.uint8)
+    for mask in all_masks[1:]:
+        merged_mask |= (mask * 255).to(torch.uint8)
+
+    return merged_mask
+
+
 def try_install_custom_node(custom_node_url, msg):
     try:
         import cm_global

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "comfyui-impact-pack"
 description = "This extension offers various detector nodes and detailer nodes that allow you to configure a workflow that automatically enhances facial details. And provide iterative upscaler."
-version = "7.3.1"
+version = "7.5.1"
 license = { file = "LICENSE.txt" }
 dependencies = ["segment-anything", "scikit-image", "piexif", "transformers", "opencv-python-headless", "GitPython", "scipy>=1.11.4"]
 

requirements.txt

@@ -6,3 +6,4 @@ opencv-python-headless
 GitPython
 scipy>=1.11.4
 numpy<2
+dill