fix: impact_sampling - potential device mismatch

Co-authored-by: snomiao <snomiao+comfy-pr@gmail.com>

README.md

@@ -7,6 +7,7 @@ This custom node helps to conveniently enhance images through Detector, Detailer
 
 
 ## NOTICE 
+* V6.0: Supports FLUX.1 model in Impact KSampler, Detailers, PreviewBridgeLatent
 * V5.0: It is no longer compatible with versions of ComfyUI before 2024.04.08. 
 * V4.87.4: Update to a version of ComfyUI after 2024.04.08 for proper functionality.
 * V4.85: Incompatible with the outdated **ComfyUI IPAdapter Plus**. (A version dated March 24th or later is required.)
@@ -189,7 +190,6 @@ This custom node helps to conveniently enhance images through Detector, Detailer
     * Furthermore, LatentSender is implemented with PreviewLatent, which stores the latent in payload form within the image thumbnail.
     * Due to the current structure of ComfyUI, it is unable to distinguish between SDXL latent and SD1.5/SD2.1 latent. Therefore, it generates thumbnails by decoding them using the SD1.5 method.
 
-
 ### Switch nodes
   * `Switch (image,mask)`, `Switch (latent)`, `Switch (SEGS)` - Among multiple inputs, it selects the input designated by the selector and outputs it. The first input must be provided, while the others are optional. However, if the input specified by the selector is not connected, an error may occur.
   * `Switch (Any)` - This is a Switch node that takes an arbitrary number of inputs and produces a single output. Its type is determined when connected to any node, and connecting inputs increases the available slots for connections.
@@ -220,6 +220,7 @@ This custom node helps to conveniently enhance images through Detector, Detailer
   * `RegionalSamplerAdvanced` - This is the Advanced version of the RegionalSampler. You can control it using `step` instead of `denoise`.
     > NOTE: The `sde` sampler and `uni_pc` sampler introduce additional noise during each step of the sampling process. To mitigate this, when sampling each region, the `uni_pc` sampler applies additional `dpmpp_fast`, and the sde sampler applies the `dpmpp_2m` sampler as an additional measure.
 
+
 ### Impact KSampler
   * These samplers support basic_pipe and AYS scheduler
   * `KSampler (pipe)` - pipe version of KSampler
@@ -278,6 +279,7 @@ This custom node helps to conveniently enhance images through Detector, Detailer
   * `String Selector` - It selects and returns a portion of the string. When `multiline` mode is disabled, it simply returns the string of the line pointed to by the selector. When `multiline` mode is enabled, it divides the string based on lines that start with `#` and returns them. If the `select` value is larger than the number of items, it will start counting from the first line again and return accordingly.
   * `Combine Conditionings` - It takes multiple conditionings as input and combines them into a single conditioning.
   * `Concat Conditionings` - It takes multiple conditionings as input and concat them into a single conditioning.
+  * `Negative Cond Placeholder` - Models like FLUX.1 do not use Negative Conditioning. This is a placeholder node for them. You can use FLUX.1 by replacing the Negative Conditioning used in Impact KSampler, KSampler (Inspire), and Detailer with this node.
 
 
 ## MMDet nodes (DEPRECATED) - Don't use these nodes

__init__.py

@@ -208,6 +208,8 @@ NODE_CLASS_MAPPINGS = {
     "KSamplerAdvancedProvider": KSamplerAdvancedProvider,
     "TwoAdvancedSamplersForMask": TwoAdvancedSamplersForMask,
 
+    "ImpactNegativeConditioningPlaceholder": NegativeConditioningPlaceholder,
+
     "PreviewBridge": PreviewBridge,
     "PreviewBridgeLatent": PreviewBridgeLatent,
     "ImageSender": ImageSender,
@@ -433,6 +435,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
 
     "ImpactSchedulerAdapter": "Impact Scheduler Adapter",
     "GITSSchedulerFuncProvider": "GITSScheduler Func Provider",
+    "ImpactNegativeConditioningPlaceholder": "Negative Cond Placeholder"
 }
 
 if not impact.config.get_config()['mmdet_skip']:

modules/impact/bridge_nodes.py

@@ -145,6 +145,9 @@ def decode_latent(latent, preview_method, vae_opt=None):
     elif preview_method == "Latent2RGB-SC-B":
         latent_format = latent_formats.SC_B()
         method = LatentPreviewMethod.Latent2RGB
+    elif preview_method == "Latent2RGB-FLUX.1":
+        latent_format = latent_formats.Flux()
+        method = LatentPreviewMethod.Latent2RGB
     else:
         print(f"[Impact Pack] PreviewBridgeLatent: '{preview_method}' is unsupported preview method.")
         latent_format = latent_formats.SD15()
@@ -169,6 +172,7 @@ class PreviewBridgeLatent:
                     "preview_method": (["Latent2RGB-SD3", "Latent2RGB-SDXL", "Latent2RGB-SD15",
                                         "Latent2RGB-SD-X4", "Latent2RGB-Playground-2.5",
                                         "Latent2RGB-SC-Prior", "Latent2RGB-SC-B",
+                                        "Latent2RGB-FLUX.1",
                                         "TAESD3", "TAESDXL", "TAESD15"],),
                     },
                 "optional": {
@@ -228,11 +232,11 @@ class PreviewBridgeLatent:
 
     def doit(self, latent, image, preview_method, vae_opt=None, unique_id=None):
         latent_channels = latent['samples'].shape[1]
-        preview_method_channels = 16 if 'SD3' in preview_method or 'SC-Prior' in preview_method else 4
+        preview_method_channels = 16 if 'SD3' in preview_method or 'SC-Prior' in preview_method or 'FLUX.1' in preview_method else 4
 
         if vae_opt is None and latent_channels != preview_method_channels:
-            print(f"[PreviewBridgeLatent] The version of latent is not compatible with preview_method.\nSD3, SD1/SD2, SDXL, SC-Prior, and SC-B are not compatible with each other.")
-            raise Exception("The version of latent is not compatible with preview_method.<BR>SD3, SD1/SD2, SDXL, SC-Prior, and SC-B are not compatible with each other.")
+            print(f"[PreviewBridgeLatent] The version of latent is not compatible with preview_method.\nSD3, SD1/SD2, SDXL, SC-Prior, SC-B and FLUX.1 are not compatible with each other.")
+            raise Exception("The version of latent is not compatible with preview_method.<BR>SD3, SD1/SD2, SDXL, SC-Prior, SC-B and FLUX.1 are not compatible with each other.")
 
         need_refresh = False
 

modules/impact/config.py

@@ -1,7 +1,7 @@
 import configparser
 import os
 
-version_code = [5, 18, 16]
+version_code = [6, 0, 2]
 version = f"V{version_code[0]}.{version_code[1]}" + (f'.{version_code[2]}' if len(version_code) > 2 else '')
 
 dependency_version = 22

modules/impact/impact_pack.py

@@ -278,13 +278,17 @@ class DetailerForEach:
                 for condition, details in positive
             ]
 
-            cropped_negative = [
-                [condition, {
-                    k: core.crop_condition_mask(v, image, seg.crop_region) if k == "mask" else v
-                    for k, v in details.items()
-                }]
-                for condition, details in negative
-            ]
+            if not isinstance(negative, str):
+                cropped_negative = [
+                    [condition, {
+                        k: core.crop_condition_mask(v, image, seg.crop_region) if k == "mask" else v
+                        for k, v in details.items()
+                    }]
+                    for condition, details in negative
+                ]
+            else:
+                # Negative Conditioning is placeholder such as FLUX.1
+                cropped_negative = negative
 
             enhanced_image, cnet_pils = core.enhance_detail(cropped_image, model, clip, vae, guide_size, guide_size_for_bbox, max_size,
                                                             seg.bbox, seg_seed, steps, cfg, sampler_name, scheduler,

modules/impact/impact_sampling.py

@@ -6,10 +6,12 @@ import latent_preview
 import comfy
 import torch
 import math
+import comfy.model_management as mm
 
 
 try:
     from comfy_extras.nodes_custom_sampler import Noise_EmptyNoise, Noise_RandomNoise
+    import node_helpers
 except:
     print(f"\n#############################################\n[Impact Pack] ComfyUI is an outdated version.\n#############################################\n")
     raise Exception("[Impact Pack] ComfyUI is an outdated version.")
@@ -140,7 +142,25 @@ def sample_with_custom_noise(model, add_noise, noise_seed, cfg, positive, negati
         touched_callback = preview_callback
 
     disable_pbar = not comfy.utils.PROGRESS_BAR_ENABLED
-    samples = comfy.sample.sample_custom(model, noise, cfg, sampler, sigmas, positive, negative, latent_image, noise_mask=noise_mask, callback=touched_callback, disable_pbar=disable_pbar, seed=noise_seed)
+
+    if negative != 'NegativePlaceholder':
+        guider = comfy.samplers.CFGGuider(model)
+        guider.set_conds(positive, negative)
+        guider.set_cfg(cfg)
+    else:
+        guider = nodes_custom_sampler.Guider_Basic(model)
+        positive = node_helpers.conditioning_set_values(positive, {"guidance": cfg})
+        guider.set_conds(positive)
+
+    device = mm.get_torch_device()
+
+    noise = noise.to(device)
+    latent_image = latent_image.to(device)
+    if noise_mask is not None:
+        noise_mask = noise_mask.to(device)
+
+    samples = guider.sample(noise, latent_image, sampler, sigmas, denoise_mask=noise_mask, callback=touched_callback, disable_pbar=disable_pbar, seed=noise_seed)
+    samples = samples.to(comfy.model_management.intermediate_device())
 
     out["samples"] = samples
     if "x0" in x0_output:

modules/impact/special_samplers.py

@@ -212,49 +212,15 @@ class TwoAdvancedSamplersForMask:
 
     CATEGORY = "ImpactPack/Sampler"
 
-    @staticmethod
-    def mask_erosion(samples, mask, grow_mask_by):
-        mask = mask.clone()
-
-        w = samples['samples'].shape[3]
-        h = samples['samples'].shape[2]
-
-        mask2 = torch.nn.functional.interpolate(mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])), size=(w, h), mode="bilinear")
-        if grow_mask_by == 0:
-            mask_erosion = mask2
-        else:
-            kernel_tensor = torch.ones((1, 1, grow_mask_by, grow_mask_by))
-            padding = math.ceil((grow_mask_by - 1) / 2)
-
-            mask_erosion = torch.clamp(torch.nn.functional.conv2d(mask2.round(), kernel_tensor, padding=padding), 0, 1)
-
-        return mask_erosion[:, :, :w, :h].round()
-
     @staticmethod
     def doit(seed, steps, denoise, samples, base_sampler, mask_sampler, mask, overlap_factor):
+        regional_prompts = RegionalPrompt().doit(mask=mask, advanced_sampler=mask_sampler)[0]
 
-        inv_mask = torch.where(mask != 1.0, torch.tensor(1.0), torch.tensor(0.0))
-
-        adv_steps = int(steps / denoise)
-        start_at_step = adv_steps - steps
-
-        new_latent_image = samples.copy()
-
-        mask_erosion = TwoAdvancedSamplersForMask.mask_erosion(samples, mask, overlap_factor)
-
-        for i in range(start_at_step, adv_steps):
-            add_noise = "enable" if i == start_at_step else "disable"
-            return_with_leftover_noise = "enable" if i+1 != adv_steps else "disable"
-
-            new_latent_image['noise_mask'] = inv_mask
-            new_latent_image = base_sampler.sample_advanced(add_noise, seed, adv_steps, new_latent_image, i, i + 1, "enable", recovery_mode="ratio additional")
-
-            new_latent_image['noise_mask'] = mask_erosion
-            new_latent_image = mask_sampler.sample_advanced("disable", seed, adv_steps, new_latent_image, i, i + 1, return_with_leftover_noise, recovery_mode="ratio additional")
-
-        del new_latent_image['noise_mask']
-
-        return (new_latent_image, )
+        return RegionalSampler().doit(seed=seed, seed_2nd=0, seed_2nd_mode="ignore", steps=steps, base_only_steps=1,
+                                      denoise=denoise, samples=samples, base_sampler=base_sampler,
+                                      regional_prompts=regional_prompts, overlap_factor=overlap_factor,
+                                      restore_latent=True, additional_mode="ratio between",
+                                      additional_sampler="AUTO", additional_sigma_ratio=0.3)
 
 
 class RegionalPrompt:
@@ -409,7 +375,7 @@ class RegionalSampler:
                      "additional_sampler": (["AUTO", "euler", "heun", "heunpp2", "dpm_2", "dpm_fast", "dpmpp_2m", "ddpm"],),
                      "additional_sigma_ratio": ("FLOAT", {"default": 0.3, "min": 0.0, "max": 1.0, "step": 0.01}),
                      },
-                 "hidden": {"unique_id": "UNIQUE_ID"},
+                "hidden": {"unique_id": "UNIQUE_ID"},
                 }
 
     TOOLTIPS = {
@@ -519,7 +485,8 @@ class RegionalSampler:
             core.update_node_status(unique_id, f"{i}/{steps} steps  |         ", ((i-start_at_step)*region_len)/total)
 
             new_latent_image['noise_mask'] = inv_mask
-            new_latent_image = base_sampler.sample_advanced(add_noise, seed, adv_steps, new_latent_image, i, i + 1, True,
+            new_latent_image = base_sampler.sample_advanced(add_noise, seed, adv_steps, new_latent_image,
+                                                            start_at_step=i, end_at_step=i + 1, return_with_leftover_noise=True,
                                                             recovery_mode=additional_mode, recovery_sampler=additional_sampler, recovery_sigma_ratio=additional_sigma_ratio, noise=noise)
 
             if restore_latent:
@@ -831,7 +798,8 @@ class GITSSchedulerFuncProvider:
 
     FUNCTION = "doit"
 
-    def doit(self, coeff, denoise):
+    @staticmethod
+    def doit(coeff, denoise):
         def f(model, sampler, steps):
             if 'GITSScheduler' not in nodes.NODE_CLASS_MAPPINGS:
                 raise Exception("[Impact Pack] ComfyUI is an outdated version. Cannot use GITSScheduler.")
@@ -840,3 +808,22 @@ class GITSSchedulerFuncProvider:
             return scheduler.get_sigmas(coeff, steps, denoise)[0]
 
         return (f, )
+
+
+class NegativeConditioningPlaceholder:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {}}
+
+    TOOLTIPS = {
+        "output": ("This is a Placeholder for the FLUX model that does not use Negative Conditioning.",)
+    }
+
+    RETURN_TYPES = ("CONDITIONING",)
+    CATEGORY = "ImpactPack/sampling"
+
+    FUNCTION = "doit"
+
+    @staticmethod
+    def doit():
+        return ("NegativePlaceholder", )

pyproject.toml

@@ -1,8 +1,8 @@
 [project]
 name = "comfyui-impact-pack"
 description = "This extension offers various detector nodes and detailer nodes that allow you to configure a workflow that automatically enhances facial details. And provide iterative upscaler."
-version = "5.18.16"
-license = "LICENSE"
+version = "6.0.2"
+license = { file = "LICENSE.txt" }
 dependencies = ["segment-anything", "scikit-image", "piexif", "transformers", "opencv-python-headless", "GitPython", "scipy>=1.11.4"]
 
 [project.urls]