feat: LamaRemoverDetailerHookProvider is added

- If everything is working as expected, this conflict shouldn't occur. However, some node pack might be adding the `impact` module to `sys.path`.

https://github.com/ltdrdata/ComfyUI-Impact-Pack/issues/1022

README.md

@@ -189,6 +189,10 @@ NOTE: The UltralyticsDetectorProvider node is not part of the ComfyUI-Impact-Pac
   * `PreviewDetailerHook` - Connecting this hook node helps provide assistance for viewing previews whenever SEGS Detailing tasks are completed. When working with a large number of SEGS, such as Make Tile SEGS, it allows for monitoring the situation as improvements progress incrementally.
     * Since this is the hook applied when pasting onto the original image, it has no effect on nodes like `SEGSDetailer`.
   * `VariationNoiseDetailerHookProvider` - Apply variation seed to the detailer. It can be applied in multiple stages through combine.
+  * `CustomSamplerDetailerHookProvider` - Apply a hook that allows you to use a custom sampler in the Detailer nodes. When using `DetailerHookCombine`, the sampler from the first hook is applied.
+  * `LamaRemoverDetailerHookProvider` – Applies Lama Remover to the upscaled image during the detailing stage. If `skip_sampling` is set to True, Lama Remover can be used alone without the detailing stage, allowing it to simply remove detected regions.
+      * Not applicable for **AnimateDiff** detailers. When using `DetailerHookCombine`, `skip_sampling` is only applied if it is set to `True` for all hooks.
+      * To use this node, the node pack at [Layer-norm/comfyui-lama-remover](https://github.com/Layer-norm/comfyui-lama-remover) must be installed.
 
 ### Iterative Upscale nodes
   * `Iterative Upscale (Latent/on Pixel Space)` - The upscaler takes the input upscaler and splits the scale_factor into steps, then iteratively performs upscaling. 
@@ -488,3 +492,5 @@ BlenderNeok/[ComfyUI_Noise](https://github.com/BlenderNeko/ComfyUI_Noise) - The
 WASasquatch/[was-node-suite-comfyui](https://github.com/WASasquatch/was-node-suite-comfyui) - A powerful custom node extensions of ComfyUI.
 
 Trung0246/[ComfyUI-0246](https://github.com/Trung0246/ComfyUI-0246) - Nice bypass hack!
+
+Layer-norm/[comfyui-lama-remover](https://github.com/Layer-norm/comfyui-lama-remover) - Required for using `LamaRemoverDetailerHook`.

__init__.py

@@ -122,6 +122,8 @@ NODE_CLASS_MAPPINGS = {
     "UnsamplerHookProvider": UnsamplerHookProvider,
     "CoreMLDetailerHookProvider": CoreMLDetailerHookProvider,
     "PreviewDetailerHookProvider": PreviewDetailerHookProvider,
+    "CustomSamplerDetailerHookProvider": CustomSamplerDetailerHookProvider,
+    "LamaRemoverDetailerHookProvider": LamaRemoverDetailerHookProvider,
 
     "DetailerHookCombine": DetailerHookCombine,
     "NoiseInjectionDetailerHookProvider": NoiseInjectionDetailerHookProvider,

modules/impact/config.py

@@ -1,7 +1,7 @@
 import configparser
 import os
 
-version_code = [8, 15, 3]
+version_code = [8, 17]
 version = f"V{version_code[0]}.{version_code[1]}" + (f'.{version_code[2]}' if len(version_code) > 2 else '')
 
 dependency_version = 24

modules/impact/core.py

@@ -320,6 +320,9 @@ def enhance_detail(image, model, clip, vae, guide_size, guide_size_for_bbox, max
     # upscale
     upscaled_image = tensor_resize(image, new_w, new_h)
 
+    if detailer_hook is not None:
+        upscaled_image = detailer_hook.post_upscale(upscaled_image, noise_mask)
+
     cnet_pils = None
     if control_net_wrapper is not None:
         positive, negative, cnet_pils = control_net_wrapper.apply(positive, negative, upscaled_image, noise_mask)
@@ -327,61 +330,69 @@ def enhance_detail(image, model, clip, vae, guide_size, guide_size_for_bbox, max
         cnet_pils.extend(cnet_pils2)
 
     # prepare mask
-    if noise_mask is not None and inpaint_model:
-        imc_encode = nodes.InpaintModelConditioning().encode
-        if 'noise_mask' in inspect.signature(imc_encode).parameters:
-            positive, negative, latent_image = imc_encode(positive, negative, upscaled_image, vae, mask=noise_mask, noise_mask=True)
+    if detailer_hook is None or not detailer_hook.get_skip_sampling():
+        if noise_mask is not None and inpaint_model:
+            imc_encode = nodes.InpaintModelConditioning().encode
+            if 'noise_mask' in inspect.signature(imc_encode).parameters:
+                positive, negative, latent_image = imc_encode(positive, negative, upscaled_image, vae, mask=noise_mask, noise_mask=True)
+            else:
+                print(f"[Impact Pack] ComfyUI is an outdated version.")
+                positive, negative, latent_image = imc_encode(positive, negative, upscaled_image, vae, noise_mask)
         else:
-            print(f"[Impact Pack] ComfyUI is an outdated version.")
-            positive, negative, latent_image = imc_encode(positive, negative, upscaled_image, vae, noise_mask)
-    else:
-        latent_image = to_latent_image(upscaled_image, vae, vae_tiled_encode=vae_tiled_encode)
-        if noise_mask is not None:
-            latent_image['noise_mask'] = noise_mask
+            latent_image = to_latent_image(upscaled_image, vae, vae_tiled_encode=vae_tiled_encode)
+            if noise_mask is not None:
+                latent_image['noise_mask'] = noise_mask
 
-    if detailer_hook is not None:
-        latent_image = detailer_hook.post_encode(latent_image)
-
-    refined_latent = latent_image
-
-    # ksampler
-    for i in range(0, cycle):
         if detailer_hook is not None:
+            latent_image = detailer_hook.post_encode(latent_image)
+
+        refined_latent = latent_image
+
+        sampler_opt=None
+        if detailer_hook is not None:
+            sampler_opt = detailer_hook.get_custom_sampler()
+
+        # ksampler
+        for i in range(0, cycle):
             if detailer_hook is not None:
-                detailer_hook.set_steps((i, cycle))
+                if detailer_hook is not None:
+                    detailer_hook.set_steps((i, cycle))
 
-            refined_latent = detailer_hook.cycle_latent(refined_latent)
+                refined_latent = detailer_hook.cycle_latent(refined_latent)
 
-            model2, seed2, steps2, cfg2, sampler_name2, scheduler2, positive2, negative2, upscaled_latent2, denoise2 = \
-                detailer_hook.pre_ksample(model, seed+i, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise)
-            noise, is_touched = detailer_hook.get_custom_noise(seed+i, torch.zeros(latent_image['samples'].size()), is_touched=False)
-            if not is_touched:
+                model2, seed2, steps2, cfg2, sampler_name2, scheduler2, positive2, negative2, upscaled_latent2, denoise2 = \
+                    detailer_hook.pre_ksample(model, seed+i, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise)
+                noise, is_touched = detailer_hook.get_custom_noise(seed+i, torch.zeros(latent_image['samples'].size()), is_touched=False)
+                if not is_touched:
+                    noise = None
+            else:
+                model2, seed2, steps2, cfg2, sampler_name2, scheduler2, positive2, negative2, upscaled_latent2, denoise2 = \
+                    model, seed + i, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise
                 noise = None
+
+            refined_latent = impact_sampling.ksampler_wrapper(model2, seed2, steps2, cfg2, sampler_name2, scheduler2, positive2, negative2,
+                                                              refined_latent, denoise2, refiner_ratio, refiner_model, refiner_clip, refiner_positive, refiner_negative,
+                                                              noise=noise, scheduler_func=scheduler_func, sampler_opt=sampler_opt)
+
+        if detailer_hook is not None:
+            refined_latent = detailer_hook.pre_decode(refined_latent)
+
+        # non-latent downscale - latent downscale cause bad quality
+        start = time.time()
+        if vae_tiled_decode:
+            (refined_image,) = nodes.VAEDecodeTiled().decode(vae, refined_latent, 512) # using default settings
+            print(f"[Impact Pack] vae decoded (tiled) in {time.time() - start:.1f}s")
         else:
-            model2, seed2, steps2, cfg2, sampler_name2, scheduler2, positive2, negative2, upscaled_latent2, denoise2 = \
-                model, seed + i, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise
-            noise = None
-
-        refined_latent = impact_sampling.ksampler_wrapper(model2, seed2, steps2, cfg2, sampler_name2, scheduler2, positive2, negative2,
-                                                          refined_latent, denoise2, refiner_ratio, refiner_model, refiner_clip, refiner_positive, refiner_negative,
-                                                          noise=noise, scheduler_func=scheduler_func)
-
-    if detailer_hook is not None:
-        refined_latent = detailer_hook.pre_decode(refined_latent)
-
-    # non-latent downscale - latent downscale cause bad quality
-    start = time.time()
-    if vae_tiled_decode:
-        (refined_image,) = nodes.VAEDecodeTiled().decode(vae, refined_latent, 512) # using default settings
-        print(f"[Impact Pack] vae decoded (tiled) in {time.time() - start:.1f}s")
+            try:
+                refined_image = vae.decode(refined_latent['samples'])
+            except Exception as e:
+                # usually an out-of-memory exception from the decode, so try a tiled approach
+                print(f"[Impact Pack] failed after {time.time() - start:.1f}s, doing vae.decode_tiled 64...")
+                refined_image = vae.decode_tiled(refined_latent["samples"], tile_x=64, tile_y=64, )
+            print(f"[Impact Pack] vae decoded in {time.time() - start:.1f}s")
     else:
-        try:
-            refined_image = vae.decode(refined_latent['samples'])
-        except Exception as e:
-            # usually an out-of-memory exception from the decode, so try a tiled approach
-            print(f"[Impact Pack] failed after {time.time() - start:.1f}s, doing vae.decode_tiled 64...")
-            refined_image = vae.decode_tiled(refined_latent["samples"], tile_x=64, tile_y=64, )
-        print(f"[Impact Pack] vae decoded in {time.time() - start:.1f}s")
+        # skipped
+        refined_image = upscaled_image
 
     if detailer_hook is not None:
         refined_image = detailer_hook.post_decode(refined_image)
@@ -513,11 +524,16 @@ def enhance_detail_for_animatediff(image_frames, model, clip, vae, guide_size, g
         'samples': latent_frames
     }
 
+
+    sampler_opt=None
+    if detailer_hook is not None:
+        sampler_opt = detailer_hook.get_custom_sampler()
+
     if detailer_hook is not None:
         latent = detailer_hook.post_encode(latent)
 
     refined_latent = impact_sampling.ksampler_wrapper(model, seed, steps, cfg, sampler_name, scheduler, positive, negative,
-                                                      latent, denoise, refiner_ratio, refiner_model, refiner_clip, refiner_positive, refiner_negative, scheduler_func=scheduler_func)
+                                                      latent, denoise, refiner_ratio, refiner_model, refiner_clip, refiner_positive, refiner_negative, scheduler_func=scheduler_func, sampler_opt=sampler_opt)
 
     if detailer_hook is not None:
         refined_latent = detailer_hook.pre_decode(refined_latent)
@@ -1077,7 +1093,7 @@ class ONNXDetector:
     def detect(self, image, threshold, dilation, crop_factor, drop_size=1, detailer_hook=None):
         drop_size = max(drop_size, 1)
         try:
-            import impact.onnx as onnx
+            import impact.impact_onnx as onnx
 
             h = image.shape[1]
             w = image.shape[2]

modules/impact/detectors.py

@@ -163,7 +163,7 @@ class SegmDetectorCombined:
         mask = segm_detector.detect_combined(image, threshold, dilation)
 
         if mask is None:
-            mask = torch.zeros((image.shape[2], image.shape[1]), dtype=torch.float32, device="cpu")
+            mask = torch.zeros((image.shape[1], image.shape[2]), dtype=torch.float32, device="cpu")
 
         return (mask.unsqueeze(0),)
 
@@ -183,7 +183,7 @@ class BboxDetectorCombined(SegmDetectorCombined):
         mask = bbox_detector.detect_combined(image, threshold, dilation)
 
         if mask is None:
-            mask = torch.zeros((image.shape[2], image.shape[1]), dtype=torch.float32, device="cpu")
+            mask = torch.zeros((image.shape[1], image.shape[2]), dtype=torch.float32, device="cpu")
 
         return (mask.unsqueeze(0),)
 

modules/impact/hook_nodes.py

@@ -1,6 +1,8 @@
 import sys
 from . import hooks
 from . import defs
+from . import utils
+import nodes
 
 
 class SEGSOrderedFilterDetailerHookProvider:
@@ -83,3 +85,24 @@ class PreviewDetailerHookProvider:
     def doit(self, quality, unique_id):
         hook = hooks.PreviewDetailerHook(unique_id, quality)
         return hook, hook
+
+
+class LamaRemoverDetailerHookProvider:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {
+            "required": {
+                "mask_threshold":("INT", {"default": 250, "min": 0, "max": 255, "step": 1, "display": "slider"}),
+                "gaussblur_radius": ("INT", {"default": 8, "min": 0, "max": 20, "step": 1, "display": "slider"}),
+                "skip_sampling": ("BOOLEAN", {"default": True}),
+            }
+        }
+
+    RETURN_TYPES = ("DETAILER_HOOK", )
+    FUNCTION = "doit"
+
+    CATEGORY = "ImpactPack/Util"
+
+    def doit(self, mask_threshold, gaussblur_radius, skip_sampling):
+        hook = hooks.LamaRemoverDetailerHook(mask_threshold, gaussblur_radius, skip_sampling)
+        return (hook, )

modules/impact/hooks.py

@@ -25,7 +25,7 @@ class PixelKSampleHook:
     def post_decode(self, pixels):
         return pixels
 
-    def post_upscale(self, pixels):
+    def post_upscale(self, pixels, mask=None):
         return pixels
 
     def post_encode(self, samples):
@@ -64,8 +64,8 @@ class PixelKSampleHookCombine(PixelKSampleHook):
     def post_decode(self, pixels):
         return self.hook2.post_decode(self.hook1.post_decode(pixels))
 
-    def post_upscale(self, pixels):
-        return self.hook2.post_upscale(self.hook1.post_upscale(pixels))
+    def post_upscale(self, pixels, mask=None):
+        return self.hook2.post_upscale(self.hook1.post_upscale(pixels, mask), mask)
 
     def post_encode(self, samples):
         return self.hook2.post_encode(self.hook1.post_encode(samples))
@@ -109,6 +109,15 @@ class DetailerHookCombine(PixelKSampleHookCombine):
         noise_2nd, is_touched = self.hook2.get_custom_noise(seed, noise, is_touched)
         return noise, is_touched
 
+    def get_custom_sampler(self):
+        if self.hook1.get_custom_sampler() is not None:
+            return self.hook1.get_custom_sampler()
+        else:
+            return self.hook2.get_custom_sampler()
+
+    def get_skip_sampling(self):
+        return self.hook1.get_skip_sampling() and self.hook2.get_skip_sampling()
+
 
 class SimpleCfgScheduleHook(PixelKSampleHook):
     target_cfg = 0
@@ -173,6 +182,21 @@ class DetailerHook(PixelKSampleHook):
     def get_custom_noise(self, seed, noise, is_touched):
         return noise, is_touched
 
+    def get_custom_sampler(self):
+        return None
+
+    def get_skip_sampling(self):
+        return False
+
+
+class CustomSamplerDetailerHookProvider(DetailerHook):
+    def __init__(self, sampler):
+        super().__init__()
+        self.sampler = sampler
+
+    def get_custom_sampler(self):
+        return self.sampler
+
 
 # class CustomNoiseDetailerHookProvider(DetailerHook):
 #     def __init__(self, noise):
@@ -486,6 +510,27 @@ class SEGSLabelFilterDetailerHook(DetailerHook):
         return segs_nodes.SEGSLabelFilter().doit(segs, "", self.labels)[0]
 
 
+class LamaRemoverDetailerHook(DetailerHook):
+    def __init__(self, mask_threshold, gaussblur_radius, skip_sampling):
+        super().__init__()
+        self.mask_threshold = mask_threshold
+        self.gaussblur_radius = gaussblur_radius
+        self.skip_sampling = skip_sampling
+
+    def post_upscale(self, img, mask=None):
+        if "LamaRemover" in nodes.NODE_CLASS_MAPPINGS:
+            lama_remover_obj = nodes.NODE_CLASS_MAPPINGS['LamaRemover']()
+        else:
+            utils.try_install_custom_node('https://github.com/Layer-norm/comfyui-lama-remover',
+                                          "To use 'LAMARemoverDetailerHookProvider', 'comfyui-lama-remover' nodepack is required.")
+            raise Exception("'LamaRemover' node is not installed.")
+
+        return lama_remover_obj.lama_remover(img, masks=mask, mask_threshold=self.mask_threshold, gaussblur_radius=self.gaussblur_radius, invert_mask=False)[0]
+
+    def get_skip_sampling(self):
+        return self.skip_sampling
+
+
 class PreviewDetailerHook(DetailerHook):
     def __init__(self, node_id, quality):
         super().__init__()

modules/impact/impact_pack.py

@@ -811,6 +811,26 @@ class CoreMLDetailerHookProvider:
         return (hook, )
 
 
+class CustomSamplerDetailerHookProvider:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {
+                    "sampler": ("SAMPLER", ),
+                    },
+                }
+
+    RETURN_TYPES = ("DETAILER_HOOK",)
+    FUNCTION = "doit"
+
+    CATEGORY = "ImpactPack/Detailer"
+    
+    DESCRIPTION = "Apply a hook that allows you to use a custom sampler in the Detailer nodes. When using `DetailerHookCombine`, the sampler from the first hook is applied."
+
+    def doit(self, sampler):
+        hook = hooks.CustomSamplerDetailerHookProvider(sampler)
+        return (hook, )
+
+
 class CfgScheduleHookProvider:
     schedules = ["simple"]
 

modules/impact/impact_sampling.py

@@ -194,7 +194,7 @@ def impact_sample(model, seed, steps, cfg, sampler_name, scheduler, positive, ne
 
 
 def ksampler_wrapper(model, seed, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise,
-                     refiner_ratio=None, refiner_model=None, refiner_clip=None, refiner_positive=None, refiner_negative=None, sigma_factor=1.0, noise=None, scheduler_func=None):
+                     refiner_ratio=None, refiner_model=None, refiner_clip=None, refiner_positive=None, refiner_negative=None, sigma_factor=1.0, noise=None, scheduler_func=None, sampler_opt=None):
 
     if refiner_ratio is None or refiner_model is None or refiner_clip is None or refiner_positive is None or refiner_negative is None:
         # Use separated_sample instead of KSampler for `AYS scheduler`
@@ -206,7 +206,7 @@ def ksampler_wrapper(model, seed, steps, cfg, sampler_name, scheduler, positive,
 
         refined_latent = separated_sample(model, True, seed, advanced_steps, cfg, sampler_name, scheduler,
                                           positive, negative, latent_image, start_at_step, end_at_step, False,
-                                          sigma_ratio=sigma_factor, noise=noise, scheduler_func=scheduler_func)
+                                          sigma_ratio=sigma_factor, sampler_opt=sampler_opt, noise=noise, scheduler_func=scheduler_func)
     else:
         advanced_steps = math.floor(steps / denoise)
         start_at_step = advanced_steps - steps
@@ -215,7 +215,7 @@ def ksampler_wrapper(model, seed, steps, cfg, sampler_name, scheduler, positive,
         # print(f"pre: {start_at_step} .. {end_at_step} / {advanced_steps}")
         temp_latent = separated_sample(model, True, seed, advanced_steps, cfg, sampler_name, scheduler,
                                        positive, negative, latent_image, start_at_step, end_at_step, True,
-                                       sigma_ratio=sigma_factor, noise=noise, scheduler_func=scheduler_func)
+                                       sigma_ratio=sigma_factor, sampler_opt=sampler_opt, noise=noise, scheduler_func=scheduler_func)
 
         if 'noise_mask' in latent_image:
             # noise_latent = \
@@ -229,7 +229,7 @@ def ksampler_wrapper(model, seed, steps, cfg, sampler_name, scheduler, positive,
         # print(f"post: {end_at_step} .. {advanced_steps + 1} / {advanced_steps}")
         refined_latent = separated_sample(refiner_model, False, seed, advanced_steps, cfg, sampler_name, scheduler,
                                           refiner_positive, refiner_negative, temp_latent, end_at_step, advanced_steps + 1, False,
-                                          sigma_ratio=sigma_factor, scheduler_func=scheduler_func)
+                                          sigma_ratio=sigma_factor, sampler_opt=sampler_opt, scheduler_func=scheduler_func)
 
     return refined_latent
 

modules/impact/wildcards.py

@@ -212,7 +212,7 @@ def process(text, seed=None):
                 selected_items = random_gen.choice(options, p=normalized_probabilities, size=select_count, replace=False)
 
             # x may be numpy.int32, convert to string
-            selected_items2 = [re.sub(r'^\s*[0-9.]+::', '', str(x), 1) for x in selected_items]
+            selected_items2 = [re.sub(r'^\s*[0-9.]+::', '', str(x), count=1) for x in selected_items]
             replacement = select_sep.join(selected_items2)
             if '::' in replacement:
                 pass
@@ -279,7 +279,7 @@ def process(text, seed=None):
 
                 normalized_probabilities = [prob / total_prob for prob in adjusted_probabilities]
                 selected_item = random_gen.choice(options, p=normalized_probabilities, replace=False)
-                replacement = re.sub(r'^\s*[0-9.]+::', '', selected_item, 1)
+                replacement = re.sub(r'^\s*[0-9.]+::', '', selected_item, count=1)
                 replacements_found = True
                 string = string.replace(f"__{match}__", replacement, 1)
             elif '*' in keyword:

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "comfyui-impact-pack"
 description = "This node pack offers various detector nodes and detailer nodes that allow you to configure a workflow that automatically enhances facial details. And provide iterative upscaler."
-version = "8.15.3"
+version = "8.17"
 license = { file = "LICENSE.txt" }
 dependencies = ["segment-anything", "scikit-image", "piexif", "transformers", "opencv-python-headless", "GitPython", "scipy>=1.11.4"]