From 1e445bcacf849ed0b16e1942f46ab683764142da Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" Date: Wed, 10 Jan 2024 00:26:26 +0900 Subject: [PATCH] feat: Detailer - support `inpaint_model` support to this PR. https://github.com/comfyanonymous/ComfyUI/pull/2501 --- modules/impact/config.py | 2 +- modules/impact/core.py | 13 +++++--- modules/impact/impact_pack.py | 57 ++++++++++++++++++++++------------- modules/impact/segs_nodes.py | 31 ++++++++++--------- 4 files changed, 62 insertions(+), 41 deletions(-) diff --git a/modules/impact/config.py b/modules/impact/config.py index 4bffd4a..ac09a29 100644 --- a/modules/impact/config.py +++ b/modules/impact/config.py @@ -2,7 +2,7 @@ import configparser import os -version_code = [4, 60, 4] +version_code = [4, 61] version = f"V{version_code[0]}.{version_code[1]}" + (f'.{version_code[2]}' if len(version_code) > 2 else '') dependency_version = 20 diff --git a/modules/impact/core.py b/modules/impact/core.py index 66635d6..373c2c1 100644 --- a/modules/impact/core.py +++ b/modules/impact/core.py @@ -187,7 +187,7 @@ def enhance_detail(image, model, clip, vae, guide_size, guide_size_for_bbox, max wildcard_opt=None, wildcard_opt_concat_mode=None, detailer_hook=None, refiner_ratio=None, refiner_model=None, refiner_clip=None, refiner_positive=None, - refiner_negative=None, control_net_wrapper=None, cycle=1): + refiner_negative=None, control_net_wrapper=None, cycle=1, inpaint_model=False): if noise_mask is not None and len(noise_mask.shape) == 3: noise_mask = noise_mask.squeeze(0) @@ -253,10 +253,13 @@ def enhance_detail(image, model, clip, vae, guide_size, guide_size_for_bbox, max upscaled_image = tensor_resize(image, new_w, new_h) # ksampler - latent_image = to_latent_image(upscaled_image, vae) - upscaled_mask = None - if noise_mask is not None: + if inpaint_model: + noise_mask = torch.from_numpy(noise_mask) + latent_image = nodes.VAEEncodeForInpaint().encode(vae, upscaled_image, noise_mask)[0] + elif noise_mask is not None: + latent_image = to_latent_image(upscaled_image, vae) + # upscale the mask tensor by a factor of 2 using bilinear interpolation noise_mask = torch.from_numpy(noise_mask) upscaled_mask = torch.nn.functional.interpolate(noise_mask.unsqueeze(0).unsqueeze(0), size=(new_h, new_w), mode='bilinear', align_corners=False) @@ -317,7 +320,7 @@ def enhance_detail_for_animatediff(image_frames, model, clip, vae, guide_size, g wildcard_opt=None, wildcard_opt_concat_mode=None, detailer_hook=None, refiner_ratio=None, refiner_model=None, refiner_clip=None, refiner_positive=None, - refiner_negative=None): + refiner_negative=None, inpaint_model=False): if noise_mask is not None and len(noise_mask.shape) == 3: noise_mask = noise_mask.squeeze(0) diff --git a/modules/impact/impact_pack.py b/modules/impact/impact_pack.py index 32424d5..057da24 100644 --- a/modules/impact/impact_pack.py +++ b/modules/impact/impact_pack.py @@ -170,7 +170,10 @@ class DetailerForEach: "cycle": ("INT", {"default": 1, "min": 1, "max": 10, "step": 1}), }, - "optional": {"detailer_hook": ("DETAILER_HOOK",), } + "optional": { + "detailer_hook": ("DETAILER_HOOK",), + "inpaint_model": ("BOOLEAN", {"default": False, "label_on": "enabled", "label_off": "disabled"}), + } } RETURN_TYPES = ("IMAGE", ) @@ -181,7 +184,8 @@ class DetailerForEach: @staticmethod def do_detail(image, segs, model, clip, vae, guide_size, guide_size_for_bbox, max_size, seed, steps, cfg, sampler_name, scheduler, positive, negative, denoise, feather, noise_mask, force_inpaint, wildcard_opt=None, detailer_hook=None, - refiner_ratio=None, refiner_model=None, refiner_clip=None, refiner_positive=None, refiner_negative=None, cycle=1): + refiner_ratio=None, refiner_model=None, refiner_clip=None, refiner_positive=None, refiner_negative=None, + cycle=1, inpaint_model=False): if len(image) > 1: raise Exception('[Impact Pack] ERROR: DetailerForEach does not allow image batches.\nPlease refer to https://github.com/ltdrdata/ComfyUI-extension-tutorials/blob/Main/ComfyUI-Impact-Pack/tutorial/batching-detailer.md for more information.') @@ -241,7 +245,8 @@ class DetailerForEach: detailer_hook=detailer_hook, refiner_ratio=refiner_ratio, refiner_model=refiner_model, refiner_clip=refiner_clip, refiner_positive=refiner_positive, - refiner_negative=refiner_negative, control_net_wrapper=seg.control_net_wrapper, cycle=cycle) + refiner_negative=refiner_negative, control_net_wrapper=seg.control_net_wrapper, + cycle=cycle, inpaint_model=inpaint_model) if cnet_pils is not None: cnet_pil_list.extend(cnet_pils) @@ -280,12 +285,13 @@ class DetailerForEach: return image_tensor, cropped_list, enhanced_list, enhanced_alpha_list, cnet_pil_list, (segs[0], new_segs) def doit(self, image, segs, model, clip, vae, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, - scheduler, positive, negative, denoise, feather, noise_mask, force_inpaint, wildcard, cycle=1, detailer_hook=None): + scheduler, positive, negative, denoise, feather, noise_mask, force_inpaint, wildcard, cycle=1, + detailer_hook=None, inpaint_model=False): enhanced_img, *_ = \ DetailerForEach.do_detail(image, segs, model, clip, vae, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, scheduler, positive, negative, denoise, feather, noise_mask, - force_inpaint, wildcard, detailer_hook, cycle=cycle) + force_inpaint, wildcard, detailer_hook, cycle=cycle, inpaint_model=inpaint_model) return (enhanced_img, ) @@ -317,6 +323,7 @@ class DetailerForEachPipe: "optional": { "detailer_hook": ("DETAILER_HOOK",), "refiner_basic_pipe_opt": ("BASIC_PIPE",), + "inpaint_model": ("BOOLEAN", {"default": False, "label_on": "enabled", "label_off": "disabled"}), } } @@ -329,7 +336,7 @@ class DetailerForEachPipe: def doit(self, image, segs, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, scheduler, denoise, feather, noise_mask, force_inpaint, basic_pipe, wildcard, - refiner_ratio=None, detailer_hook=None, refiner_basic_pipe_opt=None, cycle=1): + refiner_ratio=None, detailer_hook=None, refiner_basic_pipe_opt=None, cycle=1, inpaint_model=False): if len(image) > 1: raise Exception('[Impact Pack] ERROR: DetailerForEach does not allow image batches.\nPlease refer to https://github.com/ltdrdata/ComfyUI-extension-tutorials/blob/Main/ComfyUI-Impact-Pack/tutorial/batching-detailer.md for more information.') @@ -347,7 +354,7 @@ class DetailerForEachPipe: force_inpaint, wildcard, detailer_hook, refiner_ratio=refiner_ratio, refiner_model=refiner_model, refiner_clip=refiner_clip, refiner_positive=refiner_positive, refiner_negative=refiner_negative, - cycle=cycle) + cycle=cycle, inpaint_model=inpaint_model) # set fallback image if len(cnet_pil_list) == 0: @@ -400,7 +407,8 @@ class FaceDetailer: "optional": { "sam_model_opt": ("SAM_MODEL", ), "segm_detector_opt": ("SEGM_DETECTOR", ), - "detailer_hook": ("DETAILER_HOOK",) + "detailer_hook": ("DETAILER_HOOK",), + "inpaint_model": ("BOOLEAN", {"default": False, "label_on": "enabled", "label_off": "disabled"}), }} RETURN_TYPES = ("IMAGE", "IMAGE", "IMAGE", "MASK", "DETAILER_PIPE", "IMAGE") @@ -417,7 +425,8 @@ class FaceDetailer: sam_detection_hint, sam_dilation, sam_threshold, sam_bbox_expansion, sam_mask_hint_threshold, sam_mask_hint_use_negative, drop_size, bbox_detector, segm_detector=None, sam_model_opt=None, wildcard_opt=None, detailer_hook=None, - refiner_ratio=None, refiner_model=None, refiner_clip=None, refiner_positive=None, refiner_negative=None, cycle=1): + refiner_ratio=None, refiner_model=None, refiner_clip=None, refiner_positive=None, refiner_negative=None, cycle=1, + inpaint_model=False): # make default prompt as 'face' if empty prompt for CLIPSeg bbox_detector.setAux('face') @@ -448,7 +457,7 @@ class FaceDetailer: force_inpaint, wildcard_opt, detailer_hook, refiner_ratio=refiner_ratio, refiner_model=refiner_model, refiner_clip=refiner_clip, refiner_positive=refiner_positive, - refiner_negative=refiner_negative, cycle=cycle) + refiner_negative=refiner_negative, cycle=cycle, inpaint_model=inpaint_model) else: enhanced_img = image cropped_enhanced = [] @@ -474,7 +483,7 @@ class FaceDetailer: bbox_threshold, bbox_dilation, bbox_crop_factor, sam_detection_hint, sam_dilation, sam_threshold, sam_bbox_expansion, sam_mask_hint_threshold, sam_mask_hint_use_negative, drop_size, bbox_detector, wildcard, cycle=1, - sam_model_opt=None, segm_detector_opt=None, detailer_hook=None): + sam_model_opt=None, segm_detector_opt=None, detailer_hook=None, inpaint_model=False): result_img = None result_mask = None @@ -491,7 +500,8 @@ class FaceDetailer: positive, negative, denoise, feather, noise_mask, force_inpaint, bbox_threshold, bbox_dilation, bbox_crop_factor, sam_detection_hint, sam_dilation, sam_threshold, sam_bbox_expansion, sam_mask_hint_threshold, - sam_mask_hint_use_negative, drop_size, bbox_detector, segm_detector_opt, sam_model_opt, wildcard, detailer_hook, cycle=cycle) + sam_mask_hint_use_negative, drop_size, bbox_detector, segm_detector_opt, sam_model_opt, wildcard, detailer_hook, + cycle=cycle, inpaint_model=inpaint_model) result_img = torch.cat((result_img, enhanced_img), dim=0) if result_img is not None else enhanced_img result_mask = torch.cat((result_mask, mask), dim=0) if result_mask is not None else mask @@ -1165,7 +1175,10 @@ class FaceDetailerPipe: "refiner_ratio": ("FLOAT", {"default": 0.2, "min": 0.0, "max": 1.0}), "cycle": ("INT", {"default": 1, "min": 1, "max": 10, "step": 1}), - }, + }, + "optional": { + "inpaint_model": ("BOOLEAN", {"default": False, "label_on": "enabled", "label_off": "disabled"}), + } } RETURN_TYPES = ("IMAGE", "IMAGE", "IMAGE", "MASK", "DETAILER_PIPE", "IMAGE") @@ -1178,7 +1191,7 @@ class FaceDetailerPipe: def doit(self, image, detailer_pipe, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, scheduler, denoise, feather, noise_mask, force_inpaint, bbox_threshold, bbox_dilation, bbox_crop_factor, sam_detection_hint, sam_dilation, sam_threshold, sam_bbox_expansion, - sam_mask_hint_threshold, sam_mask_hint_use_negative, drop_size, refiner_ratio=None, cycle=1): + sam_mask_hint_threshold, sam_mask_hint_use_negative, drop_size, refiner_ratio=None, cycle=1, inpaint_model=False): result_img = None result_mask = None @@ -1201,7 +1214,7 @@ class FaceDetailerPipe: sam_mask_hint_use_negative, drop_size, bbox_detector, segm_detector, sam_model_opt, wildcard, detailer_hook, refiner_ratio=refiner_ratio, refiner_model=refiner_model, refiner_clip=refiner_clip, refiner_positive=refiner_positive, refiner_negative=refiner_negative, - cycle=cycle) + cycle=cycle, inpaint_model=inpaint_model) result_img = torch.cat((result_img, enhanced_img), dim=0) if result_img is not None else enhanced_img result_mask = torch.cat((result_mask, mask), dim=0) if result_mask is not None else mask @@ -1252,6 +1265,7 @@ class MaskDetailerPipe: "optional": { "refiner_basic_pipe_opt": ("BASIC_PIPE", ), "detailer_hook": ("DETAILER_HOOK",), + "inpaint_model": ("BOOLEAN", {"default": False, "label_on": "enabled", "label_off": "disabled"}), } } @@ -1265,7 +1279,7 @@ class MaskDetailerPipe: def doit(self, image, mask, basic_pipe, guide_size, guide_size_for, max_size, mask_mode, seed, steps, cfg, sampler_name, scheduler, denoise, feather, crop_factor, drop_size, refiner_ratio, batch_size, cycle=1, - refiner_basic_pipe_opt=None, detailer_hook=None): + refiner_basic_pipe_opt=None, detailer_hook=None, inpaint_model=False): if len(image) > 1: raise Exception('[Impact Pack] ERROR: MaskDetailer does not allow image batches.\nPlease refer to https://github.com/ltdrdata/ComfyUI-extension-tutorials/blob/Main/ComfyUI-Impact-Pack/tutorial/batching-detailer.md for more information.') @@ -1295,7 +1309,8 @@ class MaskDetailerPipe: cfg, sampler_name, scheduler, positive, negative, denoise, feather, mask_mode, force_inpaint=True, wildcard_opt=None, detailer_hook=detailer_hook, refiner_ratio=refiner_ratio, refiner_model=refiner_model, refiner_clip=refiner_clip, - refiner_positive=refiner_positive, refiner_negative=refiner_negative, cycle=cycle) + refiner_positive=refiner_positive, refiner_negative=refiner_negative, + cycle=cycle, inpaint_model=inpaint_model) else: enhanced_img, cropped_enhanced, cropped_enhanced_alpha = image, [], [] @@ -1328,7 +1343,7 @@ class DetailerForEachTest(DetailerForEach): def doit(self, image, segs, model, clip, vae, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, scheduler, positive, negative, denoise, feather, noise_mask, force_inpaint, wildcard, detailer_hook=None, - cycle=1): + cycle=1, inpaint_model=False): if len(image) > 1: raise Exception('[Impact Pack] ERROR: DetailerForEach does not allow image batches.\nPlease refer to https://github.com/ltdrdata/ComfyUI-extension-tutorials/blob/Main/ComfyUI-Impact-Pack/tutorial/batching-detailer.md for more information.') @@ -1336,7 +1351,7 @@ class DetailerForEachTest(DetailerForEach): enhanced_img, cropped, cropped_enhanced, cropped_enhanced_alpha, cnet_pil_list, new_segs = \ DetailerForEach.do_detail(image, segs, model, clip, vae, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, scheduler, positive, negative, denoise, feather, noise_mask, - force_inpaint, wildcard, detailer_hook, cycle=cycle) + force_inpaint, wildcard, detailer_hook, cycle=cycle, inpaint_model=inpaint_model) # set fallback image if len(cropped) == 0: @@ -1365,7 +1380,7 @@ class DetailerForEachTestPipe(DetailerForEachPipe): def doit(self, image, segs, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, scheduler, denoise, feather, noise_mask, force_inpaint, basic_pipe, wildcard, cycle=1, - refiner_ratio=None, detailer_hook=None, refiner_basic_pipe_opt=None): + refiner_ratio=None, detailer_hook=None, refiner_basic_pipe_opt=None, inpaint_model=False): if len(image) > 1: raise Exception('[Impact Pack] ERROR: DetailerForEach does not allow image batches.\nPlease refer to https://github.com/ltdrdata/ComfyUI-extension-tutorials/blob/Main/ComfyUI-Impact-Pack/tutorial/batching-detailer.md for more information.') @@ -1383,7 +1398,7 @@ class DetailerForEachTestPipe(DetailerForEachPipe): force_inpaint, wildcard, detailer_hook, refiner_ratio=refiner_ratio, refiner_model=refiner_model, refiner_clip=refiner_clip, refiner_positive=refiner_positive, - refiner_negative=refiner_negative, cycle=cycle) + refiner_negative=refiner_negative, cycle=cycle, inpaint_model=inpaint_model) # set fallback image if len(cropped) == 0: diff --git a/modules/impact/segs_nodes.py b/modules/impact/segs_nodes.py index fc21ee8..5f6e5fa 100644 --- a/modules/impact/segs_nodes.py +++ b/modules/impact/segs_nodes.py @@ -38,7 +38,8 @@ class SEGSDetailer: }, "optional": { "refiner_basic_pipe_opt": ("BASIC_PIPE",), - } + "inpaint_model": ("BOOLEAN", {"default": False, "label_on": "enabled", "label_off": "disabled"}), + } } RETURN_TYPES = ("SEGS", "IMAGE") @@ -52,7 +53,7 @@ class SEGSDetailer: @staticmethod def do_detail(image, segs, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, scheduler, denoise, noise_mask, force_inpaint, basic_pipe, refiner_ratio=None, batch_size=1, cycle=1, - refiner_basic_pipe_opt=None): + refiner_basic_pipe_opt=None, inpaint_model=False): model, clip, vae, positive, negative = basic_pipe if refiner_basic_pipe_opt is None: @@ -84,11 +85,11 @@ class SEGSDetailer: cropped_mask = None enhanced_image, cnet_pils = core.enhance_detail(cropped_image, model, clip, vae, guide_size, guide_size_for, max_size, - seg.bbox, seed, steps, cfg, sampler_name, scheduler, - positive, negative, denoise, cropped_mask, force_inpaint, - refiner_ratio=refiner_ratio, refiner_model=refiner_model, - refiner_clip=refiner_clip, refiner_positive=refiner_positive, refiner_negative=refiner_negative, - control_net_wrapper=seg.control_net_wrapper, cycle=cycle) + seg.bbox, seed, steps, cfg, sampler_name, scheduler, + positive, negative, denoise, cropped_mask, force_inpaint, + refiner_ratio=refiner_ratio, refiner_model=refiner_model, + refiner_clip=refiner_clip, refiner_positive=refiner_positive, refiner_negative=refiner_negative, + control_net_wrapper=seg.control_net_wrapper, cycle=cycle, inpaint_model=inpaint_model) if cnet_pils is not None: cnet_pil_list.extend(cnet_pils) @@ -105,14 +106,14 @@ class SEGSDetailer: def doit(self, image, segs, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, scheduler, denoise, noise_mask, force_inpaint, basic_pipe, refiner_ratio=None, batch_size=1, cycle=1, - refiner_basic_pipe_opt=None): + refiner_basic_pipe_opt=None, inpaint_model=False): if len(image) > 1: raise Exception('[Impact Pack] ERROR: SEGSDetailer does not allow image batches.\nPlease refer to https://github.com/ltdrdata/ComfyUI-extension-tutorials/blob/Main/ComfyUI-Impact-Pack/tutorial/batching-detailer.md for more information.') segs, cnet_pil_list = SEGSDetailer.do_detail(image, segs, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, scheduler, denoise, noise_mask, force_inpaint, basic_pipe, refiner_ratio, batch_size, cycle=cycle, - refiner_basic_pipe_opt=refiner_basic_pipe_opt) + refiner_basic_pipe_opt=refiner_basic_pipe_opt, inpaint_model=inpaint_model) # set fallback image if len(cnet_pil_list) == 0: @@ -141,7 +142,8 @@ class SEGSDetailerForAnimateDiff: }, "optional": { "refiner_basic_pipe_opt": ("BASIC_PIPE",), - } + # TODO: "inpaint_model": ("BOOLEAN", {"default": False, "label_on": "enabled", "label_off": "disabled"}), + } } RETURN_TYPES = ("SEGS",) @@ -154,7 +156,7 @@ class SEGSDetailerForAnimateDiff: @staticmethod def do_detail(image_frames, segs, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, scheduler, - denoise, basic_pipe, refiner_ratio=None, refiner_basic_pipe_opt=None): + denoise, basic_pipe, refiner_ratio=None, refiner_basic_pipe_opt=None, inpaint_model=False): model, clip, vae, positive, negative = basic_pipe if refiner_basic_pipe_opt is None: @@ -183,7 +185,8 @@ class SEGSDetailerForAnimateDiff: seg.bbox, seed, steps, cfg, sampler_name, scheduler, positive, negative, denoise, seg.cropped_mask, refiner_ratio=refiner_ratio, refiner_model=refiner_model, - refiner_clip=refiner_clip, refiner_positive=refiner_positive, refiner_negative=refiner_negative) + refiner_clip=refiner_clip, refiner_positive=refiner_positive, + refiner_negative=refiner_negative, inpaint_model=inpaint_model) if enhanced_image_tensor is None: new_cropped_image = cropped_image_frames @@ -196,10 +199,10 @@ class SEGSDetailerForAnimateDiff: return (segs[0], new_segs) def doit(self, image_frames, segs, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, scheduler, - denoise, basic_pipe, refiner_ratio=None, refiner_basic_pipe_opt=None): + denoise, basic_pipe, refiner_ratio=None, refiner_basic_pipe_opt=None, inpaint_model=False): segs = SEGSDetailerForAnimateDiff.do_detail(image_frames, segs, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, - scheduler, denoise, basic_pipe, refiner_ratio, refiner_basic_pipe_opt) + scheduler, denoise, basic_pipe, refiner_ratio, refiner_basic_pipe_opt, inpaint_model=inpaint_model) return (segs,)