From b0958383022e9f4e3908e471560e93f107d892c7 Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" Date: Sat, 27 Jan 2024 12:27:48 +0900 Subject: [PATCH] feat: DetailerForEachPipeForAnimateDiff modify: change display name of SEGSDetailerForAnimateDiff --- __init__.py | 7 +- modules/impact/animatediff_nodes.py | 145 ++++++++++++++++++++++++++++ modules/impact/config.py | 2 +- modules/impact/segs_nodes.py | 90 ----------------- 4 files changed, 151 insertions(+), 93 deletions(-) create mode 100644 modules/impact/animatediff_nodes.py diff --git a/__init__.py b/__init__.py index 692fb65..6569b79 100644 --- a/__init__.py +++ b/__init__.py @@ -106,6 +106,7 @@ from impact.special_samplers import * from impact.hf_nodes import * from impact.bridge_nodes import * from impact.hook_nodes import * +from impact.animatediff_nodes import * import threading @@ -139,6 +140,7 @@ NODE_CLASS_MAPPINGS = { "DetailerForEachDebug": DetailerForEachTest, "DetailerForEachPipe": DetailerForEachPipe, "DetailerForEachDebugPipe": DetailerForEachTestPipe, + "DetailerForEachPipeForAnimateDiff": DetailerForEachPipeForAnimateDiff, "SAMDetectorCombined": SAMDetectorCombined, "SAMDetectorSegmented": SAMDetectorSegmented, @@ -349,12 +351,13 @@ NODE_DISPLAY_NAME_MAPPINGS = { "DetailerForEachPipe": "Detailer (SEGS/pipe)", "DetailerForEachDebug": "DetailerDebug (SEGS)", "DetailerForEachDebugPipe": "DetailerDebug (SEGS/pipe)", - "SEGSDetailerForAnimateDiff": "Detailer For AnimateDiff (SEGS/pipe)", + "SEGSDetailerForAnimateDiff": "SEGSDetailer For AnimateDiff (SEGS/pipe)", + "DetailerForEachPipeForAnimateDiff": "Detailer For AnimateDiff (SEGS/pipe)", "SAMDetectorCombined": "SAMDetector (combined)", "SAMDetectorSegmented": "SAMDetector (segmented)", "FaceDetailerPipe": "FaceDetailer (pipe)", - "MaskDetailerPipe": "MaskDetailer (Pipe)", + "MaskDetailerPipe": "MaskDetailer (pipe)", "FromDetailerPipeSDXL": "FromDetailer (SDXL/pipe)", "BasicPipeToDetailerPipeSDXL": "BasicPipe -> DetailerPipe (SDXL)", diff --git a/modules/impact/animatediff_nodes.py b/modules/impact/animatediff_nodes.py new file mode 100644 index 0000000..57d8d0c --- /dev/null +++ b/modules/impact/animatediff_nodes.py @@ -0,0 +1,145 @@ +from nodes import MAX_RESOLUTION +from impact.utils import * +import impact.core as core +from impact.core import SEG +from impact.segs_nodes import SEGSPaste + + +class SEGSDetailerForAnimateDiff: + @classmethod + def INPUT_TYPES(cls): + return {"required": { + "image_frames": ("IMAGE", ), + "segs": ("SEGS", ), + "guide_size": ("FLOAT", {"default": 256, "min": 64, "max": MAX_RESOLUTION, "step": 8}), + "guide_size_for": ("BOOLEAN", {"default": True, "label_on": "bbox", "label_off": "crop_region"}), + "max_size": ("FLOAT", {"default": 768, "min": 64, "max": MAX_RESOLUTION, "step": 8}), + "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), + "steps": ("INT", {"default": 20, "min": 1, "max": 10000}), + "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0}), + "sampler_name": (comfy.samplers.KSampler.SAMPLERS,), + "scheduler": (comfy.samplers.KSampler.SCHEDULERS,), + "denoise": ("FLOAT", {"default": 0.5, "min": 0.0001, "max": 1.0, "step": 0.01}), + "basic_pipe": ("BASIC_PIPE",), + "refiner_ratio": ("FLOAT", {"default": 0.2, "min": 0.0, "max": 1.0}) + }, + "optional": { + "refiner_basic_pipe_opt": ("BASIC_PIPE",), + # TODO: "inpaint_model": ("BOOLEAN", {"default": False, "label_on": "enabled", "label_off": "disabled"}), + # TODO: "noise_mask_feather": ("INT", {"default": 10, "min": 0, "max": 100, "step": 1}), + } + } + + RETURN_TYPES = ("SEGS",) + RETURN_NAMES = ("segs",) + OUTPUT_IS_LIST = (False,) + + FUNCTION = "doit" + + CATEGORY = "ImpactPack/Detailer" + + @staticmethod + def do_detail(image_frames, segs, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, scheduler, + denoise, basic_pipe, refiner_ratio=None, refiner_basic_pipe_opt=None, inpaint_model=False, noise_mask_feather=0): + + model, clip, vae, positive, negative = basic_pipe + if refiner_basic_pipe_opt is None: + refiner_model, refiner_clip, refiner_positive, refiner_negative = None, None, None, None + else: + refiner_model, refiner_clip, _, refiner_positive, refiner_negative = refiner_basic_pipe_opt + + segs = core.segs_scale_match(segs, image_frames.shape) + + new_segs = [] + + for seg in segs[1]: + cropped_image_frames = None + + for image in image_frames: + image = image.unsqueeze(0) + cropped_image = seg.cropped_image if seg.cropped_image is not None else crop_tensor4(image, seg.crop_region) + cropped_image = to_tensor(cropped_image) + if cropped_image_frames is None: + cropped_image_frames = cropped_image + else: + cropped_image_frames = torch.concat((cropped_image_frames, cropped_image), dim=0) + + cropped_image_frames = cropped_image_frames.numpy() + enhanced_image_tensor = core.enhance_detail_for_animatediff(cropped_image_frames, model, clip, vae, guide_size, guide_size_for, max_size, + seg.bbox, seed, steps, cfg, sampler_name, scheduler, + positive, negative, denoise, seg.cropped_mask, + refiner_ratio=refiner_ratio, refiner_model=refiner_model, + refiner_clip=refiner_clip, refiner_positive=refiner_positive, + refiner_negative=refiner_negative, + inpaint_model=inpaint_model, noise_mask_feather=noise_mask_feather) + + if enhanced_image_tensor is None: + new_cropped_image = cropped_image_frames + else: + new_cropped_image = enhanced_image_tensor.numpy() + + new_seg = SEG(new_cropped_image, seg.cropped_mask, seg.confidence, seg.crop_region, seg.bbox, seg.label, None) + new_segs.append(new_seg) + + return (segs[0], new_segs) + + def doit(self, image_frames, segs, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, scheduler, + denoise, basic_pipe, refiner_ratio=None, refiner_basic_pipe_opt=None, inpaint_model=False, noise_mask_feather=0): + + segs = SEGSDetailerForAnimateDiff.do_detail(image_frames, segs, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, + scheduler, denoise, basic_pipe, refiner_ratio, refiner_basic_pipe_opt, + inpaint_model=inpaint_model, noise_mask_feather=noise_mask_feather) + + return (segs,) + + +class DetailerForEachPipeForAnimateDiff: + @classmethod + def INPUT_TYPES(cls): + return {"required": { + "image_frames": ("IMAGE", ), + "segs": ("SEGS", ), + "guide_size": ("FLOAT", {"default": 384, "min": 64, "max": nodes.MAX_RESOLUTION, "step": 8}), + "guide_size_for": ("BOOLEAN", {"default": True, "label_on": "bbox", "label_off": "crop_region"}), + "max_size": ("FLOAT", {"default": 1024, "min": 64, "max": nodes.MAX_RESOLUTION, "step": 8}), + "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), + "steps": ("INT", {"default": 20, "min": 1, "max": 10000}), + "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0}), + "sampler_name": (comfy.samplers.KSampler.SAMPLERS,), + "scheduler": (comfy.samplers.KSampler.SCHEDULERS,), + "denoise": ("FLOAT", {"default": 0.5, "min": 0.0001, "max": 1.0, "step": 0.01}), + "feather": ("INT", {"default": 5, "min": 0, "max": 100, "step": 1}), + "basic_pipe": ("BASIC_PIPE", ), + "refiner_ratio": ("FLOAT", {"default": 0.2, "min": 0.0, "max": 1.0}), + }, + "optional": { + "detailer_hook": ("DETAILER_HOOK",), + "refiner_basic_pipe_opt": ("BASIC_PIPE",), + # "inpaint_model": ("BOOLEAN", {"default": False, "label_on": "enabled", "label_off": "disabled"}), + # "noise_mask_feather": ("INT", {"default": 10, "min": 0, "max": 100, "step": 1}), + } + } + + RETURN_TYPES = ("IMAGE", "SEGS", "BASIC_PIPE") + RETURN_NAMES = ("image", "segs", "basic_pipe") + OUTPUT_IS_LIST = (False, False, False, True) + FUNCTION = "doit" + + CATEGORY = "ImpactPack/Detailer" + + @staticmethod + def doit(image_frames, segs, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, scheduler, + denoise, feather, basic_pipe, refiner_ratio=None, detailer_hook=None, refiner_basic_pipe_opt=None, + inpaint_model=False, noise_mask_feather=0): + + enhanced_segs = [] + for sub_seg in segs[1]: + single_seg = segs[0], [sub_seg] + enhanced_seg = SEGSDetailerForAnimateDiff().do_detail(image_frames, single_seg, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, scheduler, + denoise, basic_pipe, refiner_ratio, refiner_basic_pipe_opt, inpaint_model, noise_mask_feather) + + image_frames = SEGSPaste.doit(image_frames, enhanced_seg, feather, alpha=255)[0] + enhanced_segs += enhanced_seg[1] + + new_segs = segs[0], enhanced_segs + return image_frames, new_segs, basic_pipe diff --git a/modules/impact/config.py b/modules/impact/config.py index 1168634..fc40699 100644 --- a/modules/impact/config.py +++ b/modules/impact/config.py @@ -2,7 +2,7 @@ import configparser import os -version_code = [4, 68] +version_code = [4, 69] version = f"V{version_code[0]}.{version_code[1]}" + (f'.{version_code[2]}' if len(version_code) > 2 else '') dependency_version = 20 diff --git a/modules/impact/segs_nodes.py b/modules/impact/segs_nodes.py index be15ecf..856a7bc 100644 --- a/modules/impact/segs_nodes.py +++ b/modules/impact/segs_nodes.py @@ -1,8 +1,6 @@ import os import sys -import torch - import impact.impact_server from nodes import MAX_RESOLUTION @@ -125,94 +123,6 @@ class SEGSDetailer: return (segs, cnet_pil_list) -class SEGSDetailerForAnimateDiff: - @classmethod - def INPUT_TYPES(s): - return {"required": { - "image_frames": ("IMAGE", ), - "segs": ("SEGS", ), - "guide_size": ("FLOAT", {"default": 256, "min": 64, "max": MAX_RESOLUTION, "step": 8}), - "guide_size_for": ("BOOLEAN", {"default": True, "label_on": "bbox", "label_off": "crop_region"}), - "max_size": ("FLOAT", {"default": 768, "min": 64, "max": MAX_RESOLUTION, "step": 8}), - "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), - "steps": ("INT", {"default": 20, "min": 1, "max": 10000}), - "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0}), - "sampler_name": (comfy.samplers.KSampler.SAMPLERS,), - "scheduler": (comfy.samplers.KSampler.SCHEDULERS,), - "denoise": ("FLOAT", {"default": 0.5, "min": 0.0001, "max": 1.0, "step": 0.01}), - "basic_pipe": ("BASIC_PIPE",), - "refiner_ratio": ("FLOAT", {"default": 0.2, "min": 0.0, "max": 1.0}) - }, - "optional": { - "refiner_basic_pipe_opt": ("BASIC_PIPE",), - # TODO: "inpaint_model": ("BOOLEAN", {"default": False, "label_on": "enabled", "label_off": "disabled"}), - # TODO: "noise_mask_feather": ("INT", {"default": 10, "min": 0, "max": 100, "step": 1}), - } - } - - RETURN_TYPES = ("SEGS",) - RETURN_NAMES = ("segs",) - OUTPUT_IS_LIST = (False,) - - FUNCTION = "doit" - - CATEGORY = "ImpactPack/Detailer" - - @staticmethod - def do_detail(image_frames, segs, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, scheduler, - denoise, basic_pipe, refiner_ratio=None, refiner_basic_pipe_opt=None, inpaint_model=False, noise_mask_feather=0): - - model, clip, vae, positive, negative = basic_pipe - if refiner_basic_pipe_opt is None: - refiner_model, refiner_clip, refiner_positive, refiner_negative = None, None, None, None - else: - refiner_model, refiner_clip, _, refiner_positive, refiner_negative = refiner_basic_pipe_opt - - segs = core.segs_scale_match(segs, image_frames.shape) - - new_segs = [] - - for seg in segs[1]: - cropped_image_frames = None - - for image in image_frames: - image = image.unsqueeze(0) - cropped_image = seg.cropped_image if seg.cropped_image is not None else crop_tensor4(image, seg.crop_region) - cropped_image = to_tensor(cropped_image) - if cropped_image_frames is None: - cropped_image_frames = cropped_image - else: - cropped_image_frames = torch.concat((cropped_image_frames, cropped_image), dim=0) - - cropped_image_frames = cropped_image_frames.numpy() - enhanced_image_tensor = core.enhance_detail_for_animatediff(cropped_image_frames, model, clip, vae, guide_size, guide_size_for, max_size, - seg.bbox, seed, steps, cfg, sampler_name, scheduler, - positive, negative, denoise, seg.cropped_mask, - refiner_ratio=refiner_ratio, refiner_model=refiner_model, - refiner_clip=refiner_clip, refiner_positive=refiner_positive, - refiner_negative=refiner_negative, - inpaint_model=inpaint_model, noise_mask_feather=noise_mask_feather) - - if enhanced_image_tensor is None: - new_cropped_image = cropped_image_frames - else: - new_cropped_image = enhanced_image_tensor.numpy() - - new_seg = SEG(new_cropped_image, seg.cropped_mask, seg.confidence, seg.crop_region, seg.bbox, seg.label, None) - new_segs.append(new_seg) - - return (segs[0], new_segs) - - def doit(self, image_frames, segs, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, scheduler, - denoise, basic_pipe, refiner_ratio=None, refiner_basic_pipe_opt=None, inpaint_model=False, noise_mask_feather=0): - - segs = SEGSDetailerForAnimateDiff.do_detail(image_frames, segs, guide_size, guide_size_for, max_size, seed, steps, cfg, sampler_name, - scheduler, denoise, basic_pipe, refiner_ratio, refiner_basic_pipe_opt, - inpaint_model=inpaint_model, noise_mask_feather=noise_mask_feather) - - return (segs,) - - class SEGSPaste: @classmethod def INPUT_TYPES(s):