From a5ca503dc07ec8bbbbcb8328edb635e93c855aef Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" Date: Tue, 27 Jun 2023 16:48:42 +0900 Subject: [PATCH] added: TwoAdvancedSamplersForMask, KSamplerAdvancedProvider --- README.md | 4 + __init__.py | 3 + modules/impact/config.py | 2 +- modules/impact/core.py | 20 + modules/impact/impact_pack.py | 85 +++ temp/advanced-sampler.json | 978 ++++++++++++++++++++++++++++++++++ 6 files changed, 1091 insertions(+), 1 deletion(-) create mode 100644 temp/advanced-sampler.json diff --git a/README.md b/README.md index 58e98bea..d747bd79 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,9 @@ This takes latent as input and outputs latent as the result. * TiledKSamplerProvider - ComfyUI_TiledKSampler is a wrapper that provides KSAMPLER. * You need to install the [ComfyUI_TiledKSampler](https://github.com/BlenderNeko/ComfyUI_TiledKSampler) node extension. +* TwoAdvancedSamplersForMask - TwoSamplersForMask is similar to TwoAdvancedSamplersForMask, but they differ in their operation. TwoSamplersForMask performs sampling in the mask area only after all the samples in the base area are finished. On the other hand, TwoAdvancedSamplersForMask performs sampling in both the base area and the mask area sequentially at each step. +* KSamplerAdvancedProvider - This is a wrapper that enables KSampler to be used in TwoAdvancedSamplersForMask. + * TwoSamplersForMaskUpscalerProvider - This is an Upscaler that extends TwoSamplersForMask to be used in Iterative Upscale. * TwoSamplersForMaskUpscalerProviderPipe - pipe version of TwoSamplersForMaskUpscalerProvider. @@ -194,6 +197,7 @@ This takes latent as input and outputs latent as the result. * [CLIPSeg Tutorial](https://github.com/ltdrdata/ComfyUI-extension-tutorials/blob/Main/ComfyUI-Impact-Pack/tutorial/clipseg.md) * [Extreme Highresolution Upscale](https://github.com/ltdrdata/ComfyUI-extension-tutorials/blob/Main/ComfyUI-Impact-Pack/tutorial/extreme-upscale.md) * [TwoSamplersForMask](https://github.com/ltdrdata/ComfyUI-extension-tutorials/blob/Main/ComfyUI-Impact-Pack/tutorial/TwoSamplers.md) +* [TwoAdvancedSamplersForMask](https://github.com/ltdrdata/ComfyUI-extension-tutorials/blob/Main/ComfyUI-Impact-Pack/tutorial/TwoAdvancedSamplers.md) * [Advanced Iterative Upscale: PK_HOOK](https://github.com/ltdrdata/ComfyUI-extension-tutorials/blob/Main/ComfyUI-Impact-Pack/tutorial/pk_hook.md) * [Advanced Iterative Upscale: TwoSamplersForMask Upscale Provider](https://github.com/ltdrdata/ComfyUI-extension-tutorials/blob/Main/ComfyUI-Impact-Pack/tutorial/TwoSamplersUpscale.md) diff --git a/__init__.py b/__init__.py index dc076fea..5cd7f835 100644 --- a/__init__.py +++ b/__init__.py @@ -135,6 +135,9 @@ NODE_CLASS_MAPPINGS = { "TwoSamplersForMask": TwoSamplersForMask, "TiledKSamplerProvider": TiledKSamplerProvider, + "KSamplerAdvancedProvider": KSamplerAdvancedProvider, + "TwoAdvancedSamplersForMask": TwoAdvancedSamplersForMask, + "PreviewBridge": PreviewBridge, "ImageSender": ImageSender, "ImageReceiver": ImageReceiver, diff --git a/modules/impact/config.py b/modules/impact/config.py index e046bf5d..db97344c 100644 --- a/modules/impact/config.py +++ b/modules/impact/config.py @@ -1,7 +1,7 @@ import configparser import os -version = "V2.15.3" +version = "V2.16" dependency_version = 1 diff --git a/modules/impact/core.py b/modules/impact/core.py index 246fd5b0..e374e86b 100644 --- a/modules/impact/core.py +++ b/modules/impact/core.py @@ -674,6 +674,26 @@ class KSamplerWrapper: return nodes.common_ksampler(model, seed, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise=denoise)[0] +class KSamplerAdvancedWrapper: + params = None + + def __init__(self, model, cfg, sampler_name, scheduler, positive, negative): + self.params = model, cfg, sampler_name, scheduler, positive, negative + + def sample_advanced(self, add_noise, seed, steps, latent_image, start_at_step, end_at_step, return_with_leftover_noise, hook=None): + model, cfg, sampler_name, scheduler, positive, negative = self.params + + if hook is not None: + model, seed, steps, cfg, sampler_name, scheduler, positive, negative, upscaled_latent = \ + hook.pre_ksample_advanced(model, add_noise, seed, steps, cfg, sampler_name, scheduler, + positive, negative, latent_image, start_at_step, end_at_step, + return_with_leftover_noise) + + return nodes.KSamplerAdvanced().sample(model, add_noise, seed, steps, cfg, sampler_name, scheduler, + positive, negative, latent_image, start_at_step, end_at_step, + return_with_leftover_noise)[0] + + class PixelKSampleHook: cur_step = 0 total_step = 0 diff --git a/modules/impact/impact_pack.py b/modules/impact/impact_pack.py index d8ffa1ae..7dfd67b2 100644 --- a/modules/impact/impact_pack.py +++ b/modules/impact/impact_pack.py @@ -7,6 +7,7 @@ import warnings from segment_anything import sam_model_registry from io import BytesIO import piexif +import math import zipfile import re from server import PromptServer @@ -432,6 +433,28 @@ class KSamplerProvider: return (sampler, ) +class KSamplerAdvancedProvider: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0}), + "sampler_name": (comfy.samplers.KSampler.SAMPLERS, ), + "scheduler": (comfy.samplers.KSampler.SCHEDULERS, ), + "basic_pipe": ("BASIC_PIPE", ) + }, + } + + RETURN_TYPES = ("KSAMPLER_ADVANCED",) + FUNCTION = "doit" + + CATEGORY = "ImpactPack/Sampler" + + def doit(self, cfg, sampler_name, scheduler, basic_pipe): + model, _, _, positive, negative = basic_pipe + sampler = core.KSamplerAdvancedWrapper(model, cfg, sampler_name, scheduler, positive, negative) + return (sampler, ) + + class TwoSamplersForMask: @classmethod def INPUT_TYPES(s): @@ -462,6 +485,68 @@ class TwoSamplersForMask: return (new_latent_image, ) +class TwoAdvancedSamplersForMask: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), + "steps": ("INT", {"default": 20, "min": 1, "max": 10000}), + "denoise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), + "latent_image": ("LATENT", ), + "base_sampler": ("KSAMPLER_ADVANCED", ), + "mask_sampler": ("KSAMPLER_ADVANCED", ), + "mask": ("MASK", ), + "overlap_factor": ("INT", {"default": 10, "min": 0, "max": 10000}) + }, + } + + RETURN_TYPES = ("LATENT", ) + FUNCTION = "doit" + + CATEGORY = "ImpactPack/Sampler" + + @staticmethod + def mask_erosion(latent_image, mask, grow_mask_by): + w = latent_image['samples'].shape[3] + h = latent_image['samples'].shape[2] + + mask2 = torch.nn.functional.interpolate(mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])), size=(w, h), mode="bilinear") + if grow_mask_by == 0: + mask_erosion = mask2 + else: + kernel_tensor = torch.ones((1, 1, grow_mask_by, grow_mask_by)) + padding = math.ceil((grow_mask_by - 1) / 2) + + mask_erosion = torch.clamp(torch.nn.functional.conv2d(mask2.round(), kernel_tensor, padding=padding), 0, 1) + + return mask_erosion[:, :, :w, :h].round() + + def doit(self, seed, steps, denoise, latent_image, base_sampler, mask_sampler, mask, overlap_factor): + + inv_mask = torch.where(mask != 1.0, torch.tensor(1.0), torch.tensor(0.0)) + + adv_steps = int(steps / denoise) + start_at_step = adv_steps - steps + + new_latent_image = {'samples': latent_image['samples']} + + mask_erosion = TwoAdvancedSamplersForMask.mask_erosion(latent_image, mask, overlap_factor) + + for i in range(start_at_step, adv_steps): + add_noise = "enable" if i == start_at_step else "disable" + return_with_leftover_noise = "enable" if i+1 != adv_steps else "disable" + + new_latent_image['noise_mask'] = inv_mask + new_latent_image = base_sampler.sample_advanced(add_noise, seed, adv_steps, new_latent_image, i, i + 1, "enable") + + new_latent_image['noise_mask'] = mask_erosion + new_latent_image = mask_sampler.sample_advanced("disable", seed, adv_steps, new_latent_image, i, i + 1, return_with_leftover_noise) + + del new_latent_image['noise_mask'] + + return (new_latent_image, ) + + class FaceDetailer: @classmethod def INPUT_TYPES(s): diff --git a/temp/advanced-sampler.json b/temp/advanced-sampler.json new file mode 100644 index 00000000..d5c23a44 --- /dev/null +++ b/temp/advanced-sampler.json @@ -0,0 +1,978 @@ +{ + "last_node_id": 25, + "last_link_id": 41, + "nodes": [ + { + "id": 11, + "type": "EditBasicPipe", + "pos": [ + 1260, + 590 + ], + "size": { + "0": 267, + "1": 126 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "basic_pipe", + "type": "BASIC_PIPE", + "link": 15 + }, + { + "name": "model", + "type": "MODEL", + "link": null + }, + { + "name": "clip", + "type": "CLIP", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 17 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + } + ], + "outputs": [ + { + "name": "basic_pipe", + "type": "BASIC_PIPE", + "links": [ + 20 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EditBasicPipe" + } + }, + { + "id": 12, + "type": "CLIPTextEncode", + "pos": [ + 420, + 670 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 16 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 17 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "photorealistic:1.4, best quality:1.4, masterpiece, 1girl is sitting in the cafe terrace, colorful hair" + ] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [ + 415, + 186 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 13 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "photorealistic:1.4, best quality:1.4, masterpiece, 1girl is sitting in the cafe terrace" + ] + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 413, + 389 + ], + "size": { + "0": 425.27801513671875, + "1": 180.6060791015625 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 14 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "text, watermark, low quality:1.4, worst quality:1.4" + ] + }, + { + "id": 16, + "type": "EmptyLatentImage", + "pos": [ + 532, + 1143 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 25, + 28 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 792, + 512, + 1 + ] + }, + { + "id": 10, + "type": "ToBasicPipe", + "pos": [ + 952, + 189 + ], + "size": { + "0": 241.79998779296875, + "1": 106 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 10 + }, + { + "name": "clip", + "type": "CLIP", + "link": 11 + }, + { + "name": "vae", + "type": "VAE", + "link": 12 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 13 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 14 + } + ], + "outputs": [ + { + "name": "basic_pipe", + "type": "BASIC_PIPE", + "links": [ + 15, + 19, + 33 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ToBasicPipe" + } + }, + { + "id": 20, + "type": "VAEDecode", + "pos": [ + 1579, + 1077 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 30 + }, + { + "name": "vae", + "type": "VAE", + "link": 31 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 37 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 23, + "type": "PreviewBridge", + "pos": [ + 1829, + 1076 + ], + "size": { + "0": 315, + "1": 290 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 37 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": null, + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 38 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "PreviewBridge" + }, + "widgets_values": [ + { + "filename": "clipspace-mask-75860.60000000894.png", + "subfolder": "clipspace", + "type": "input", + "image_hash": -3419835239699805000, + "forward_filename": "ComfyUI_00003_.png", + "forward_subfolder": "", + "forward_type": "temp" + } + ] + }, + { + "id": 14, + "type": "TwoAdvancedSamplersForMask", + "pos": [ + 2253, + 383 + ], + "size": { + "0": 315, + "1": 354 + }, + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "latent_image", + "type": "LATENT", + "link": 25, + "slot_index": 0 + }, + { + "name": "base_sampler", + "type": "KSAMPLER_ADVANCED", + "link": 24 + }, + { + "name": "mask_sampler", + "type": "KSAMPLER_ADVANCED", + "link": 23 + }, + { + "name": "mask", + "type": "MASK", + "link": 38 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 39 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "TwoAdvancedSamplersForMask" + }, + "widgets_values": [ + 20, + 1 + ] + }, + { + "id": 22, + "type": "FromBasicPipe", + "pos": [ + 880, + 1040 + ], + "size": { + "0": 241.79998779296875, + "1": 106 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "basic_pipe", + "type": "BASIC_PIPE", + "link": 33 + } + ], + "outputs": [ + { + "name": "model", + "type": "MODEL", + "links": [ + 34 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "clip", + "type": "CLIP", + "links": null, + "shape": 3 + }, + { + "name": "vae", + "type": "VAE", + "links": [ + 40 + ], + "shape": 3, + "slot_index": 2 + }, + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 35 + ], + "shape": 3, + "slot_index": 3 + }, + { + "name": "negative", + "type": "CONDITIONING", + "links": [ + 36 + ], + "shape": 3, + "slot_index": 4 + } + ], + "properties": { + "Node name for S&R": "FromBasicPipe" + } + }, + { + "id": 24, + "type": "VAEDecode", + "pos": [ + 1938, + 935 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 39 + }, + { + "name": "vae", + "type": "VAE", + "link": 40 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 41 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [ + -5, + 212 + ], + "size": { + "0": 315, + "1": 98 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 10 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 3, + 5, + 11, + 16 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 12, + 31 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "V07_v07.safetensors" + ] + }, + { + "id": 19, + "type": "KSampler", + "pos": [ + 1194.657802060547, + 1075.971700888672 + ], + "size": [ + 315, + 473.9999771118164 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 34 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 35 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 36 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 28 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 30 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 1107040072933062, + "fixed", + 20, + 8, + "euler", + "normal", + 1 + ] + }, + { + "id": 13, + "type": "KSamplerAdvancedProvider", + "pos": [ + 1727, + 192 + ], + "size": { + "0": 355.20001220703125, + "1": 154 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "basic_pipe", + "type": "BASIC_PIPE", + "link": 19 + } + ], + "outputs": [ + { + "name": "KSAMPLER_ADVANCED", + "type": "KSAMPLER_ADVANCED", + "links": [ + 24 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSamplerAdvancedProvider" + }, + "widgets_values": [ + 1107040072933062, + "fixed", + 8, + "euler", + "normal" + ] + }, + { + "id": 15, + "type": "KSamplerAdvancedProvider", + "pos": [ + 1719, + 592 + ], + "size": { + "0": 355.20001220703125, + "1": 154 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "basic_pipe", + "type": "BASIC_PIPE", + "link": 20 + } + ], + "outputs": [ + { + "name": "KSAMPLER_ADVANCED", + "type": "KSAMPLER_ADVANCED", + "links": [ + 23 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSamplerAdvancedProvider" + }, + "widgets_values": [ + 1107040072933062, + "fixed", + 8, + "euler", + "normal" + ] + }, + { + "id": 25, + "type": "PreviewImage", + "pos": [ + 2175, + 1079 + ], + "size": [ + 516, + 424 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 41 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + } + ], + "links": [ + [ + 3, + 4, + 1, + 6, + 0, + "CLIP" + ], + [ + 5, + 4, + 1, + 7, + 0, + "CLIP" + ], + [ + 10, + 4, + 0, + 10, + 0, + "MODEL" + ], + [ + 11, + 4, + 1, + 10, + 1, + "CLIP" + ], + [ + 12, + 4, + 2, + 10, + 2, + "VAE" + ], + [ + 13, + 6, + 0, + 10, + 3, + "CONDITIONING" + ], + [ + 14, + 7, + 0, + 10, + 4, + "CONDITIONING" + ], + [ + 15, + 10, + 0, + 11, + 0, + "BASIC_PIPE" + ], + [ + 16, + 4, + 1, + 12, + 0, + "CLIP" + ], + [ + 17, + 12, + 0, + 11, + 4, + "CONDITIONING" + ], + [ + 19, + 10, + 0, + 13, + 0, + "BASIC_PIPE" + ], + [ + 20, + 11, + 0, + 15, + 0, + "BASIC_PIPE" + ], + [ + 23, + 15, + 0, + 14, + 2, + "KSAMPLER_ADVANCED" + ], + [ + 24, + 13, + 0, + 14, + 1, + "KSAMPLER_ADVANCED" + ], + [ + 25, + 16, + 0, + 14, + 0, + "LATENT" + ], + [ + 28, + 16, + 0, + 19, + 3, + "LATENT" + ], + [ + 30, + 19, + 0, + 20, + 0, + "LATENT" + ], + [ + 31, + 4, + 2, + 20, + 1, + "VAE" + ], + [ + 33, + 10, + 0, + 22, + 0, + "BASIC_PIPE" + ], + [ + 34, + 22, + 0, + 19, + 0, + "MODEL" + ], + [ + 35, + 22, + 3, + 19, + 1, + "CONDITIONING" + ], + [ + 36, + 22, + 4, + 19, + 2, + "CONDITIONING" + ], + [ + 37, + 20, + 0, + 23, + 0, + "IMAGE" + ], + [ + 38, + 23, + 1, + 14, + 3, + "MASK" + ], + [ + 39, + 14, + 0, + 24, + 0, + "LATENT" + ], + [ + 40, + 22, + 2, + 24, + 1, + "VAE" + ], + [ + 41, + 24, + 0, + 25, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file