From eef38128d6479bd99a9178d9d4d494a0589c56cb Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" Date: Tue, 4 Jul 2023 00:38:39 +0900 Subject: [PATCH] RegionalSampler, CombineRegionalPrompts, RegionalPrompt added --- README.md | 3 + __init__.py | 4 + modules/impact/config.py | 2 +- modules/impact/core.py | 34 ++ modules/impact/impact_pack.py | 104 ++++ test/regional_prompt.json | 975 ++++++++++++++++++++++++++++++++++ 6 files changed, 1121 insertions(+), 1 deletion(-) create mode 100644 test/regional_prompt.json diff --git a/README.md b/README.md index e90e98f..28144ec 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,9 @@ This takes latent as input and outputs latent as the result. * When an image is generated with the "fixed" mode, the prompt used for that particular generation is stored in the metadata. * Known Issue: The presetText.js script from **pythongosssss's [ComfyUI-Custom-Scripts](https://github.com/pythongosssss/ComfyUI-Custom-Scripts)** is causing a conflict, preventing it from being used together. +* RegionalSampler, CombineRegionalPrompts, RegionalPrompt - experimental feature +- multiple region version of TwoAdvancedSamplersForMask + ## Feature * Interactive SAM Detector (Clipspace) - When you right-click on a node that has 'MASK' and 'IMAGE' outputs, a context menu will open. From this menu, you can either open a dialog to create a SAM Mask using 'Open in SAM Detector', or copy the content (likely mask data) using 'Copy (Clipspace)' and generate a mask using 'Impact SAM Detector' from the clipspace menu, and then paste it using 'Paste (Clipspace)'. diff --git a/__init__.py b/__init__.py index 96dc417..72e5d7a 100644 --- a/__init__.py +++ b/__init__.py @@ -163,6 +163,10 @@ NODE_CLASS_MAPPINGS = { # "SEGPick": SEGPick, # "SEGEdit": SEGEdit, + "RegionalSampler": RegionalSampler, + "CombineRegionalPrompts": CombineRegionalPrompts, + "RegionalPrompt": RegionalPrompt, + "MaskPainter": impact.legacy_nodes.MaskPainter, "MMDetLoader": impact.legacy_nodes.MMDetLoader, "SegsMaskCombine": impact.legacy_nodes.SegsMaskCombine, diff --git a/modules/impact/config.py b/modules/impact/config.py index 3312b93..e479e83 100644 --- a/modules/impact/config.py +++ b/modules/impact/config.py @@ -1,7 +1,7 @@ import configparser import os -version = "V2.20.2" +version = "V2.21" dependency_version = 1 diff --git a/modules/impact/core.py b/modules/impact/core.py index 5351828..71ef80a 100644 --- a/modules/impact/core.py +++ b/modules/impact/core.py @@ -15,11 +15,45 @@ import comfy_extras.nodes_upscale_model as model_upscale from server import PromptServer import comfy import impact.wildcards as wildcards +import math SEG = namedtuple("SEG", ['cropped_image', 'cropped_mask', 'confidence', 'crop_region', 'bbox', 'label'], defaults=[None]) +def erosion_mask(mask, grow_mask_by): + w = mask.shape[1] + h = mask.shape[0] + + mask = mask.clone() + + mask2 = torch.nn.functional.interpolate(mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])), size=(w, h), + mode="bilinear") + if grow_mask_by == 0: + mask_erosion = mask2 + else: + kernel_tensor = torch.ones((1, 1, grow_mask_by, grow_mask_by)) + padding = math.ceil((grow_mask_by - 1) / 2) + + mask_erosion = torch.clamp(torch.nn.functional.conv2d(mask2.round(), kernel_tensor, padding=padding), 0, 1) + + return mask_erosion[:, :, :w, :h].round() + + +class REGIONAL_PROMPT: + def __init__(self, mask, sampler): + self.mask = mask + self.sampler = sampler + self.mask_erosion = None + self.erosion_factor = None + + def get_mask_erosion(self, factor): + if self.mask_erosion is None or self.erosion_factor != factor: + self.mask_erosion = erosion_mask(self.mask, factor) + + return self.mask_erosion + + class NO_BBOX_DETECTOR: pass diff --git a/modules/impact/impact_pack.py b/modules/impact/impact_pack.py index f8fcc73..54a2062 100644 --- a/modules/impact/impact_pack.py +++ b/modules/impact/impact_pack.py @@ -608,6 +608,110 @@ class TwoAdvancedSamplersForMask: return (new_latent_image, ) +class RegionalPrompt: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "mask": ("MASK", ), + "advanced_sampler": ("KSAMPLER_ADVANCED", ), + }, + } + + RETURN_TYPES = ("REGIONAL_PROMPTS", ) + FUNCTION = "doit" + + CATEGORY = "ImpactPack/experimental" + + def doit(self, mask, advanced_sampler): + regional_prompt = core.REGIONAL_PROMPT(mask, advanced_sampler) + return ([regional_prompt], ) + + +class CombineRegionalPrompts: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "regional_prompts1": ("REGIONAL_PROMPTS", ), + "regional_prompts2": ("REGIONAL_PROMPTS", ), + }, + } + + RETURN_TYPES = ("REGIONAL_PROMPTS", ) + FUNCTION = "doit" + + CATEGORY = "ImpactPack/experimental" + + def doit(self, regional_prompts1, regional_prompts2): + return (regional_prompts1 + regional_prompts2, ) + + +class RegionalSampler: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), + "steps": ("INT", {"default": 20, "min": 1, "max": 10000}), + "denoise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), + "samples": ("LATENT", ), + "base_sampler": ("KSAMPLER_ADVANCED", ), + "regional_prompts": ("REGIONAL_PROMPTS", ), + "overlap_factor": ("INT", {"default": 10, "min": 0, "max": 10000}) + }, + } + + RETURN_TYPES = ("LATENT", ) + FUNCTION = "doit" + + CATEGORY = "ImpactPack/experimental" + + @staticmethod + def mask_erosion(samples, mask, grow_mask_by): + mask = mask.clone() + + w = samples['samples'].shape[3] + h = samples['samples'].shape[2] + + mask2 = torch.nn.functional.interpolate(mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])), size=(w, h), mode="bilinear") + if grow_mask_by == 0: + mask_erosion = mask2 + else: + kernel_tensor = torch.ones((1, 1, grow_mask_by, grow_mask_by)) + padding = math.ceil((grow_mask_by - 1) / 2) + + mask_erosion = torch.clamp(torch.nn.functional.conv2d(mask2.round(), kernel_tensor, padding=padding), 0, 1) + + return mask_erosion[:, :, :w, :h].round() + + def doit(self, seed, steps, denoise, samples, base_sampler, regional_prompts, overlap_factor): + + masks = [regional_prompt.mask.numpy() for regional_prompt in regional_prompts] + masks = [np.ceil(mask).astype(np.int32) for mask in masks] + combined_mask = torch.from_numpy(np.bitwise_or.reduce(masks)) + + inv_mask = torch.where(combined_mask == 0, torch.tensor(1.0), torch.tensor(0.0)) + + adv_steps = int(steps / denoise) + start_at_step = adv_steps - steps + + new_latent_image = samples.copy() + + for i in range(start_at_step, adv_steps): + add_noise = "enable" if i == start_at_step else "disable" + return_with_leftover_noise = "enable" if i+1 != adv_steps else "disable" + + new_latent_image['noise_mask'] = inv_mask + new_latent_image = base_sampler.sample_advanced(add_noise, seed, adv_steps, new_latent_image, i, i + 1, "enable") + + for regional_prompt in regional_prompts: + new_latent_image['noise_mask'] = regional_prompt.get_mask_erosion(overlap_factor) + new_latent_image = regional_prompt.sampler.sample_advanced("disable", seed, adv_steps, new_latent_image, + i, i + 1, return_with_leftover_noise) + + del new_latent_image['noise_mask'] + + return (new_latent_image, ) + + class FaceDetailer: @classmethod def INPUT_TYPES(s): diff --git a/test/regional_prompt.json b/test/regional_prompt.json new file mode 100644 index 0000000..cbc964c --- /dev/null +++ b/test/regional_prompt.json @@ -0,0 +1,975 @@ +{ + "last_node_id": 22, + "last_link_id": 28, + "nodes": [ + { + "id": 7, + "type": "VAEDecode", + "pos": [ + 2400, + 270 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 9 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 1, + "type": "LoadImage", + "pos": [ + -250, + 747 + ], + "size": { + "0": 915.1032104492188, + "1": 860.6505126953125 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": null, + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 15 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "clipspace/clipspace-mask-1650741.700000003.png [input]", + "image" + ] + }, + { + "id": 9, + "type": "EditBasicPipe", + "pos": [ + 888, + 222 + ], + "size": { + "0": 267, + "1": 126 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "basic_pipe", + "type": "BASIC_PIPE", + "link": 11 + }, + { + "name": "model", + "type": "MODEL", + "link": null + }, + { + "name": "clip", + "type": "CLIP", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 13 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + } + ], + "outputs": [ + { + "name": "basic_pipe", + "type": "BASIC_PIPE", + "links": [ + 16 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EditBasicPipe" + } + }, + { + "id": 10, + "type": "CLIPTextEncode", + "pos": [ + 460, + 242 + ], + "size": { + "0": 292.0009765625, + "1": 115.41679382324219 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 12 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 13 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "photorealistic:1.4, 1girl wear red hat" + ] + }, + { + "id": 14, + "type": "EmptyLatentImage", + "pos": [ + 1050, + 877 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 19 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 1024, + 768, + 1 + ] + }, + { + "id": 8, + "type": "PreviewImage", + "pos": [ + 2476, + 319 + ], + "size": [ + 1069.308299510546, + 1128.9237824485897 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 6, + "type": "FromBasicPipe", + "pos": [ + 167, + 238 + ], + "size": { + "0": 241.79998779296875, + "1": 106 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "basic_pipe", + "type": "BASIC_PIPE", + "link": 3 + } + ], + "outputs": [ + { + "name": "model", + "type": "MODEL", + "links": null, + "shape": 3 + }, + { + "name": "clip", + "type": "CLIP", + "links": [ + 12, + 20 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "vae", + "type": "VAE", + "links": [ + 8 + ], + "shape": 3, + "slot_index": 2 + }, + { + "name": "positive", + "type": "CONDITIONING", + "links": [], + "shape": 3, + "slot_index": 3 + }, + { + "name": "negative", + "type": "CONDITIONING", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FromBasicPipe" + } + }, + { + "id": 18, + "type": "CLIPTextEncode", + "pos": [ + 502, + 507 + ], + "size": { + "0": 292.0009765625, + "1": 115.41679382324219 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 20 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 21 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "photorealistic:1.4, 1 man wear jeans" + ] + }, + { + "id": 5, + "type": "## make-basic_pipe [2c8c61]", + "pos": [ + -227, + -290 + ], + "size": { + "0": 400, + "1": 200 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "vae_opt", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "BASIC_PIPE", + "type": "BASIC_PIPE", + "links": [ + 1, + 3, + 11, + 22 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "## make-basic_pipe", + "properties": { + "Node name for S&R": "## make-basic_pipe [2c8c61]" + }, + "widgets_values": [ + "V07_v07.safetensors", + "photorealistic:1.4, 1girl and 1man sitting in the cafe", + "" + ] + }, + { + "id": 17, + "type": "EditBasicPipe", + "pos": [ + 975, + 597 + ], + "size": { + "0": 267, + "1": 126 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "basic_pipe", + "type": "BASIC_PIPE", + "link": 22 + }, + { + "name": "model", + "type": "MODEL", + "link": null + }, + { + "name": "clip", + "type": "CLIP", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 21 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + } + ], + "outputs": [ + { + "name": "basic_pipe", + "type": "BASIC_PIPE", + "links": [ + 24 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EditBasicPipe" + } + }, + { + "id": 2, + "type": "RegionalSampler", + "pos": [ + 2050, + 200 + ], + "size": [ + 315, + 406 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 19, + "slot_index": 0 + }, + { + "name": "base_sampler", + "type": "KSAMPLER_ADVANCED", + "link": 10 + }, + { + "name": "regional_prompts", + "type": "REGIONAL_PROMPTS", + "link": 27 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 7 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "RegionalSampler" + }, + "widgets_values": [ + 212608453650633, + "randomize", + 20, + 1, + 10 + ] + }, + { + "id": 12, + "type": "RegionalPrompt", + "pos": [ + 1361, + 497 + ], + "size": { + "0": 418.1999816894531, + "1": 46 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 15 + }, + { + "name": "advanced_sampler", + "type": "KSAMPLER_ADVANCED", + "link": 17 + } + ], + "outputs": [ + { + "name": "REGIONAL_PROMPTS", + "type": "REGIONAL_PROMPTS", + "links": [ + 25 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "RegionalPrompt" + } + }, + { + "id": 20, + "type": "RegionalPrompt", + "pos": [ + 1520, + 940 + ], + "size": { + "0": 418.1999816894531, + "1": 46 + }, + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 28 + }, + { + "name": "advanced_sampler", + "type": "KSAMPLER_ADVANCED", + "link": 23 + } + ], + "outputs": [ + { + "name": "REGIONAL_PROMPTS", + "type": "REGIONAL_PROMPTS", + "links": [ + 26 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "RegionalPrompt" + } + }, + { + "id": 22, + "type": "CombineRegionalPrompts", + "pos": [ + 1851.5471568957514, + 660.6109041998288 + ], + "size": { + "0": 430.8000183105469, + "1": 46 + }, + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "regional_prompts1", + "type": "REGIONAL_PROMPTS", + "link": 25 + }, + { + "name": "regional_prompts2", + "type": "REGIONAL_PROMPTS", + "link": 26 + } + ], + "outputs": [ + { + "name": "REGIONAL_PROMPTS", + "type": "REGIONAL_PROMPTS", + "links": [ + 27 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CombineRegionalPrompts" + } + }, + { + "id": 15, + "type": "LoadImage", + "pos": [ + -278, + 1721 + ], + "size": [ + 1069.0971236588748, + 886.818909225281 + ], + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": null, + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 28 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "clipspace/clipspace-mask-1683237.5.png [input]", + "image" + ] + }, + { + "id": 13, + "type": "KSamplerAdvancedProvider", + "pos": [ + 1209, + 225 + ], + "size": { + "0": 355.20001220703125, + "1": 106 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "basic_pipe", + "type": "BASIC_PIPE", + "link": 16 + } + ], + "outputs": [ + { + "name": "KSAMPLER_ADVANCED", + "type": "KSAMPLER_ADVANCED", + "links": [ + 17 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSamplerAdvancedProvider" + }, + "widgets_values": [ + 8, + "euler", + "normal" + ] + }, + { + "id": 21, + "type": "KSamplerAdvancedProvider", + "pos": [ + 1360, + 670 + ], + "size": { + "0": 355.20001220703125, + "1": 106 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "basic_pipe", + "type": "BASIC_PIPE", + "link": 24 + } + ], + "outputs": [ + { + "name": "KSAMPLER_ADVANCED", + "type": "KSAMPLER_ADVANCED", + "links": [ + 23 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSamplerAdvancedProvider" + }, + "widgets_values": [ + 8, + "euler", + "normal" + ] + }, + { + "id": 4, + "type": "KSamplerAdvancedProvider", + "pos": [ + 1289, + -267 + ], + "size": { + "0": 355.20001220703125, + "1": 106 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "basic_pipe", + "type": "BASIC_PIPE", + "link": 1 + } + ], + "outputs": [ + { + "name": "KSAMPLER_ADVANCED", + "type": "KSAMPLER_ADVANCED", + "links": [ + 10 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSamplerAdvancedProvider" + }, + "widgets_values": [ + 8, + "euler", + "normal" + ] + } + ], + "links": [ + [ + 1, + 5, + 0, + 4, + 0, + "BASIC_PIPE" + ], + [ + 3, + 5, + 0, + 6, + 0, + "BASIC_PIPE" + ], + [ + 7, + 2, + 0, + 7, + 0, + "LATENT" + ], + [ + 8, + 6, + 2, + 7, + 1, + "VAE" + ], + [ + 9, + 7, + 0, + 8, + 0, + "IMAGE" + ], + [ + 10, + 4, + 0, + 2, + 1, + "KSAMPLER_ADVANCED" + ], + [ + 11, + 5, + 0, + 9, + 0, + "BASIC_PIPE" + ], + [ + 12, + 6, + 1, + 10, + 0, + "CLIP" + ], + [ + 13, + 10, + 0, + 9, + 4, + "CONDITIONING" + ], + [ + 15, + 1, + 1, + 12, + 0, + "MASK" + ], + [ + 16, + 9, + 0, + 13, + 0, + "BASIC_PIPE" + ], + [ + 17, + 13, + 0, + 12, + 1, + "KSAMPLER_ADVANCED" + ], + [ + 19, + 14, + 0, + 2, + 0, + "LATENT" + ], + [ + 20, + 6, + 1, + 18, + 0, + "CLIP" + ], + [ + 21, + 18, + 0, + 17, + 4, + "CONDITIONING" + ], + [ + 22, + 5, + 0, + 17, + 0, + "BASIC_PIPE" + ], + [ + 23, + 21, + 0, + 20, + 1, + "KSAMPLER_ADVANCED" + ], + [ + 24, + 17, + 0, + 21, + 0, + "BASIC_PIPE" + ], + [ + 25, + 12, + 0, + 22, + 0, + "REGIONAL_PROMPTS" + ], + [ + 26, + 20, + 0, + 22, + 1, + "REGIONAL_PROMPTS" + ], + [ + 27, + 22, + 0, + 2, + 2, + "REGIONAL_PROMPTS" + ], + [ + 28, + 15, + 1, + 20, + 0, + "MASK" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file