nodes/ltdrdata-ComfyUI-Impact-Pack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: RegionalPrompt - support variation seed

Browse Source 
refactor: convert several doit method to staticmethod
This commit is contained in:
Dr.Lt.Data
2024-06-02 17:12:25 +09:00
parent b1b68426f9
commit 8f7d07bb23
6 changed files with 117 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2 -3
View File
@@ -474,10 +474,9 @@ open-mmlab/[mmdetection](https://github.com/open-mmlab/mmdetection) - Object det
biegert/[ComfyUI-CLIPSeg](https://github.com/biegert/ComfyUI-CLIPSeg) - This is a custom node that enables the use of CLIPSeg technology, which can find segments through prompts, in ComfyUI.
BlenderNeok/[ComfyUI-TiledKSampler](https://github.com/BlenderNeko/ComfyUI_TiledKSampler) -
The tile sampler allows high-resolution sampling even in places with low GPU VRAM.
BlenderNeok/[ComfyUI-TiledKSampler](https://github.com/BlenderNeko/ComfyUI_TiledKSampler) - The tile sampler allows high-resolution sampling even in places with low GPU VRAM.
BlenderNeok/[ComfyUI_Noise](https://github.com/BlenderNeko/ComfyUI_Noise) - The noise injection feature relies on this function.
BlenderNeok/[ComfyUI_Noise](https://github.com/BlenderNeko/ComfyUI_Noise) - The noise injection feature relies on this function and slerp code for noise variation
WASasquatch/[was-node-suite-comfyui](https://github.com/WASasquatch/was-node-suite-comfyui) - A powerful custom node extensions of ComfyUI.
modules/impact/config.py
+1 -1
View File
@@ -2,7 +2,7 @@ import configparser
import os
version_code = [5, 9]
version_code = [5, 10]
version = f"V{version_code[0]}.{version_code[1]}" + (f'.{version_code[2]}' if len(version_code) > 2 else '')
dependency_version = 20
modules/impact/core.py
+55 -1
View File
@@ -6,6 +6,7 @@ import numpy
import torch
from segment_anything import SamPredictor
from comfy_extras.nodes_custom_sampler import Noise_RandomNoise
from impact.utils import *
from collections import namedtuple
import numpy as np
@@ -80,19 +81,60 @@ def erosion_mask(mask, grow_mask_by):
return mask_erosion[:, :, :w, :h].round().cpu()
# CREDIT: https://github.com/BlenderNeko/ComfyUI_Noise/blob/afb14757216257b12268c91845eac248727a55e2/nodes.py#L68
# https://discuss.pytorch.org/t/help-regarding-slerp-function-for-generative-model-sampling/32475/3
def slerp(val, low, high):
dims = low.shape
low = low.reshape(dims[0], -1)
high = high.reshape(dims[0], -1)
low_norm = low/torch.norm(low, dim=1, keepdim=True)
high_norm = high/torch.norm(high, dim=1, keepdim=True)
low_norm[low_norm != low_norm] = 0.0
high_norm[high_norm != high_norm] = 0.0
omega = torch.acos((low_norm*high_norm).sum(1))
so = torch.sin(omega)
res = (torch.sin((1.0-val)*omega)/so).unsqueeze(1)*low + (torch.sin(val*omega)/so).unsqueeze(1) * high
return res.reshape(dims)
def mix_noise(from_noise, to_noise, strength, variation_method):
if variation_method == 'slerp':
mixed_noise = slerp(strength, from_noise, to_noise)
else:
# linear
mixed_noise = (1 - strength) * from_noise + strength * to_noise
# NOTE: Since the variance of the Gaussian noise in mixed_noise has changed, it must be corrected through scaling.
scale_factor = math.sqrt((1 - strength) ** 2 + strength ** 2)
mixed_noise /= scale_factor
return mixed_noise
class REGIONAL_PROMPT:
def __init__(self, mask, sampler):
def __init__(self, mask, sampler, variation_seed=0, variation_strength=0.0, variation_method='linear'):
mask = make_2d_mask(mask)
self.mask = mask
self.sampler = sampler
self.mask_erosion = None
self.erosion_factor = None
self.variation_seed = variation_seed
self.variation_strength = variation_strength
self.variation_method = variation_method
def clone_with_sampler(self, sampler):
rp = REGIONAL_PROMPT(self.mask, sampler)
rp.mask_erosion = self.mask_erosion
rp.erosion_factor = self.erosion_factor
rp.variation_seed = self.variation_seed
rp.variation_strength = self.variation_strength
rp.variation_method = self.variation_method
return rp
def get_mask_erosion(self, factor):
@@ -102,6 +144,18 @@ class REGIONAL_PROMPT:
return self.mask_erosion
def touch_noise(self, noise):
if self.variation_strength > 0.0:
mask = utils.make_3d_mask(self.mask)
mask = utils.resize_mask(mask, (noise.shape[2], noise.shape[3])).unsqueeze(0)
regional_noise = Noise_RandomNoise(self.variation_seed).generate_noise({'samples': noise})
mixed_noise = mix_noise(noise, regional_noise, self.variation_strength, variation_method=self.variation_method)
return (mask == 1).float() * mixed_noise + (mask == 0).float() * noise
return noise
class NO_BBOX_DETECTOR:
pass
modules/impact/impact_sampling.py
+2 -2
View File
@@ -240,7 +240,7 @@ class KSamplerAdvancedWrapper:
return KSamplerAdvancedWrapper(model, cfg, sampler_name, scheduler, positive, negative, self.sampler_opt)
def sample_advanced(self, add_noise, seed, steps, latent_image, start_at_step, end_at_step, return_with_leftover_noise, hook=None,
recovery_mode="ratio additional", recovery_sampler="AUTO", recovery_sigma_ratio=1.0):
recovery_mode="ratio additional", recovery_sampler="AUTO", recovery_sigma_ratio=1.0, noise=None):
model, cfg, sampler_name, scheduler, positive, negative, sigma_factor = self.params
# steps, start_at_step, end_at_step = self.compensate_denoise(steps, start_at_step, end_at_step)
@@ -264,7 +264,7 @@ class KSamplerAdvancedWrapper:
if sigma_ratio > 0:
latent_image = separated_sample(model, add_noise, seed, steps, cfg, sampler_name, scheduler,
positive, negative, latent_image, start_at_step, end_at_step,
return_with_leftover_noise, sigma_ratio=sigma_ratio * sigma_factor, sampler_opt=self.sampler_opt)
return_with_leftover_noise, sigma_ratio=sigma_ratio * sigma_factor, sampler_opt=self.sampler_opt, noise=noise)
except ValueError as e:
if str(e) == 'sigma_min and sigma_max must not be 0':
print(f"\nWARN: sampling skipped - sigma_min and sigma_max are 0")
modules/impact/special_samplers.py
+56 -20
View File
@@ -1,5 +1,6 @@
import math
import impact.core as core
from comfy_extras.nodes_custom_sampler import Noise_RandomNoise
from impact.utils import *
from nodes import MAX_RESOLUTION
import nodes
@@ -27,7 +28,8 @@ class TiledKSamplerProvider:
CATEGORY = "ImpactPack/Sampler"
def doit(self, seed, steps, cfg, sampler_name, scheduler, denoise,
@staticmethod
def doit(seed, steps, cfg, sampler_name, scheduler, denoise,
tile_width, tile_height, tiling_strategy, basic_pipe):
model, _, _, positive, negative = basic_pipe
sampler = core.TiledKSamplerWrapper(model, seed, steps, cfg, sampler_name, scheduler, positive, negative, denoise,
@@ -54,7 +56,8 @@ class KSamplerProvider:
CATEGORY = "ImpactPack/Sampler"
def doit(self, seed, steps, cfg, sampler_name, scheduler, denoise, basic_pipe):
@staticmethod
def doit(seed, steps, cfg, sampler_name, scheduler, denoise, basic_pipe):
model, _, _, positive, negative = basic_pipe
sampler = KSamplerWrapper(model, seed, steps, cfg, sampler_name, scheduler, positive, negative, denoise)
return (sampler, )
@@ -80,7 +83,8 @@ class KSamplerAdvancedProvider:
CATEGORY = "ImpactPack/Sampler"
def doit(self, cfg, sampler_name, scheduler, basic_pipe, sigma_factor=1.0, sampler_opt=None):
@staticmethod
def doit(cfg, sampler_name, scheduler, basic_pipe, sigma_factor=1.0, sampler_opt=None):
model, _, _, positive, negative = basic_pipe
sampler = KSamplerAdvancedWrapper(model, cfg, sampler_name, scheduler, positive, negative, sampler_opt=sampler_opt, sigma_factor=sigma_factor)
return (sampler, )
@@ -102,7 +106,8 @@ class TwoSamplersForMask:
CATEGORY = "ImpactPack/Sampler"
def doit(self, latent_image, base_sampler, mask_sampler, mask):
@staticmethod
def doit(latent_image, base_sampler, mask_sampler, mask):
inv_mask = torch.where(mask != 1.0, torch.tensor(1.0), torch.tensor(0.0))
latent_image['noise_mask'] = inv_mask
@@ -154,7 +159,8 @@ class TwoAdvancedSamplersForMask:
return mask_erosion[:, :, :w, :h].round()
def doit(self, seed, steps, denoise, samples, base_sampler, mask_sampler, mask, overlap_factor):
@staticmethod
def doit(seed, steps, denoise, samples, base_sampler, mask_sampler, mask, overlap_factor):
inv_mask = torch.where(mask != 1.0, torch.tensor(1.0), torch.tensor(0.0))
@@ -184,9 +190,14 @@ class RegionalPrompt:
@classmethod
def INPUT_TYPES(s):
return {"required": {
"mask": ("MASK", ),
"advanced_sampler": ("KSAMPLER_ADVANCED", ),
},
"mask": ("MASK", ),
"advanced_sampler": ("KSAMPLER_ADVANCED", ),
},
"optional": {
"variation_seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
"variation_strength": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
"variation_method": (["linear", "slerp"],),
}
}
RETURN_TYPES = ("REGIONAL_PROMPTS", )
@@ -194,8 +205,9 @@ class RegionalPrompt:
CATEGORY = "ImpactPack/Regional"
def doit(self, mask, advanced_sampler):
regional_prompt = core.REGIONAL_PROMPT(mask, advanced_sampler)
@staticmethod
def doit(mask, advanced_sampler, variation_seed=0, variation_strength=0.0, variation_method="linear"):
regional_prompt = core.REGIONAL_PROMPT(mask, advanced_sampler, variation_seed=variation_seed, variation_strength=variation_strength, variation_method=variation_method)
return ([regional_prompt], )
@@ -212,7 +224,8 @@ class CombineRegionalPrompts:
CATEGORY = "ImpactPack/Regional"
def doit(self, **kwargs):
@staticmethod
def doit(**kwargs):
res = []
for k, v in kwargs.items():
res += v
@@ -233,7 +246,8 @@ class CombineConditionings:
CATEGORY = "ImpactPack/Util"
def doit(self, **kwargs):
@staticmethod
def doit(**kwargs):
res = []
for k, v in kwargs.items():
res += v
@@ -254,7 +268,8 @@ class ConcatConditionings:
CATEGORY = "ImpactPack/Util"
def doit(self, **kwargs):
@staticmethod
def doit(**kwargs):
conditioning_to = list(kwargs.values())[0]
for k, conditioning_from in list(kwargs.items())[1:]:
@@ -324,7 +339,8 @@ class RegionalSampler:
return mask_erosion[:, :, :w, :h].round()
def doit(self, seed, seed_2nd, seed_2nd_mode, steps, base_only_steps, denoise, samples, base_sampler, regional_prompts, overlap_factor, restore_latent,
@staticmethod
def doit(seed, seed_2nd, seed_2nd_mode, steps, base_only_steps, denoise, samples, base_sampler, regional_prompts, overlap_factor, restore_latent,
additional_mode, additional_sampler, additional_sigma_ratio, unique_id=None):
if restore_latent:
latent_compositor = nodes.NODE_CLASS_MAPPINGS['LatentCompositeMasked']()
@@ -348,7 +364,12 @@ class RegionalSampler:
if seed_2nd_mode == 'ignore':
leftover_noise = True
samples = base_sampler.sample_advanced(True, seed, adv_steps, samples, start_at_step, start_at_step + base_only_steps, leftover_noise, recovery_mode="DISABLE")
noise = Noise_RandomNoise(seed).generate_noise(samples)
for rp in regional_prompts:
noise = rp.touch_noise(noise)
samples = base_sampler.sample_advanced(True, seed, adv_steps, samples, start_at_step, start_at_step + base_only_steps, leftover_noise, recovery_mode="DISABLE", noise=noise)
if seed_2nd_mode == "seed+seed_2nd":
seed += seed_2nd
@@ -366,15 +387,20 @@ class RegionalSampler:
if not leftover_noise:
add_noise = True
noise = Noise_RandomNoise(seed).generate_noise(samples)
for rp in regional_prompts:
noise = rp.touch_noise(noise)
else:
add_noise = False
noise = None
for i in range(start_at_step+base_only_steps, adv_steps):
core.update_node_status(unique_id, f"{i}/{steps} steps | ", ((i-start_at_step)*region_len)/total)
new_latent_image['noise_mask'] = inv_mask
new_latent_image = base_sampler.sample_advanced(add_noise, seed, adv_steps, new_latent_image, i, i + 1, True,
recovery_mode=additional_mode, recovery_sampler=additional_sampler, recovery_sigma_ratio=additional_sigma_ratio)
recovery_mode=additional_mode, recovery_sampler=additional_sampler, recovery_sigma_ratio=additional_sigma_ratio, noise=noise)
if restore_latent:
if 'noise_mask' in new_latent_image:
@@ -453,7 +479,8 @@ class RegionalSamplerAdvanced:
CATEGORY = "ImpactPack/Regional"
def doit(self, add_noise, noise_seed, steps, start_at_step, end_at_step, overlap_factor, restore_latent, return_with_leftover_noise, latent_image, base_sampler, regional_prompts,
@staticmethod
def doit(add_noise, noise_seed, steps, start_at_step, end_at_step, overlap_factor, restore_latent, return_with_leftover_noise, latent_image, base_sampler, regional_prompts,
additional_mode, additional_sampler, additional_sigma_ratio, unique_id):
if restore_latent:
@@ -480,9 +507,16 @@ class RegionalSamplerAdvanced:
cur_add_noise = True if i == start_at_step and add_noise else False
if cur_add_noise:
noise = Noise_RandomNoise(noise_seed).generate_noise(new_latent_image)
for rp in regional_prompts:
noise = rp.touch_noise(noise)
else:
noise = None
new_latent_image['noise_mask'] = inv_mask
new_latent_image = base_sampler.sample_advanced(cur_add_noise, noise_seed, steps, new_latent_image, i, i + 1, True,
recovery_mode=additional_mode, recovery_sampler=additional_sampler, recovery_sigma_ratio=additional_sigma_ratio)
recovery_mode=additional_mode, recovery_sampler=additional_sampler, recovery_sigma_ratio=additional_sigma_ratio, noise=noise)
if restore_latent:
del new_latent_image['noise_mask']
@@ -555,7 +589,8 @@ class KSamplerBasicPipe:
CATEGORY = "sampling"
def sample(self, basic_pipe, seed, steps, cfg, sampler_name, scheduler, latent_image, denoise=1.0):
@staticmethod
def sample(basic_pipe, seed, steps, cfg, sampler_name, scheduler, latent_image, denoise=1.0):
model, clip, vae, positive, negative = basic_pipe
latent = impact_sample(model, seed, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise)
return basic_pipe, latent, vae
@@ -584,7 +619,8 @@ class KSamplerAdvancedBasicPipe:
CATEGORY = "sampling"
def sample(self, basic_pipe, add_noise, noise_seed, steps, cfg, sampler_name, scheduler, latent_image, start_at_step, end_at_step, return_with_leftover_noise, denoise=1.0):
@staticmethod
def sample(basic_pipe, add_noise, noise_seed, steps, cfg, sampler_name, scheduler, latent_image, start_at_step, end_at_step, return_with_leftover_noise, denoise=1.0):
model, clip, vae, positive, negative = basic_pipe
latent = separated_sample(model, add_noise, noise_seed, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, start_at_step, end_at_step, return_with_leftover_noise)
pyproject.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[project]
name = "comfyui-impact-pack"
description = "This extension offers various detector nodes and detailer nodes that allow you to configure a workflow that automatically enhances facial details. And provide iterative upscaler."
version = "5.9"
version = "5.10"
license = "LICENSE"
dependencies = ["segment-anything", "scikit-image", "piexif", "transformers", "opencv-python-headless", "GitPython", "scipy>=1.11.4"]