967 lines
56 KiB
Python
967 lines
56 KiB
Python
import math
|
|
from copy import deepcopy
|
|
from torch.nn import Upsample
|
|
from comfy.model_patcher import set_model_options_patch_replace
|
|
from comfy.ldm.modules.attention import attention_basic
|
|
import comfy.samplers
|
|
import comfy.utils
|
|
import numpy as np
|
|
import torch
|
|
import torch.nn.functional as F
|
|
from colorama import Fore, Style
|
|
import json
|
|
import os
|
|
|
|
original_sampling_function = None
|
|
current_dir = os.path.dirname(os.path.realpath(__file__))
|
|
json_preset_path = os.path.join(current_dir, 'presets')
|
|
|
|
def sampling_function_patched(model, x, timestep, uncond, cond, cond_scale, model_options={}, seed=None, **kwargs):
|
|
for fn in model_options.get("sampler_patch_model_pre_cfg_function", []):
|
|
args = {"model": model, "sigma": timestep, "model_options": model_options}
|
|
model, model_options = fn(args)
|
|
|
|
cond_copy = deepcopy(cond)
|
|
uncond_copy = deepcopy(uncond)
|
|
|
|
if "sampler_pre_cfg_function" in model_options:
|
|
uncond, cond, cond_scale = model_options["sampler_pre_cfg_function"](
|
|
sigma=timestep, uncond=uncond, cond=cond, cond_scale=cond_scale
|
|
)
|
|
|
|
if math.isclose(cond_scale, 1.0) and model_options.get("disable_cfg1_optimization", False) == False:
|
|
uncond_ = None
|
|
else:
|
|
uncond_ = uncond
|
|
|
|
conds = [cond, uncond_]
|
|
|
|
out = comfy.samplers.calc_cond_batch(model, conds, x, timestep, model_options)
|
|
cond_pred = out[0]
|
|
uncond_pred = out[1]
|
|
|
|
if "sampler_cfg_function" in model_options:
|
|
args = {"cond": x - cond_pred, "uncond": x - uncond_pred, "cond_scale": cond_scale, "timestep": timestep, "input": x, "sigma": timestep,
|
|
"cond_denoised": cond_pred, "uncond_denoised": uncond_pred, "model": model, "model_options": model_options, "cond_pos": cond_copy, "cond_neg": uncond_copy}
|
|
cfg_result = x - model_options["sampler_cfg_function"](args)
|
|
else:
|
|
cfg_result = uncond_pred + (cond_pred - uncond_pred) * cond_scale
|
|
|
|
for fn in model_options.get("sampler_post_cfg_function", []):
|
|
args = {"denoised": cfg_result, "cond": cond, "uncond": uncond_copy, "model": model, "uncond_denoised": uncond_pred, "cond_denoised": cond_pred,
|
|
"sigma": timestep, "model_options": model_options, "input": x}
|
|
cfg_result = fn(args)
|
|
|
|
return cfg_result
|
|
|
|
|
|
def monkey_patching_comfy_sampling_function():
|
|
global original_sampling_function
|
|
|
|
if original_sampling_function is None:
|
|
original_sampling_function = comfy.samplers.sampling_function
|
|
# Make sure to only patch once
|
|
if hasattr(comfy.samplers.sampling_function, '_automatic_cfg_decorated'):
|
|
return
|
|
comfy.samplers.sampling_function = sampling_function_patched
|
|
comfy.samplers.sampling_function._automatic_cfg_decorated = True # flag to check monkey patch
|
|
|
|
def make_sampler_pre_cfg_function(minimum_sigma_to_disable_uncond=0, maximum_sigma_to_enable_uncond=1000000, disabled_cond_start=10000,disabled_cond_end=10000):
|
|
def sampler_pre_cfg_function(sigma, uncond, cond, cond_scale, **kwargs):
|
|
if sigma[0] < minimum_sigma_to_disable_uncond or sigma[0] > maximum_sigma_to_enable_uncond:
|
|
uncond = None
|
|
if sigma[0] <= disabled_cond_start and sigma[0] > disabled_cond_end:
|
|
cond = None
|
|
return uncond, cond, cond_scale
|
|
return sampler_pre_cfg_function
|
|
|
|
def get_entropy(tensor):
|
|
hist = np.histogram(tensor.cpu(), bins=100)[0]
|
|
hist = hist / hist.sum()
|
|
hist = hist[hist > 0]
|
|
return -np.sum(hist * np.log2(hist))
|
|
|
|
def map_sigma(sigma, sigmax, sigmin):
|
|
return 1 + ((sigma - sigmax) * (0 - 1)) / (sigmin - sigmax)
|
|
|
|
def center_latent_mean_values(latent, per_channel, mult):
|
|
for b in range(len(latent)):
|
|
if per_channel:
|
|
for c in range(len(latent[b])):
|
|
latent[b][c] -= latent[b][c].mean() * mult
|
|
else:
|
|
latent[b] -= latent[b].mean() * mult
|
|
return latent
|
|
|
|
def get_denoised_ranges(latent, measure="hard", top_k=0.25):
|
|
chans = []
|
|
for x in range(len(latent)):
|
|
max_values = torch.topk(latent[x] - latent[x].mean() if measure == "range" else latent[x], k=int(len(latent[x])*top_k), largest=True).values
|
|
min_values = torch.topk(latent[x] - latent[x].mean() if measure == "range" else latent[x], k=int(len(latent[x])*top_k), largest=False).values
|
|
max_val = torch.mean(max_values).item()
|
|
min_val = abs(torch.mean(min_values).item()) if measure == "soft" else torch.mean(torch.abs(min_values)).item()
|
|
denoised_range = (max_val + min_val) / 2
|
|
chans.append(denoised_range**2 if measure == "hard_squared" else denoised_range)
|
|
return chans
|
|
|
|
def get_sigmin_sigmax(model):
|
|
model_sampling = model.model.model_sampling
|
|
sigmin = model_sampling.sigma(model_sampling.timestep(model_sampling.sigma_min))
|
|
sigmax = model_sampling.sigma(model_sampling.timestep(model_sampling.sigma_max))
|
|
return sigmin, sigmax
|
|
|
|
def gaussian_similarity(x, y, sigma=1.0):
|
|
diff = (x - y) ** 2
|
|
return torch.exp(-diff / (2 * sigma ** 2))
|
|
|
|
def check_skip(sigma, high_sigma_threshold, low_sigma_threshold):
|
|
return sigma > high_sigma_threshold or sigma < low_sigma_threshold
|
|
|
|
def max_abs(tensors):
|
|
shape = tensors.shape
|
|
tensors = tensors.reshape(shape[0], -1)
|
|
tensors_abs = torch.abs(tensors)
|
|
max_abs_idx = torch.argmax(tensors_abs, dim=0)
|
|
result = tensors[max_abs_idx, torch.arange(tensors.shape[1])]
|
|
return result.reshape(shape[1:])
|
|
|
|
def gaussian_kernel(size, sigma):
|
|
ax = torch.arange(-size // 2 + 1., size // 2 + 1.)
|
|
xx, yy = torch.meshgrid(ax, ax, indexing='ij')
|
|
kernel = torch.exp(-(xx**2 + yy**2) / (2. * sigma**2))
|
|
return kernel / kernel.sum()
|
|
|
|
def blur_tensor(input_tensor,kernel_size = 7, sigma = 2.0):
|
|
device = input_tensor.device
|
|
x_coord = torch.arange(kernel_size) - (kernel_size - 1) / 2
|
|
xy_grid = torch.square(x_coord.repeat(kernel_size).view(kernel_size, kernel_size)) + torch.square(x_coord.repeat(kernel_size).view(kernel_size, kernel_size).t())
|
|
gaussian_kernel = torch.exp(-xy_grid / (2 * sigma ** 2)) / torch.sum(torch.exp(-xy_grid / (2 * sigma ** 2)))
|
|
gaussian_kernel = gaussian_kernel.to(device).to(input_tensor.dtype)
|
|
gaussian_kernel = gaussian_kernel.view(1, 1, kernel_size, kernel_size).repeat(1, 1, 1, 1)
|
|
return F.conv2d(input_tensor.unsqueeze(0), gaussian_kernel, padding=int(kernel_size/2), groups=1).squeeze(0).to(device).to(input_tensor.dtype)
|
|
|
|
def smallest_distances(tensors):
|
|
if all(torch.equal(tensors[0], tensor) for tensor in tensors[1:]):
|
|
return tensors[0]
|
|
set_device = tensors.device
|
|
min_val = torch.full(tensors[0].shape, float("inf")).to(set_device)
|
|
result = torch.zeros_like(tensors[0])
|
|
for idx1, t1 in enumerate(tensors):
|
|
temp_diffs = torch.zeros_like(tensors[0])
|
|
for idx2, t2 in enumerate(tensors):
|
|
if idx1 != idx2:
|
|
temp_diffs += torch.abs(torch.sub(t1, t2))
|
|
min_val = torch.minimum(min_val, temp_diffs)
|
|
mask = torch.eq(min_val,temp_diffs)
|
|
result[mask] = t1[mask]
|
|
return result
|
|
|
|
# def rescale(h,downscale_factor=2,downscale_method="bicubic"):
|
|
# return comfy.utils.common_upscale(h, round(h.shape[-1] * (1.0 / downscale_factor)), round(h.shape[-2] * (1.0 / downscale_factor)), downscale_method, "disabled")
|
|
|
|
def rescale(tensor, multiplier=2):
|
|
batch, seq_length, features = tensor.shape
|
|
H = W = int(seq_length**0.5)
|
|
tensor_reshaped = tensor.view(batch, features, H, W)
|
|
new_H = new_W = int(H * multiplier)
|
|
resized_tensor = F.interpolate(tensor_reshaped, size=(new_H, new_W), mode='bilinear', align_corners=False)
|
|
return resized_tensor.view(batch, new_H * new_W, features)
|
|
|
|
# from https://discuss.pytorch.org/t/help-regarding-slerp-function-for-generative-model-sampling/32475
|
|
def slerp(high, low, val):
|
|
dims = low.shape
|
|
|
|
#flatten to batches
|
|
low = low.reshape(dims[0], -1)
|
|
high = high.reshape(dims[0], -1)
|
|
|
|
low_norm = low/torch.norm(low, dim=1, keepdim=True)
|
|
high_norm = high/torch.norm(high, dim=1, keepdim=True)
|
|
|
|
# in case we divide by zero
|
|
low_norm[low_norm != low_norm] = 0.0
|
|
high_norm[high_norm != high_norm] = 0.0
|
|
|
|
omega = torch.acos((low_norm*high_norm).sum(1))
|
|
so = torch.sin(omega)
|
|
res = (torch.sin((1.0-val)*omega)/so).unsqueeze(1)*low + (torch.sin(val*omega)/so).unsqueeze(1) * high
|
|
return res.reshape(dims)
|
|
|
|
normalize_tensor = lambda x: x / x.norm()
|
|
|
|
def random_swap(tensors, proportion=1):
|
|
num_tensors = tensors.shape[0]
|
|
if num_tensors < 2: return tensors[0],0
|
|
tensor_size = tensors[0].numel()
|
|
if tensor_size < 100: return tensors[0],0
|
|
|
|
true_count = int(tensor_size * proportion)
|
|
mask = torch.cat((torch.ones(true_count, dtype=torch.bool, device=tensors[0].device),
|
|
torch.zeros(tensor_size - true_count, dtype=torch.bool, device=tensors[0].device)))
|
|
mask = mask[torch.randperm(tensor_size)].reshape(tensors[0].shape)
|
|
if num_tensors == 2 and proportion < 1:
|
|
index_tensor = torch.ones_like(tensors[0], dtype=torch.int64, device=tensors[0].device)
|
|
else:
|
|
index_tensor = torch.randint(1 if proportion < 1 else 0, num_tensors, tensors[0].shape, device=tensors[0].device)
|
|
for i, t in enumerate(tensors):
|
|
if i == 0: continue
|
|
merge_mask = index_tensor == i & mask
|
|
tensors[0][merge_mask] = t[merge_mask]
|
|
return tensors[0]
|
|
|
|
def multi_tensor_check_mix(tensors):
|
|
if tensors[0].numel() < 2 or len(tensors) < 2:
|
|
return tensors[0]
|
|
ref_tensor_shape = tensors[0].shape
|
|
sequence_tensor = torch.arange(tensors[0].numel(), device=tensors[0].device) % len(tensors)
|
|
reshaped_sequence = sequence_tensor.view(ref_tensor_shape)
|
|
for i in range(len(tensors)):
|
|
if i == 0: continue
|
|
mask = reshaped_sequence == i
|
|
tensors[0][mask] = tensors[i][mask]
|
|
return tensors[0]
|
|
|
|
# I asked GPT-4 for a function to deal with the attention and it made this. It kinda sorta works.
|
|
def normal_attention(q, k, v, mask=None):
|
|
attention_scores = torch.matmul(q, k.transpose(-2, -1))
|
|
d_k = k.size(-1)
|
|
attention_scores = attention_scores / torch.sqrt(torch.tensor(d_k, dtype=torch.float32))
|
|
if mask is not None:
|
|
attention_scores = attention_scores.masked_fill(mask == 0, float('-inf'))
|
|
attention_weights = F.softmax(attention_scores, dim=-1)
|
|
output = torch.matmul(attention_weights, v)
|
|
return output
|
|
|
|
def sspow(input_tensor, p=2):
|
|
return input_tensor.abs().pow(p) * input_tensor.sign()
|
|
|
|
def gradient_merge(tensor1, tensor2, start_value=0, dim=0):
|
|
if torch.numel(tensor1) <= 1: return tensor1
|
|
if dim >= tensor1.dim(): dim = 0
|
|
size = tensor1.size(dim)
|
|
alpha = torch.linspace(start_value, 1-start_value, steps=size, device=tensor1.device).view([-1 if i == dim else 1 for i in range(tensor1.dim())])
|
|
return tensor1 * alpha + tensor2 * (1 - alpha)
|
|
|
|
# eval() results share pointers and would therefore apply the same formula to all layers.
|
|
class attention_modifier():
|
|
def __init__(self, self_attn_mod_eval, conds = None):
|
|
self.self_attn_mod_eval = self_attn_mod_eval
|
|
self.conds = conds
|
|
def modified_attention(self, q, k, v, extra_options, mask=None):
|
|
if "attnbc" in self.self_attn_mod_eval:
|
|
attnbc = attention_basic(q, k, v, extra_options['n_heads'], mask)
|
|
if "normattn" in self.self_attn_mod_eval:
|
|
normattn = normal_attention(q, k, v, mask)
|
|
if self.conds is not None:
|
|
cond_pos_l = self.conds[0][..., :768].cuda()
|
|
cond_neg_l = self.conds[1][..., :768].cuda()
|
|
if self.conds[0].shape[-1] > 768:
|
|
cond_pos_g = self.conds[0][..., 768:2048].cuda()
|
|
cond_neg_g = self.conds[1][..., 768:2048].cuda()
|
|
return eval(self.self_attn_mod_eval)
|
|
|
|
def experimental_functions(cond_input, method, exp_value, exp_normalize, pcp, psi, sigma, sigmax, attention_modifiers_input, args, model_options_copy, eval_string = ""):
|
|
"""
|
|
There may or may not be an actual reasoning behind each of these methods.
|
|
Some like the sine value have interesting properties. Enabled for both cond and uncond preds it somehow make them stronger.
|
|
Note that there is a "normalize" toggle and it may change greatly the end result since some operation will totaly butcher the values.
|
|
"theDaRkNeSs" for example without normalizing seems to darken if used for cond/uncond (not with the cond as the uncond or something).
|
|
Maybe just with the positive. I don't remember. I leave it for now if you want to play around.
|
|
|
|
The eval_string can be used to create the uncond replacement.
|
|
I made it so it's split by semicolons and only the last split is the value in used.
|
|
What is before is added in an array named "v".
|
|
pcp is previous cond_pred
|
|
psi is previous sigma
|
|
args is the CFG function input arguments with the added cond/unconds (like the actual activation conditionings) named respectively "cond_pos" and "cond_neg"
|
|
|
|
So if you write:
|
|
|
|
pcp if sigma < 7 else -pcp;
|
|
print("it works too just don't use the output I guess");
|
|
v[0] if sigma < 14 else torch.zeros_like(cond);
|
|
v[-1]*2
|
|
|
|
Well the first line becomes v[0], second v[1] etc.
|
|
The last one becomes the result.
|
|
Note that it's just an example, I don't see much interest in that one.
|
|
|
|
Using comfy.samplers.calc_cond_batch(args["model"], [args["cond_pos"], None], args["input"], args["timestep"], args["model_options"])[0]
|
|
can work too.
|
|
|
|
This whole mess has for initial goal to attempt to find the best way (or have some bruteforcing fun) to replace the uncond pred for as much as possible.
|
|
Or simply to try things around :)
|
|
"""
|
|
if method == "cond_pred":
|
|
return cond_input
|
|
default_device = cond_input.device
|
|
# print()
|
|
# print(get_entropy(cond))
|
|
cond = cond_input.clone()
|
|
cond_norm = cond.norm()
|
|
if method == "amplify":
|
|
mask = torch.abs(cond) >= 1
|
|
cond_copy = cond.clone()
|
|
cond = torch.pow(torch.abs(cond), ( 1 / exp_value)) * cond.sign()
|
|
cond[mask] = torch.pow(torch.abs(cond_copy[mask]), exp_value) * cond[mask].sign()
|
|
elif method == "root":
|
|
cond = torch.pow(torch.abs(cond), ( 1 / exp_value)) * cond.sign()
|
|
elif method == "power":
|
|
cond = torch.pow(torch.abs(cond), exp_value) * cond.sign()
|
|
elif method == "erf":
|
|
cond = torch.erf(cond)
|
|
elif method == "exp_erf":
|
|
cond = torch.pow(torch.erf(cond), exp_value)
|
|
elif method == "root_erf":
|
|
cond = torch.erf(cond)
|
|
cond = torch.pow(torch.abs(cond), 1 / exp_value ) * cond.sign()
|
|
elif method == "erf_amplify":
|
|
cond = torch.erf(cond)
|
|
mask = torch.abs(cond) >= 1
|
|
cond_copy = cond.clone()
|
|
cond = torch.pow(torch.abs(cond), 1 / exp_value ) * cond.sign()
|
|
cond[mask] = torch.pow(torch.abs(cond_copy[mask]), exp_value) * cond[mask].sign()
|
|
elif method == "sine":
|
|
cond = torch.sin(torch.abs(cond)) * cond.sign()
|
|
elif method == "sine_exp":
|
|
cond = torch.sin(torch.abs(cond)) * cond.sign()
|
|
cond = torch.pow(torch.abs(cond), exp_value) * cond.sign()
|
|
elif method == "sine_exp_diff":
|
|
cond = torch.sin(torch.abs(cond)) * cond.sign()
|
|
cond = torch.pow(torch.abs(cond_input), exp_value) * cond.sign() - cond
|
|
elif method == "sine_exp_diff_to_sine":
|
|
cond = torch.sin(torch.abs(cond)) * cond.sign()
|
|
cond = torch.pow(torch.abs(cond), exp_value) * cond.sign() - cond
|
|
elif method == "sine_root":
|
|
cond = torch.sin(torch.abs(cond)) * cond.sign()
|
|
cond = torch.pow(torch.abs(cond), ( 1 / exp_value)) * cond.sign()
|
|
elif method == "sine_root_diff":
|
|
cond = torch.sin(torch.abs(cond)) * cond.sign()
|
|
cond = torch.pow(torch.abs(cond_input), 1 / exp_value) * cond.sign() - cond
|
|
elif method == "sine_root_diff_to_sine":
|
|
cond = torch.sin(torch.abs(cond)) * cond.sign()
|
|
cond = torch.pow(torch.abs(cond), 1 / exp_value) * cond.sign() - cond
|
|
elif method == "theDaRkNeSs":
|
|
cond = torch.sin(cond)
|
|
cond = torch.pow(torch.abs(cond), 1 / exp_value) * cond.sign() - cond
|
|
elif method == "cosine":
|
|
cond = torch.cos(torch.abs(cond)) * cond.sign()
|
|
elif method == "sign":
|
|
cond = cond.sign()
|
|
elif method == "zero":
|
|
cond = torch.zeros_like(cond)
|
|
elif method in ["attention_modifiers_input_using_cond","attention_modifiers_input_using_uncond","subtract_attention_modifiers_input_using_cond","subtract_attention_modifiers_input_using_uncond"]:
|
|
cond_to_use = args["cond_pos"] if method in ["attention_modifiers_input_using_cond","subtract_attention_modifiers_input_using_cond"] else args["cond_neg"]
|
|
tmp_model_options = deepcopy(model_options_copy)
|
|
for atm in attention_modifiers_input:
|
|
if sigma <= atm['sigma_start'] and sigma > atm['sigma_end']:
|
|
block_layers = {"input": atm['unet_block_id_input'], "middle": atm['unet_block_id_middle'], "output": atm['unet_block_id_output']}
|
|
for unet_block in block_layers:
|
|
for unet_block_id in block_layers[unet_block].split(","):
|
|
if unet_block_id != "":
|
|
unet_block_id = int(unet_block_id)
|
|
tmp_model_options = set_model_options_patch_replace(tmp_model_options, attention_modifier(atm['self_attn_mod_eval'], [args["cond_pos"][0]["cross_attn"], args["cond_neg"][0]["cross_attn"]]if "cond" in atm['self_attn_mod_eval'] else None).modified_attention, atm['unet_attn'], unet_block, unet_block_id)
|
|
|
|
cond = comfy.samplers.calc_cond_batch(args["model"], [cond_to_use], args["input"], args["timestep"], tmp_model_options)[0]
|
|
if method in ["subtract_attention_modifiers_input_using_cond","subtract_attention_modifiers_input_using_uncond"]:
|
|
cond = cond_input + (cond_input - cond) * exp_value
|
|
|
|
elif method == "previous_average":
|
|
if sigma > (sigmax - 1):
|
|
cond = torch.zeros_like(cond)
|
|
else:
|
|
cond = (pcp / psi * sigma + cond) / 2
|
|
elif method == "eval":
|
|
if "condmix" in eval_string:
|
|
def condmix(args, mult=2):
|
|
cond_pos_tmp = deepcopy(args["cond_pos"])
|
|
cond_pos_tmp[0]["cross_attn"] += (args["cond_pos"][0]["cross_attn"] - args["cond_neg"][0]["cross_attn"]*-1) * mult
|
|
return cond_pos_tmp
|
|
v = []
|
|
evals_strings = eval_string.split(";")
|
|
if len(evals_strings) > 1:
|
|
for i in range(len(evals_strings[:-1])):
|
|
v.append(eval(evals_strings[i]))
|
|
cond = eval(evals_strings[-1])
|
|
if exp_normalize and torch.all(cond != 0):
|
|
cond = cond * cond_norm / cond.norm()
|
|
# print(get_entropy(cond))
|
|
return cond.to(device=default_device)
|
|
|
|
class advancedDynamicCFG:
|
|
def __init__(self):
|
|
self.last_cfg_ht_one = 8
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"model": ("MODEL",),
|
|
|
|
"automatic_cfg" : (["None", "soft", "hard", "hard_squared", "range"], {"default": "hard"},),
|
|
|
|
"skip_uncond" : ("BOOLEAN", {"default": True}),
|
|
"fake_uncond_start" : ("BOOLEAN", {"default": False}),
|
|
"uncond_sigma_start": ("FLOAT", {"default": 1000, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"uncond_sigma_end": ("FLOAT", {"default": 1, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
|
|
"lerp_uncond" : ("BOOLEAN", {"default": False}),
|
|
"lerp_uncond_strength": ("FLOAT", {"default": 2, "min": 0.0, "max": 10.0, "step": 0.1, "round": 0.1}),
|
|
"lerp_uncond_sigma_start": ("FLOAT", {"default": 1000, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"lerp_uncond_sigma_end": ("FLOAT", {"default": 1, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
|
|
"subtract_latent_mean" : ("BOOLEAN", {"default": False}),
|
|
"subtract_latent_mean_sigma_start": ("FLOAT", {"default": 1000, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"subtract_latent_mean_sigma_end": ("FLOAT", {"default": 1, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
|
|
"latent_intensity_rescale" : ("BOOLEAN", {"default": False}),
|
|
"latent_intensity_rescale_method" : (["soft","hard","range"], {"default": "hard"},),
|
|
"latent_intensity_rescale_cfg": ("FLOAT", {"default": 8, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.1}),
|
|
"latent_intensity_rescale_sigma_start": ("FLOAT", {"default": 1000, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"latent_intensity_rescale_sigma_end": ("FLOAT", {"default": 3, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
|
|
"cond_exp": ("BOOLEAN", {"default": False}),
|
|
"cond_exp_normalize": ("BOOLEAN", {"default": False}),
|
|
"cond_exp_sigma_start": ("FLOAT", {"default": 1000, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"cond_exp_sigma_end": ("FLOAT", {"default": 1, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"cond_exp_method": (["amplify", "root", "power", "erf", "erf_amplify", "exp_erf", "root_erf", "sine", "sine_exp", "sine_exp_diff", "sine_exp_diff_to_sine", "sine_root", "sine_root_diff", "sine_root_diff_to_sine", "theDaRkNeSs", "cosine", "sign", "zero", "previous_average", "eval",
|
|
"attention_modifiers_input_using_cond","attention_modifiers_input_using_uncond",
|
|
"subtract_attention_modifiers_input_using_cond","subtract_attention_modifiers_input_using_uncond"],),
|
|
"cond_exp_value": ("FLOAT", {"default": 2, "min": 0, "max": 100, "step": 0.1, "round": 0.01}),
|
|
|
|
"uncond_exp": ("BOOLEAN", {"default": False}),
|
|
"uncond_exp_normalize": ("BOOLEAN", {"default": False}),
|
|
"uncond_exp_sigma_start": ("FLOAT", {"default": 1000, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"uncond_exp_sigma_end": ("FLOAT", {"default": 1, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"uncond_exp_method": (["amplify", "root", "power", "erf", "erf_amplify", "exp_erf", "root_erf", "sine", "sine_exp", "sine_exp_diff", "sine_exp_diff_to_sine", "sine_root", "sine_root_diff", "sine_root_diff_to_sine", "theDaRkNeSs", "cosine", "sign", "zero", "previous_average", "eval",
|
|
"subtract_attention_modifiers_input_using_cond","subtract_attention_modifiers_input_using_uncond"],),
|
|
"uncond_exp_value": ("FLOAT", {"default": 2, "min": 0, "max": 100, "step": 0.1, "round": 0.01}),
|
|
|
|
"fake_uncond_exp": ("BOOLEAN", {"default": False}),
|
|
"fake_uncond_exp_normalize": ("BOOLEAN", {"default": False}),
|
|
"fake_uncond_exp_method" : (["cond_pred", "previous_average",
|
|
"amplify", "root", "power", "erf", "erf_amplify", "exp_erf", "root_erf", "sine", "sine_exp", "sine_exp_diff", "sine_exp_diff_to_sine", "sine_root", "sine_root_diff",
|
|
"sine_root_diff_to_sine", "theDaRkNeSs", "cosine", "sign", "zero", "eval",
|
|
"subtract_attention_modifiers_input_using_cond","subtract_attention_modifiers_input_using_uncond",
|
|
"attention_modifiers_input_using_cond","attention_modifiers_input_using_uncond"],),
|
|
"fake_uncond_exp_value": ("FLOAT", {"default": 2, "min": 0, "max": 1000, "step": 0.1, "round": 0.01}),
|
|
"fake_uncond_multiplier": ("INT", {"default": 1, "min": -1, "max": 1, "step": 1}),
|
|
"fake_uncond_sigma_start": ("FLOAT", {"default": 1000, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"fake_uncond_sigma_end": ("FLOAT", {"default": 1, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"auto_cfg_topk": ("FLOAT", {"default": 0.25, "min": 0.0, "max": 0.5, "step": 0.05, "round": 0.01}),
|
|
"auto_cfg_ref": ("FLOAT", {"default": 8, "min": 0.0, "max": 100, "step": 0.5, "round": 0.01}),
|
|
"attention_modifiers_global_enabled": ("BOOLEAN", {"default": False}),
|
|
"disable_cond": ("BOOLEAN", {"default": False}),
|
|
"disable_cond_sigma_start": ("FLOAT", {"default": 1000, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"disable_cond_sigma_end": ("FLOAT", {"default": 0, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"save_as_preset": ("BOOLEAN", {"default": False}),
|
|
"preset_name": ("STRING", {"multiline": False}),
|
|
},
|
|
"optional":{
|
|
"eval_string_cond": ("STRING", {"multiline": True}),
|
|
"eval_string_uncond": ("STRING", {"multiline": True}),
|
|
"eval_string_fake": ("STRING", {"multiline": True}),
|
|
"args_filter": ("STRING", {"multiline": True, "forceInput": True}),
|
|
"attention_modifiers_positive": ("ATTNMOD", {"forceInput": True}),
|
|
"attention_modifiers_negative": ("ATTNMOD", {"forceInput": True}),
|
|
"attention_modifiers_fake_negative": ("ATTNMOD", {"forceInput": True}),
|
|
"attention_modifiers_global": ("ATTNMOD", {"forceInput": True}),
|
|
}
|
|
}
|
|
RETURN_TYPES = ("MODEL","STRING",)
|
|
FUNCTION = "patch"
|
|
|
|
CATEGORY = "model_patches/automatic_cfg"
|
|
|
|
def patch(self, model, automatic_cfg = "None",
|
|
skip_uncond = False, fake_uncond_start = False, uncond_sigma_start = 1000, uncond_sigma_end = 0,
|
|
lerp_uncond = False, lerp_uncond_strength = 1, lerp_uncond_sigma_start = 1000, lerp_uncond_sigma_end = 1,
|
|
subtract_latent_mean = False, subtract_latent_mean_sigma_start = 1000, subtract_latent_mean_sigma_end = 1,
|
|
latent_intensity_rescale = False, latent_intensity_rescale_sigma_start = 1000, latent_intensity_rescale_sigma_end = 1,
|
|
cond_exp = False, cond_exp_sigma_start = 1000, cond_exp_sigma_end = 1000, cond_exp_method = "amplify", cond_exp_value = 2, cond_exp_normalize = False,
|
|
uncond_exp = False, uncond_exp_sigma_start = 1000, uncond_exp_sigma_end = 1000, uncond_exp_method = "amplify", uncond_exp_value = 2, uncond_exp_normalize = False,
|
|
fake_uncond_exp = False, fake_uncond_exp_method = "amplify", fake_uncond_exp_value = 2, fake_uncond_exp_normalize = False, fake_uncond_multiplier = 1, fake_uncond_sigma_start = 1000, fake_uncond_sigma_end = 1,
|
|
latent_intensity_rescale_cfg = 8, latent_intensity_rescale_method = "hard",
|
|
ignore_pre_cfg_func = False, args_filter = "", auto_cfg_topk = 0.25, auto_cfg_ref = 8,
|
|
eval_string_cond = "", eval_string_uncond = "", eval_string_fake = "",
|
|
attention_modifiers_global_enabled = False,
|
|
attention_modifiers_positive = [], attention_modifiers_negative = [], attention_modifiers_fake_negative = [], attention_modifiers_global = [],
|
|
disable_cond=False, disable_cond_sigma_start=1000,disable_cond_sigma_end=1000, save_as_preset = False, preset_name = "", **kwargs
|
|
):
|
|
|
|
model_options_copy = deepcopy(model.model_options)
|
|
monkey_patching_comfy_sampling_function()
|
|
if args_filter != "":
|
|
args_filter = args_filter.split(",")
|
|
else:
|
|
args_filter = [k for k, v in locals().items()]
|
|
not_in_filter = ['self','model','args','args_filter','save_as_preset','preset_name','model_options_copy']
|
|
if fake_uncond_exp_method != "eval":
|
|
not_in_filter.append("eval_string")
|
|
|
|
if save_as_preset and preset_name != "":
|
|
preset_parameters = {key: value for key, value in locals().items() if key not in not_in_filter}
|
|
with open(os.path.join(json_preset_path, preset_name+".json"), 'w', encoding='utf-8') as f:
|
|
json.dump(preset_parameters, f)
|
|
print(f"Preset saved with the name: {Fore.GREEN}{preset_name}{Fore.RESET}")
|
|
print(f"{Fore.RED}Don't forget to turn the save toggle OFF to not overwrite!{Fore.RESET}")
|
|
|
|
args_str = '\n'.join(f'{k}: {v}' for k, v in locals().items() if k not in not_in_filter and k in args_filter)
|
|
|
|
sigmin, sigmax = get_sigmin_sigmax(model)
|
|
|
|
lerp_start, lerp_end = lerp_uncond_sigma_start, lerp_uncond_sigma_end
|
|
subtract_start, subtract_end = subtract_latent_mean_sigma_start, subtract_latent_mean_sigma_end
|
|
rescale_start, rescale_end = latent_intensity_rescale_sigma_start, latent_intensity_rescale_sigma_end
|
|
print(f"Model maximum sigma: {sigmax} / Model minimum sigma: {sigmin}")
|
|
m = model.clone()
|
|
|
|
if skip_uncond or disable_cond:
|
|
# set model_options sampler_pre_cfg_function
|
|
m.model_options["sampler_pre_cfg_function"] = make_sampler_pre_cfg_function(uncond_sigma_end if skip_uncond else 0, uncond_sigma_start if skip_uncond else 100000,\
|
|
disable_cond_sigma_start if disable_cond else 100000, disable_cond_sigma_end if disable_cond else 100000)
|
|
print(f"Sampling function patched. Uncond enabled from {round(uncond_sigma_start,2)} to {round(uncond_sigma_end,2)}")
|
|
elif not ignore_pre_cfg_func:
|
|
m.model_options.pop("sampler_pre_cfg_function", None)
|
|
uncond_sigma_start, uncond_sigma_end = 1000000, 0
|
|
|
|
top_k = auto_cfg_topk
|
|
previous_cond_pred = None
|
|
previous_sigma = None
|
|
def automatic_cfg_function(args):
|
|
nonlocal previous_cond_pred, previous_sigma
|
|
cond_scale = args["cond_scale"]
|
|
input_x = args["input"]
|
|
cond_pred = args["cond_denoised"]
|
|
uncond_pred = args["uncond_denoised"]
|
|
sigma = args["sigma"][0]
|
|
model_options = args["model_options"]
|
|
if previous_cond_pred is None:
|
|
previous_cond_pred = deepcopy(cond_pred)
|
|
if previous_sigma is None:
|
|
previous_sigma = sigma.item()
|
|
reference_cfg = auto_cfg_ref if auto_cfg_ref > 0 else cond_scale
|
|
|
|
def fake_uncond_step():
|
|
return fake_uncond_start and skip_uncond and (sigma > uncond_sigma_start or sigma < uncond_sigma_end) and sigma <= fake_uncond_sigma_start and sigma >= fake_uncond_sigma_end
|
|
|
|
if fake_uncond_step():
|
|
uncond_pred = cond_pred.clone() * fake_uncond_multiplier
|
|
|
|
if cond_exp and sigma <= cond_exp_sigma_start and sigma >= cond_exp_sigma_end:
|
|
cond_pred = experimental_functions(cond_pred, cond_exp_method, cond_exp_value, cond_exp_normalize, previous_cond_pred, previous_sigma, sigma.item(), sigmax, attention_modifiers_positive, args, model_options_copy, eval_string_cond)
|
|
if uncond_exp and sigma <= uncond_exp_sigma_start and sigma >= uncond_exp_sigma_end and not fake_uncond_step():
|
|
uncond_pred = experimental_functions(uncond_pred, uncond_exp_method, uncond_exp_value, uncond_exp_normalize, previous_cond_pred, previous_sigma, sigma.item(), sigmax, attention_modifiers_negative, args, model_options_copy, eval_string_uncond)
|
|
if fake_uncond_step() and fake_uncond_exp:
|
|
uncond_pred = experimental_functions(uncond_pred, fake_uncond_exp_method, fake_uncond_exp_value, fake_uncond_exp_normalize, previous_cond_pred, previous_sigma, sigma.item(), sigmax, attention_modifiers_fake_negative, args, model_options_copy, eval_string_fake)
|
|
previous_cond_pred = deepcopy(cond_pred)
|
|
|
|
if sigma >= sigmax or cond_scale > 1:
|
|
self.last_cfg_ht_one = cond_scale
|
|
target_intensity = self.last_cfg_ht_one / 10
|
|
|
|
if ((check_skip(sigma, uncond_sigma_start, uncond_sigma_end) and skip_uncond) and not fake_uncond_step()) or cond_scale == 1:
|
|
return input_x - cond_pred
|
|
|
|
if lerp_uncond and not check_skip(sigma, lerp_start, lerp_end) and lerp_uncond_strength != 1:
|
|
uncond_pred_norm = uncond_pred.norm()
|
|
uncond_pred = torch.lerp(cond_pred, uncond_pred, lerp_uncond_strength)
|
|
uncond_pred = uncond_pred * uncond_pred_norm / uncond_pred.norm()
|
|
cond = input_x - cond_pred
|
|
uncond = input_x - uncond_pred
|
|
|
|
if automatic_cfg == "None":
|
|
return uncond + cond_scale * (cond - uncond)
|
|
|
|
denoised_tmp = input_x - (uncond + reference_cfg * (cond - uncond))
|
|
|
|
for b in range(len(denoised_tmp)):
|
|
denoised_ranges = get_denoised_ranges(denoised_tmp[b], automatic_cfg, top_k)
|
|
for c in range(len(denoised_tmp[b])):
|
|
fixeds_scale = reference_cfg * target_intensity / denoised_ranges[c]
|
|
denoised_tmp[b][c] = uncond[b][c] + fixeds_scale * (cond[b][c] - uncond[b][c])
|
|
|
|
return denoised_tmp
|
|
|
|
def center_mean_latent_post_cfg(args):
|
|
denoised = args["denoised"]
|
|
sigma = args["sigma"][0]
|
|
if check_skip(sigma, subtract_start, subtract_end):
|
|
return denoised
|
|
denoised = center_latent_mean_values(denoised, False, 1)
|
|
return denoised
|
|
|
|
def rescale_post_cfg(args):
|
|
denoised = args["denoised"]
|
|
sigma = args["sigma"][0]
|
|
|
|
if check_skip(sigma, rescale_start, rescale_end):
|
|
return denoised
|
|
target_intensity = latent_intensity_rescale_cfg / 10
|
|
for b in range(len(denoised)):
|
|
denoised_ranges = get_denoised_ranges(denoised[b], latent_intensity_rescale_method)
|
|
for c in range(len(denoised[b])):
|
|
scale_correction = target_intensity / denoised_ranges[c]
|
|
denoised[b][c] = denoised[b][c] * scale_correction
|
|
return denoised
|
|
|
|
tmp_model_options = deepcopy(m.model_options)
|
|
if attention_modifiers_global_enabled:
|
|
print(f"{Fore.GREEN}Sigma timings are ignored for global modifiers.{Fore.RESET}")
|
|
for atm in attention_modifiers_global:
|
|
block_layers = {"input": atm['unet_block_id_input'], "middle": atm['unet_block_id_middle'], "output": atm['unet_block_id_output']}
|
|
for unet_block in block_layers:
|
|
for unet_block_id in block_layers[unet_block].split(","):
|
|
if unet_block_id != "":
|
|
unet_block_id = int(unet_block_id)
|
|
tmp_model_options = set_model_options_patch_replace(tmp_model_options, attention_modifier(atm['self_attn_mod_eval']).modified_attention, atm['unet_attn'], unet_block, unet_block_id)
|
|
m.model_options = tmp_model_options
|
|
|
|
if not ignore_pre_cfg_func:
|
|
m.set_model_sampler_cfg_function(automatic_cfg_function, disable_cfg1_optimization = False)
|
|
if subtract_latent_mean:
|
|
m.set_model_sampler_post_cfg_function(center_mean_latent_post_cfg)
|
|
if latent_intensity_rescale:
|
|
m.set_model_sampler_post_cfg_function(rescale_post_cfg)
|
|
return (m, args_str, )
|
|
|
|
class attentionModifierParametersNode:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"sigma_start": ("FLOAT", {"default": 1000, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"sigma_end": ("FLOAT", {"default": 0, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"self_attn_mod_eval": ("STRING", {"multiline": True }, {"default": ""}),
|
|
"unet_block_id_input": ("STRING", {"multiline": False}, {"default": ""}),
|
|
"unet_block_id_middle": ("STRING", {"multiline": False}, {"default": ""}),
|
|
"unet_block_id_output": ("STRING", {"multiline": False}, {"default": ""}),
|
|
"unet_attn": (["attn1","attn2"],),
|
|
},
|
|
"optional":{
|
|
"join_parameters": ("ATTNMOD", {"forceInput": True}),
|
|
}}
|
|
|
|
RETURN_TYPES = ("ATTNMOD","STRING",)
|
|
RETURN_NAMES = ("Attention modifier", "Parameters as string")
|
|
FUNCTION = "exec"
|
|
CATEGORY = "model_patches/automatic_cfg/experimental_attention_modifiers"
|
|
def exec(self, join_parameters=None, **kwargs):
|
|
info_string = "\n".join([f"{k}: {v}" for k,v in kwargs.items() if v != ""])
|
|
return ([kwargs] if join_parameters is None else join_parameters + [kwargs], info_string, )
|
|
|
|
class attentionModifierBruteforceParametersNode:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
|
|
"sigma_start": ("FLOAT", {"default": 1000, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"sigma_end": ("FLOAT", {"default": 0, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"self_attn_mod_eval": ("STRING", {"multiline": True }, {"default": ""}),
|
|
"unet_block_id_input": ("STRING", {"multiline": False}, {"default": "4,5,7,8"}),
|
|
"unet_block_id_middle": ("STRING", {"multiline": False}, {"default": "0"}),
|
|
"unet_block_id_output": ("STRING", {"multiline": False}, {"default": "0,1,2,3,4,5"}),
|
|
"unet_attn": (["attn1","attn2"],),
|
|
},
|
|
"optional":{
|
|
"join_parameters": ("ATTNMOD", {"forceInput": True}),
|
|
}}
|
|
|
|
RETURN_TYPES = ("ATTNMOD","STRING",)
|
|
RETURN_NAMES = ("Attention modifier", "Parameters as string")
|
|
FUNCTION = "exec"
|
|
CATEGORY = "model_patches/automatic_cfg/experimental_attention_modifiers"
|
|
|
|
def create_sequence_parameters(self, input_str, middle_str, output_str):
|
|
input_values = input_str.split(",") if input_str else []
|
|
middle_values = middle_str.split(",") if middle_str else []
|
|
output_values = output_str.split(",") if output_str else []
|
|
result = []
|
|
result.extend([{"unet_block_id_input": val, "unet_block_id_middle": "", "unet_block_id_output": ""} for val in input_values])
|
|
result.extend([{"unet_block_id_input": "", "unet_block_id_middle": val, "unet_block_id_output": ""} for val in middle_values])
|
|
result.extend([{"unet_block_id_input": "", "unet_block_id_middle": "", "unet_block_id_output": val} for val in output_values])
|
|
return result
|
|
|
|
def exec(self, seed, join_parameters=None, **kwargs):
|
|
sequence_parameters = self.create_sequence_parameters(kwargs['unet_block_id_input'],kwargs['unet_block_id_middle'],kwargs['unet_block_id_output'])
|
|
lenseq = len(sequence_parameters)
|
|
current_index = seed % lenseq
|
|
current_sequence = sequence_parameters[current_index]
|
|
kwargs["unet_block_id_input"] = current_sequence["unet_block_id_input"]
|
|
kwargs["unet_block_id_middle"] = current_sequence["unet_block_id_middle"]
|
|
kwargs["unet_block_id_output"] = current_sequence["unet_block_id_output"]
|
|
if current_sequence["unet_block_id_input"] != "":
|
|
current_block_string = f"unet_block_id_input: {current_sequence['unet_block_id_input']}"
|
|
elif current_sequence["unet_block_id_middle"] != "":
|
|
current_block_string = f"unet_block_id_middle: {current_sequence['unet_block_id_middle']}"
|
|
elif current_sequence["unet_block_id_output"] != "":
|
|
current_block_string = f"unet_block_id_output: {current_sequence['unet_block_id_output']}"
|
|
info_string = f"Progress: {current_index+1}/{lenseq}\n{kwargs['self_attn_mod_eval']}\n{kwargs['unet_attn']} {current_block_string}"
|
|
return ([kwargs] if join_parameters is None else join_parameters + [kwargs], info_string, )
|
|
|
|
class attentionModifierConcatNode:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"parameters_1": ("ATTNMOD", {"forceInput": True}),
|
|
"parameters_2": ("ATTNMOD", {"forceInput": True}),
|
|
}}
|
|
|
|
RETURN_TYPES = ("ATTNMOD",)
|
|
FUNCTION = "exec"
|
|
CATEGORY = "model_patches/automatic_cfg/experimental_attention_modifiers"
|
|
def exec(self, parameters_1, parameters_2):
|
|
output_parms = parameters_1 + parameters_2
|
|
return (output_parms, )
|
|
|
|
class simpleDynamicCFG:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"model": ("MODEL",),
|
|
"hard_mode" : ("BOOLEAN", {"default": True}),
|
|
"boost" : ("BOOLEAN", {"default": True}),
|
|
}}
|
|
RETURN_TYPES = ("MODEL",)
|
|
FUNCTION = "patch"
|
|
|
|
CATEGORY = "model_patches/automatic_cfg/presets"
|
|
|
|
def patch(self, model, hard_mode, boost):
|
|
advcfg = advancedDynamicCFG()
|
|
m = advcfg.patch(model,
|
|
skip_uncond = boost,
|
|
uncond_sigma_start = 1000, uncond_sigma_end = 1,
|
|
automatic_cfg = "hard" if hard_mode else "soft"
|
|
)[0]
|
|
return (m, )
|
|
|
|
class presetLoader:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
presets_files = [pj.replace(".json","") for pj in os.listdir(json_preset_path) if ".json" in pj]
|
|
presets_files = sorted(presets_files, key=str.lower)
|
|
return {"required": {
|
|
"model": ("MODEL",),
|
|
"preset" : (presets_files, {"default": "The red riding latent"}),
|
|
"uncond_sigma_end": ("FLOAT", {"default": 1, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"use_uncond_sigma_end_from_preset" : ("BOOLEAN", {"default": True}),
|
|
},
|
|
"optional":{
|
|
"join_global_parameters": ("ATTNMOD", {"forceInput": True}),
|
|
}}
|
|
RETURN_TYPES = ("MODEL", "STRING", "STRING",)
|
|
RETURN_NAMES = ("Model", "Preset name", "Parameters as string",)
|
|
FUNCTION = "patch"
|
|
|
|
CATEGORY = "model_patches/automatic_cfg"
|
|
|
|
def patch(self, model, preset, uncond_sigma_end, use_uncond_sigma_end_from_preset, join_global_parameters=None):
|
|
with open(os.path.join(json_preset_path, preset+".json"), 'r', encoding='utf-8') as f:
|
|
preset_args = json.load(f)
|
|
if not use_uncond_sigma_end_from_preset:
|
|
preset_args["uncond_sigma_end"] = uncond_sigma_end
|
|
preset_args["fake_uncond_sigma_end"] = uncond_sigma_end
|
|
|
|
if join_global_parameters is not None:
|
|
preset_args["attention_modifiers_global"] = preset_args["attention_modifiers_global"] + join_global_parameters
|
|
preset_args["attention_modifiers_global_enabled"] = True
|
|
|
|
advcfg = advancedDynamicCFG()
|
|
m = advcfg.patch(model, **preset_args)[0]
|
|
info_string = "\n".join([f"{k}: {v}" for k,v in preset_args.items() if v != ""])
|
|
print(f"Preset {Fore.GREEN}{preset}{Fore.RESET} loaded successfully!")
|
|
return (m, preset, info_string,)
|
|
|
|
class simpleDynamicCFGlerpUncond:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"model": ("MODEL",),
|
|
"boost" : ("BOOLEAN", {"default": True}),
|
|
"negative_strength": ("FLOAT", {"default": 1, "min": 0.0, "max": 5.0, "step": 0.1, "round": 0.1}),
|
|
}}
|
|
RETURN_TYPES = ("MODEL",)
|
|
FUNCTION = "patch"
|
|
|
|
CATEGORY = "model_patches/automatic_cfg/presets"
|
|
|
|
def patch(self, model, boost, negative_strength):
|
|
advcfg = advancedDynamicCFG()
|
|
m = advcfg.patch(model=model,
|
|
automatic_cfg="hard", skip_uncond=boost,
|
|
uncond_sigma_start = 15, uncond_sigma_end = 1,
|
|
lerp_uncond=negative_strength != 1, lerp_uncond_strength=negative_strength,
|
|
lerp_uncond_sigma_start = 15, lerp_uncond_sigma_end = 1
|
|
)[0]
|
|
return (m, )
|
|
|
|
class postCFGrescaleOnly:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"model": ("MODEL",),
|
|
"subtract_latent_mean" : ("BOOLEAN", {"default": True}),
|
|
"subtract_latent_mean_sigma_start": ("FLOAT", {"default": 1000, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.1}),
|
|
"subtract_latent_mean_sigma_end": ("FLOAT", {"default": 7.5, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.1}),
|
|
"latent_intensity_rescale" : ("BOOLEAN", {"default": True}),
|
|
"latent_intensity_rescale_method" : (["soft","hard","range"], {"default": "hard"},),
|
|
"latent_intensity_rescale_cfg" : ("FLOAT", {"default": 8, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.1}),
|
|
"latent_intensity_rescale_sigma_start": ("FLOAT", {"default": 1000, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.1}),
|
|
"latent_intensity_rescale_sigma_end": ("FLOAT", {"default": 5, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.1}),
|
|
}}
|
|
RETURN_TYPES = ("MODEL",)
|
|
FUNCTION = "patch"
|
|
|
|
CATEGORY = "model_patches/automatic_cfg/utils"
|
|
|
|
def patch(self, model,
|
|
subtract_latent_mean, subtract_latent_mean_sigma_start, subtract_latent_mean_sigma_end,
|
|
latent_intensity_rescale, latent_intensity_rescale_method, latent_intensity_rescale_cfg, latent_intensity_rescale_sigma_start, latent_intensity_rescale_sigma_end
|
|
):
|
|
advcfg = advancedDynamicCFG()
|
|
m = advcfg.patch(model=model,
|
|
subtract_latent_mean = subtract_latent_mean,
|
|
subtract_latent_mean_sigma_start = subtract_latent_mean_sigma_start, subtract_latent_mean_sigma_end = subtract_latent_mean_sigma_end,
|
|
latent_intensity_rescale = latent_intensity_rescale, latent_intensity_rescale_cfg = latent_intensity_rescale_cfg, latent_intensity_rescale_method = latent_intensity_rescale_method,
|
|
latent_intensity_rescale_sigma_start = latent_intensity_rescale_sigma_start, latent_intensity_rescale_sigma_end = latent_intensity_rescale_sigma_end,
|
|
ignore_pre_cfg_func = True
|
|
)[0]
|
|
return (m, )
|
|
|
|
class simpleDynamicCFGHighSpeed:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"model": ("MODEL",),
|
|
}}
|
|
RETURN_TYPES = ("MODEL",)
|
|
FUNCTION = "patch"
|
|
|
|
CATEGORY = "model_patches/automatic_cfg/presets"
|
|
|
|
def patch(self, model):
|
|
advcfg = advancedDynamicCFG()
|
|
m = advcfg.patch(model=model, automatic_cfg = "hard",
|
|
skip_uncond = True, uncond_sigma_start = 7.5, uncond_sigma_end = 1)[0]
|
|
return (m, )
|
|
|
|
class simpleDynamicCFGwarpDrive:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"model": ("MODEL",),
|
|
"uncond_sigma_start": ("FLOAT", {"default": 5.5, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"uncond_sigma_end": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"fake_uncond_sigma_end": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
}}
|
|
RETURN_TYPES = ("MODEL",)
|
|
FUNCTION = "patch"
|
|
|
|
CATEGORY = "model_patches/automatic_cfg/presets"
|
|
|
|
def patch(self, model, uncond_sigma_start, uncond_sigma_end, fake_uncond_sigma_end):
|
|
advcfg = advancedDynamicCFG()
|
|
print(f" {Fore.CYAN}WARP DRIVE MODE ENGAGED!{Style.RESET_ALL}\n Settings suggestions:\n"
|
|
f" {Fore.GREEN}1/1/1: {Fore.YELLOW}Maaaxxxiiimum speeeeeed.{Style.RESET_ALL} {Fore.RED}Uncond disabled.{Style.RESET_ALL} {Fore.MAGENTA}Fasten your seatbelt!{Style.RESET_ALL}\n"
|
|
f" {Fore.GREEN}3/1/1: {Fore.YELLOW}Risky space-time continuum distortion.{Style.RESET_ALL} {Fore.MAGENTA}Awesome for prompts with a clear subject!{Style.RESET_ALL}\n"
|
|
f" {Fore.GREEN}5.5/1/1: {Fore.YELLOW}Frameshift Drive Autopilot: {Fore.GREEN}Engaged.{Style.RESET_ALL} {Fore.MAGENTA}Should work with anything but do it better and faster!{Style.RESET_ALL}")
|
|
|
|
m = advcfg.patch(model=model, automatic_cfg = "hard",
|
|
skip_uncond = True, uncond_sigma_start = uncond_sigma_start, uncond_sigma_end = uncond_sigma_end,
|
|
fake_uncond_sigma_end = fake_uncond_sigma_end, fake_uncond_sigma_start = 1000, fake_uncond_start=True,
|
|
fake_uncond_exp=True,fake_uncond_exp_normalize=True,fake_uncond_exp_method="previous_average",
|
|
cond_exp = False, cond_exp_sigma_start = 9, cond_exp_sigma_end = uncond_sigma_start, cond_exp_method = "erf", cond_exp_normalize = True,
|
|
)[0]
|
|
return (m, )
|
|
|
|
class simpleDynamicCFGunpatch:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"model": ("MODEL",),
|
|
}}
|
|
RETURN_TYPES = ("MODEL",)
|
|
FUNCTION = "unpatch"
|
|
|
|
CATEGORY = "model_patches/automatic_cfg/utils"
|
|
|
|
def unpatch(self, model):
|
|
m = model.clone()
|
|
m.model_options.pop("sampler_pre_cfg_function", None)
|
|
return (m, )
|
|
|
|
class simpleDynamicCFGExcellentattentionPatch:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"model": ("MODEL",),
|
|
"Auto_CFG": ("BOOLEAN", {"default": True}),
|
|
"patch_multiplier": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
"patch_cond": ("BOOLEAN", {"default": True}),
|
|
"patch_uncond": ("BOOLEAN", {"default": True}),
|
|
"light_patch": ("BOOLEAN", {"default": False}),
|
|
"mute_self_input_layer_8_cond": ("BOOLEAN", {"default": False}),
|
|
"mute_cross_input_layer_8_cond": ("BOOLEAN", {"default": False}),
|
|
"mute_self_input_layer_8_uncond": ("BOOLEAN", {"default": True}),
|
|
"mute_cross_input_layer_8_uncond": ("BOOLEAN", {"default": False}),
|
|
"uncond_sigma_end": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10000.0, "step": 0.1, "round": 0.01}),
|
|
}}
|
|
RETURN_TYPES = ("MODEL","STRING",)
|
|
RETURN_NAMES = ("Model", "Parameters as string",)
|
|
FUNCTION = "patch"
|
|
|
|
CATEGORY = "model_patches/automatic_cfg/presets"
|
|
|
|
def patch(self, model, Auto_CFG, patch_multiplier, patch_cond, patch_uncond, light_patch,
|
|
mute_self_input_layer_8_cond, mute_cross_input_layer_8_cond,
|
|
mute_self_input_layer_8_uncond, mute_cross_input_layer_8_uncond,
|
|
uncond_sigma_end):
|
|
|
|
parameters_as_string = "Excellent attention:\n" + "\n".join([f"{k}: {v}" for k, v in locals().items() if k not in ["self", "model"]])
|
|
|
|
with open(os.path.join(json_preset_path, "Excellent_attention.json"), 'r', encoding='utf-8') as f:
|
|
patch_parameters = json.load(f)
|
|
|
|
attn_patch = {"sigma_start": 1000, "sigma_end": 0,
|
|
"self_attn_mod_eval": f"normalize_tensor(q+(q-attention_basic(attnbc, k, v, extra_options['n_heads'])))*attnbc.norm()*{patch_multiplier}",
|
|
"unet_block_id_input": "", "unet_block_id_middle": "0", "unet_block_id_output": "", "unet_attn": "attn2"}
|
|
attn_patch_light = {"sigma_start": 1000, "sigma_end": 0,
|
|
"self_attn_mod_eval": f"q*{patch_multiplier}",
|
|
"unet_block_id_input": "", "unet_block_id_middle": "0", "unet_block_id_output": "", "unet_attn": "attn2"}
|
|
kill_self_input_8 = {
|
|
"sigma_start": 1000,
|
|
"sigma_end": 0,
|
|
"self_attn_mod_eval": "torch.zeros_like(q)",
|
|
"unet_block_id_input": "8",
|
|
"unet_block_id_middle": "",
|
|
"unet_block_id_output": "",
|
|
"unet_attn": "attn1"}
|
|
kill_cross_input_8 = {
|
|
"sigma_start": 1000,
|
|
"sigma_end": 0,
|
|
"self_attn_mod_eval": "torch.zeros_like(q)",
|
|
"unet_block_id_input": "8",
|
|
"unet_block_id_middle": "",
|
|
"unet_block_id_output": "",
|
|
"unet_attn": "attn2"}
|
|
|
|
attention_modifiers_positive = []
|
|
attention_modifiers_fake_negative = []
|
|
|
|
if patch_cond: attention_modifiers_positive.append(attn_patch) if not light_patch else attention_modifiers_positive.append(attn_patch_light)
|
|
if mute_self_input_layer_8_cond: attention_modifiers_positive.append(kill_self_input_8)
|
|
if mute_cross_input_layer_8_cond: attention_modifiers_positive.append(kill_cross_input_8)
|
|
|
|
if patch_uncond: attention_modifiers_fake_negative.append(attn_patch) if not light_patch else attention_modifiers_fake_negative.append(attn_patch_light)
|
|
if mute_self_input_layer_8_uncond: attention_modifiers_fake_negative.append(kill_self_input_8)
|
|
if mute_cross_input_layer_8_uncond: attention_modifiers_fake_negative.append(kill_cross_input_8)
|
|
|
|
patch_parameters['attention_modifiers_positive'] = attention_modifiers_positive
|
|
patch_parameters['attention_modifiers_fake_negative'] = attention_modifiers_fake_negative
|
|
patch_parameters["uncond_sigma_end"] = uncond_sigma_end
|
|
patch_parameters["fake_uncond_sigma_end"] = uncond_sigma_end
|
|
patch_parameters["automatic_cfg"] = "hard" if Auto_CFG else "None"
|
|
|
|
advcfg = advancedDynamicCFG()
|
|
m = advcfg.patch(model, **patch_parameters)[0]
|
|
|
|
return (m, parameters_as_string, ) |