Merge pull request #62 from IDGallagher/feature/memory_optimization
Feature/memory optimization
This commit is contained in:
POM
+87
-22
@@ -11,7 +11,6 @@ import matplotlib.pyplot as plt
|
||||
from .imports.ComfyUI_IPAdapter_plus.IPAdapterPlus import IPAdapterBatchImport, IPAdapterTiledBatchImport, IPAdapterTiledImport, PrepImageForClipVisionImport, IPAdapterAdvancedImport, IPAdapterNoiseImport
|
||||
from .imports.AdvancedControlNet.nodes_sparsectrl import SparseIndexMethodNodeImport
|
||||
|
||||
|
||||
class BatchCreativeInterpolationNode:
|
||||
@classmethod
|
||||
def IS_CHANGED(cls, **kwargs):
|
||||
@@ -369,21 +368,55 @@ class BatchCreativeInterpolationNode:
|
||||
else:
|
||||
last_key_frame_position = (keyframe_positions[-1])
|
||||
|
||||
class IPBin:
|
||||
def __init__(self):
|
||||
self.indicies = []
|
||||
self.image_schedule = []
|
||||
self.weight_schedule = []
|
||||
self.imageBatch = []
|
||||
self.bigImageBatch = []
|
||||
self.noiseBatch = []
|
||||
self.bigNoiseBatch = []
|
||||
|
||||
def length(self):
|
||||
return len(self.image_schedule)
|
||||
|
||||
def add(self, image, big_image, noise, big_noise, image_index, frame_numbers, weights):
|
||||
# Map frames to their corresponding reversed weights for easy lookup
|
||||
frame_to_weight = {frame: weights[i] for i, frame in enumerate(frame_numbers)}
|
||||
# Search for image index, if it isn't there add the image
|
||||
try:
|
||||
index = self.indicies.index(image_index)
|
||||
except ValueError:
|
||||
self.imageBatch.append(image)
|
||||
self.bigImageBatch.append(big_image)
|
||||
if noise is not None: self.noiseBatch.append(noise)
|
||||
if big_noise is not None: self.bigNoiseBatch.append(big_noise)
|
||||
self.indicies.append(image_index)
|
||||
index = self.indicies.index(image_index)
|
||||
|
||||
self.image_schedule.extend([index] * (frame_numbers[-1] + 1 - len(self.image_schedule)))
|
||||
self.weight_schedule.extend([0] * (frame_numbers[0] - len(self.weight_schedule)))
|
||||
self.weight_schedule.extend(frame_to_weight[frame] for frame in range(frame_numbers[0], frame_numbers[-1] + 1))
|
||||
|
||||
# CREATE LISTS FOR WEIGHTS AND FRAME NUMBERS
|
||||
all_cn_frame_numbers = []
|
||||
all_cn_weights = []
|
||||
all_ipa_weights = []
|
||||
all_ipa_frame_numbers = []
|
||||
# Start with one bin
|
||||
bins = [IPBin()]
|
||||
|
||||
for i in range(len(keyframe_positions)):
|
||||
|
||||
keyframe_position = keyframe_positions[i]
|
||||
interpolation = "ease-in-out"
|
||||
# strength_from = strength_to = 1.0
|
||||
|
||||
image_index = 0
|
||||
if i == 0: # buffer
|
||||
|
||||
image = images[0]
|
||||
image_index = 0
|
||||
strength_from = strength_to = strength_values[0][1]
|
||||
|
||||
batch_index_from = 0
|
||||
@@ -394,6 +427,7 @@ class BatchCreativeInterpolationNode:
|
||||
|
||||
# GET IMAGE AND KEYFRAME INFLUENCE VALUES
|
||||
image = images[i-1]
|
||||
image_index = i-1
|
||||
key_frame_influence_from, key_frame_influence_to = key_frame_influence_values[i-1]
|
||||
start_strength, mid_strength, end_strength = strength_values[i-1]
|
||||
|
||||
@@ -409,6 +443,7 @@ class BatchCreativeInterpolationNode:
|
||||
|
||||
# GET IMAGE AND KEYFRAME INFLUENCE VALUES
|
||||
image = images[i-1]
|
||||
image_index = i - 1
|
||||
key_frame_influence_from,key_frame_influence_to = key_frame_influence_values[i-1]
|
||||
start_strength, mid_strength, end_strength = strength_values[i-1]
|
||||
if len(keyframe_positions) == 4:
|
||||
@@ -427,6 +462,7 @@ class BatchCreativeInterpolationNode:
|
||||
elif i == len(keyframe_positions) - 1: # buffer
|
||||
|
||||
image = images[i-2]
|
||||
image_index = i - 2
|
||||
strength_from = strength_to = strength_values[i-2][1]
|
||||
|
||||
if len(keyframe_positions) == 4:
|
||||
@@ -441,7 +477,8 @@ class BatchCreativeInterpolationNode:
|
||||
else: # middle images
|
||||
|
||||
# GET IMAGE AND KEYFRAME INFLUENCE VALUES
|
||||
image = images[i-1]
|
||||
image = images[i-1]
|
||||
image_index = i - 1
|
||||
key_frame_influence_from,key_frame_influence_to = key_frame_influence_values[i-1]
|
||||
start_strength, mid_strength, end_strength = strength_values[i-1]
|
||||
keyframe_position = keyframe_positions[i]
|
||||
@@ -465,11 +502,17 @@ class BatchCreativeInterpolationNode:
|
||||
# PROCESS WEIGHTS
|
||||
ipa_frame_numbers, ipa_weights = process_weights(frame_numbers, weights, base_ipa_advanced_settings["ipa_weight"])
|
||||
|
||||
# print(f'i {i} image index {image_index} ====')
|
||||
# # print(f"frame numbers {frame_numbers}")
|
||||
# # print(f"weights {weights}")
|
||||
# print(f"frame numbers {ipa_frame_numbers}")
|
||||
# print(f"weights {ipa_weights}")
|
||||
# print("------")
|
||||
|
||||
# Prepare images and noise
|
||||
prepare_for_clip_vision = PrepImageForClipVisionImport()
|
||||
prepped_image, = prepare_for_clip_vision.prep_image(image=image.unsqueeze(0), interpolation="LANCZOS", crop_position="pad", sharpening=0.1)
|
||||
|
||||
weight_batch = create_weight_batch(last_key_frame_position, ipa_weights, ipa_frame_numbers)
|
||||
|
||||
|
||||
if base_ipa_advanced_settings["ipa_noise_strength"] > 0:
|
||||
if base_ipa_advanced_settings["use_image_for_noise"]:
|
||||
noise_image = prepped_image
|
||||
@@ -480,28 +523,50 @@ class BatchCreativeInterpolationNode:
|
||||
else:
|
||||
negative_noise = None
|
||||
|
||||
ipadapter_application = IPAdapterBatchImport()
|
||||
model, = ipadapter_application.apply_ipadapter(model=model, ipadapter=ipadapter, image=prepped_image, weight=weight_batch, weight_type=base_ipa_advanced_settings["ipa_weight_type"], start_at=base_ipa_advanced_settings["ipa_starts_at"], end_at=base_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,image_negative=negative_noise,embeds_scaling=base_ipa_advanced_settings["ipa_embeds_scaling"])
|
||||
|
||||
if high_detail_mode:
|
||||
if detail_ipa_advanced_settings["ipa_noise_strength"] > 0:
|
||||
if detail_ipa_advanced_settings["use_image_for_noise"]:
|
||||
noise_image = image.unsqueeze(0)
|
||||
else:
|
||||
noise_image = None
|
||||
ipa_noise = IPAdapterNoiseImport()
|
||||
negative_noise, = ipa_noise.make_noise(type=detail_ipa_advanced_settings["type_of_noise"], strength=detail_ipa_advanced_settings["ipa_noise_strength"], blur=detail_ipa_advanced_settings["noise_blur"], image_optional=noise_image)
|
||||
if high_detail_mode and detail_ipa_advanced_settings["ipa_noise_strength"] > 0:
|
||||
if detail_ipa_advanced_settings["use_image_for_noise"]:
|
||||
noise_image = image.unsqueeze(0)
|
||||
else:
|
||||
negative_noise = None
|
||||
|
||||
tiled_ipa_application = IPAdapterTiledBatchImport()
|
||||
model, *_ = tiled_ipa_application.apply_tiled(model=model, ipadapter=ipadapter, image=image.unsqueeze(0), weight=weight_batch, weight_type=detail_ipa_advanced_settings["ipa_weight_type"], start_at=detail_ipa_advanced_settings["ipa_starts_at"], end_at=detail_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,sharpening=0.1,image_negative=negative_noise,embeds_scaling=detail_ipa_advanced_settings["ipa_embeds_scaling"])
|
||||
noise_image = None
|
||||
ipa_noise = IPAdapterNoiseImport()
|
||||
big_negative_noise, = ipa_noise.make_noise(type=detail_ipa_advanced_settings["type_of_noise"], strength=detail_ipa_advanced_settings["ipa_noise_strength"], blur=detail_ipa_advanced_settings["noise_blur"], image_optional=noise_image)
|
||||
else:
|
||||
big_negative_noise = None
|
||||
|
||||
# Fill up bins with image frames. Bins will automatically be created when needed but all the frames should be able to be packed into two bins
|
||||
active_index = -1
|
||||
# Find a bin that we can fit the next image into
|
||||
for i, bin in enumerate(bins):
|
||||
if bin.length() <= ipa_frame_numbers[0]:
|
||||
active_index = i
|
||||
break
|
||||
# If we didn't find a suitable bin, add a new one
|
||||
if active_index == -1:
|
||||
bins.append(IPBin())
|
||||
active_index = len(bins) - 1
|
||||
|
||||
# Add the image to the bin
|
||||
bins[active_index].add(prepped_image, image.unsqueeze(0), negative_noise, big_negative_noise, image_index, ipa_frame_numbers, ipa_weights)
|
||||
|
||||
# for i, bin in enumerate(bins):
|
||||
# print(f"{i} schedule {bin.image_schedule}")
|
||||
# print(f"{i} weights {bin.weight_schedule}")
|
||||
# i += 1
|
||||
|
||||
all_ipa_frame_numbers.append(ipa_frame_numbers)
|
||||
all_ipa_weights.append(ipa_weights)
|
||||
|
||||
comparison_diagram, = plot_weight_comparison(all_cn_frame_numbers, all_cn_weights, all_ipa_frame_numbers, all_ipa_weights, buffer)
|
||||
# Go through the bins and create IPAdapters for them
|
||||
for i, bin in enumerate(bins):
|
||||
ipadapter_application = IPAdapterBatchImport()
|
||||
negative_noise = torch.cat(bin.noiseBatch, dim=0) if len(bin.noiseBatch) > 0 else None
|
||||
model, = ipadapter_application.apply_ipadapter(model=model, ipadapter=ipadapter, image=torch.cat(bin.imageBatch, dim=0), weight=[x * base_ipa_advanced_settings["ipa_weight"] for x in bin.weight_schedule], weight_type=base_ipa_advanced_settings["ipa_weight_type"], start_at=base_ipa_advanced_settings["ipa_starts_at"], end_at=base_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,image_negative=negative_noise,embeds_scaling=base_ipa_advanced_settings["ipa_embeds_scaling"], image_schedule=bin.image_schedule)
|
||||
if high_detail_mode:
|
||||
tiled_ipa_application = IPAdapterTiledBatchImport()
|
||||
negative_noise = torch.cat(bin.bigNoiseBatch, dim=0) if len(bin.bigNoiseBatch) > 0 else None
|
||||
model, *_ = tiled_ipa_application.apply_tiled(model=model, ipadapter=ipadapter, image=torch.cat(bin.bigImageBatch, dim=0), weight=[x * detail_ipa_advanced_settings["ipa_weight"] for x in bin.weight_schedule], weight_type=detail_ipa_advanced_settings["ipa_weight_type"], start_at=detail_ipa_advanced_settings["ipa_starts_at"], end_at=detail_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,sharpening=0.1,image_negative=negative_noise,embeds_scaling=detail_ipa_advanced_settings["ipa_embeds_scaling"], image_schedule=bin.image_schedule)
|
||||
|
||||
comparison_diagram, = plot_weight_comparison(all_cn_frame_numbers, all_cn_weights, all_ipa_frame_numbers, all_ipa_weights, buffer)
|
||||
return comparison_diagram, positive, negative, model, sparse_indexes, last_key_frame_position, buffer
|
||||
|
||||
class IpaConfigurationNode:
|
||||
|
||||
@@ -6,7 +6,7 @@ from .utils import tensor_to_size
|
||||
|
||||
class CrossAttentionPatchImport:
|
||||
# forward for patching
|
||||
def __init__(self, ipadapter=None, number=0, weight=1.0, cond=None, cond_alt=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, embeds_scaling='V only'):
|
||||
def __init__(self, ipadapter=None, number=0, weight=1.0, cond=None, cond_alt=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, image_schedule=None, embeds_scaling='V only'):
|
||||
self.weights = [weight]
|
||||
self.ipadapters = [ipadapter]
|
||||
self.conds = [cond]
|
||||
@@ -17,6 +17,7 @@ class CrossAttentionPatchImport:
|
||||
self.sigma_starts = [sigma_start]
|
||||
self.sigma_ends = [sigma_end]
|
||||
self.unfold_batch = [unfold_batch]
|
||||
self.image_schedule = [image_schedule]
|
||||
self.embeds_scaling = [embeds_scaling]
|
||||
self.number = number
|
||||
self.layers = 11 if '101_to_k_ip' in ipadapter.ip_layers.to_kvs else 16 # TODO: check if this is a valid condition to detect all models
|
||||
@@ -24,7 +25,7 @@ class CrossAttentionPatchImport:
|
||||
self.k_key = str(self.number*2+1) + "_to_k_ip"
|
||||
self.v_key = str(self.number*2+1) + "_to_v_ip"
|
||||
|
||||
def set_new_condition(self, ipadapter=None, number=0, weight=1.0, cond=None, cond_alt=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, embeds_scaling='V only'):
|
||||
def set_new_condition(self, ipadapter=None, number=0, weight=1.0, cond=None, cond_alt=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, image_schedule=None, embeds_scaling='V only'):
|
||||
self.weights.append(weight)
|
||||
self.ipadapters.append(ipadapter)
|
||||
self.conds.append(cond)
|
||||
@@ -35,6 +36,7 @@ class CrossAttentionPatchImport:
|
||||
self.sigma_starts.append(sigma_start)
|
||||
self.sigma_ends.append(sigma_end)
|
||||
self.unfold_batch.append(unfold_batch)
|
||||
self.image_schedule.append(image_schedule)
|
||||
self.embeds_scaling.append(embeds_scaling)
|
||||
|
||||
def __call__(self, q, k, v, extra_options):
|
||||
@@ -54,7 +56,7 @@ class CrossAttentionPatchImport:
|
||||
out = optimized_attention(q, k, v, extra_options["n_heads"])
|
||||
_, _, oh, ow = extra_options["original_shape"]
|
||||
|
||||
for weight, cond, cond_alt, uncond, ipadapter, mask, weight_type, sigma_start, sigma_end, unfold_batch, embeds_scaling in zip(self.weights, self.conds, self.conds_alt, self.unconds, self.ipadapters, self.masks, self.weight_types, self.sigma_starts, self.sigma_ends, self.unfold_batch, self.embeds_scaling):
|
||||
for weight, cond, cond_alt, uncond, ipadapter, mask, weight_type, sigma_start, sigma_end, unfold_batch, image_schedule, embeds_scaling in zip(self.weights, self.conds, self.conds_alt, self.unconds, self.ipadapters, self.masks, self.weight_types, self.sigma_starts, self.sigma_ends, self.unfold_batch, self.image_schedule, self.embeds_scaling):
|
||||
if sigma <= sigma_start and sigma >= sigma_end:
|
||||
if weight_type == 'ease in':
|
||||
weight = weight * (0.05 + 0.95 * (1 - t_idx / self.layers))
|
||||
@@ -94,16 +96,23 @@ class CrossAttentionPatchImport:
|
||||
elif weight == 0:
|
||||
continue
|
||||
|
||||
# if image length matches or exceeds full_length get sub_idx images
|
||||
if cond.shape[0] >= ad_params["full_length"]:
|
||||
cond = torch.Tensor(cond[ad_params["sub_idxs"]])
|
||||
uncond = torch.Tensor(uncond[ad_params["sub_idxs"]])
|
||||
# otherwise get sub_idxs images
|
||||
if image_schedule is not None:
|
||||
# Use the image_schedule as a lookup table to get the embedded image corresponding to each sub_idx
|
||||
# If image_schedule isn't long enough then use the last image
|
||||
cond_idxs = [image_schedule[i if i < len(image_schedule) else -1] for i in ad_params["sub_idxs"]]
|
||||
cond = torch.Tensor(cond[cond_idxs])
|
||||
uncond = torch.Tensor(uncond[cond_idxs])
|
||||
else:
|
||||
cond = tensor_to_size(cond, ad_params["full_length"])
|
||||
uncond = tensor_to_size(uncond, ad_params["full_length"])
|
||||
cond = cond[ad_params["sub_idxs"]]
|
||||
uncond = uncond[ad_params["sub_idxs"]]
|
||||
# if image length matches or exceeds full_length get sub_idx images
|
||||
if cond.shape[0] >= ad_params["full_length"]:
|
||||
cond = torch.Tensor(cond[ad_params["sub_idxs"]])
|
||||
uncond = torch.Tensor(uncond[ad_params["sub_idxs"]])
|
||||
# otherwise get sub_idxs images
|
||||
else:
|
||||
cond = tensor_to_size(cond, ad_params["full_length"])
|
||||
uncond = tensor_to_size(uncond, ad_params["full_length"])
|
||||
cond = cond[ad_params["sub_idxs"]]
|
||||
uncond = uncond[ad_params["sub_idxs"]]
|
||||
else:
|
||||
if isinstance(weight, torch.Tensor):
|
||||
weight = tensor_to_size(weight, batch_prompt)
|
||||
|
||||
@@ -45,7 +45,7 @@ WEIGHT_TYPES = ["linear", "ease in", "ease out", 'ease in-out', 'reverse in-out'
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
"""
|
||||
class IPAdapterImport(nn.Module):
|
||||
def __init__(self, ipadapter_model, cross_attention_dim=1024, output_cross_attention_dim=1024, clip_embeddings_dim=1024, clip_extra_context_tokens=4, is_sdxl=False, is_plus=False, is_full=False, is_faceid=False):
|
||||
def __init__(self, ipadapter_model, cross_attention_dim=1024, output_cross_attention_dim=1024, clip_embeddings_dim=1024, clip_extra_context_tokens=4, is_sdxl=False, is_plus=False, is_full=False, is_faceid=False, is_portrait_unnorm=False):
|
||||
super().__init__()
|
||||
|
||||
self.clip_embeddings_dim = clip_embeddings_dim
|
||||
@@ -55,12 +55,13 @@ class IPAdapterImport(nn.Module):
|
||||
self.is_sdxl = is_sdxl
|
||||
self.is_full = is_full
|
||||
self.is_plus = is_plus
|
||||
self.is_portrait_unnorm = is_portrait_unnorm
|
||||
|
||||
if is_faceid:
|
||||
if is_faceid and not is_portrait_unnorm:
|
||||
self.image_proj_model = self.init_proj_faceid()
|
||||
elif is_full:
|
||||
self.image_proj_model = self.init_proj_full()
|
||||
elif is_plus:
|
||||
elif is_plus or is_portrait_unnorm:
|
||||
self.image_proj_model = self.init_proj_plus()
|
||||
else:
|
||||
self.image_proj_model = self.init_proj()
|
||||
@@ -161,6 +162,7 @@ def ipadapter_execute(model,
|
||||
pos_embed=None,
|
||||
neg_embed=None,
|
||||
unfold_batch=False,
|
||||
image_schedule=None,
|
||||
embeds_scaling='V only',
|
||||
layer_weights=None):
|
||||
device = model_management.get_torch_device()
|
||||
@@ -170,8 +172,9 @@ def ipadapter_execute(model,
|
||||
|
||||
is_full = "proj.3.weight" in ipadapter["image_proj"]
|
||||
is_portrait = "proj.2.weight" in ipadapter["image_proj"] and not "proj.3.weight" in ipadapter["image_proj"] and not "0.to_q_lora.down.weight" in ipadapter["ip_adapter"]
|
||||
is_faceid = is_portrait or "0.to_q_lora.down.weight" in ipadapter["ip_adapter"]
|
||||
is_plus = is_full or "latents" in ipadapter["image_proj"] or "perceiver_resampler.proj_in.weight" in ipadapter["image_proj"]
|
||||
is_portrait_unnorm = "portraitunnorm" in ipadapter
|
||||
is_faceid = is_portrait or "0.to_q_lora.down.weight" in ipadapter["ip_adapter"] or is_portrait_unnorm
|
||||
is_plus = (is_full or "latents" in ipadapter["image_proj"] or "perceiver_resampler.proj_in.weight" in ipadapter["image_proj"]) and not is_portrait_unnorm
|
||||
is_faceidv2 = "faceidplusv2" in ipadapter
|
||||
output_cross_attention_dim = ipadapter["ip_adapter"]["1.to_k_ip.weight"].shape[1]
|
||||
is_sdxl = output_cross_attention_dim == 2048
|
||||
@@ -182,14 +185,14 @@ def ipadapter_execute(model,
|
||||
if is_faceidv2:
|
||||
weight_faceidv2 = weight_faceidv2 if weight_faceidv2 is not None else weight*2
|
||||
|
||||
cross_attention_dim = 1280 if is_plus and is_sdxl and not is_faceid else output_cross_attention_dim
|
||||
clip_extra_context_tokens = 16 if (is_plus and not is_faceid) or is_portrait else 4
|
||||
cross_attention_dim = 1280 if (is_plus and is_sdxl and not is_faceid) or is_portrait_unnorm else output_cross_attention_dim
|
||||
clip_extra_context_tokens = 16 if (is_plus and not is_faceid) or is_portrait or is_portrait_unnorm else 4
|
||||
|
||||
if image is not None and image.shape[1] != image.shape[2]:
|
||||
print("\033[33mINFO: the IPAdapter reference image is not a square, CLIPImageProcessor will resize and crop it at the center. If the main focus of the picture is not in the middle the result might not be what you are expecting.\033[0m")
|
||||
|
||||
if isinstance(weight, list):
|
||||
weight = torch.tensor(weight).unsqueeze(-1).unsqueeze(-1).to(device, dtype=dtype) if unfold_batch else weight[0]
|
||||
weight = torch.tensor(weight).unsqueeze(-1).unsqueeze(-1).to(device, dtype=dtype) if unfold_batch else weight[0]
|
||||
|
||||
# special weight types
|
||||
if layer_weights is not None and layer_weights != '':
|
||||
@@ -233,7 +236,10 @@ def ipadapter_execute(model,
|
||||
insightface.det_model.input_size = size # TODO: hacky but seems to be working
|
||||
face = insightface.get(image_iface[i])
|
||||
if face:
|
||||
face_cond_embeds.append(torch.from_numpy(face[0].normed_embedding).unsqueeze(0))
|
||||
if not is_portrait_unnorm:
|
||||
face_cond_embeds.append(torch.from_numpy(face[0].normed_embedding).unsqueeze(0))
|
||||
else:
|
||||
face_cond_embeds.append(torch.from_numpy(face[0].embedding).unsqueeze(0))
|
||||
image.append(image_to_tensor(face_align.norm_crop(image_iface[i], landmark=face[0].kps, image_size=256)))
|
||||
|
||||
if 640 not in size:
|
||||
@@ -330,7 +336,8 @@ def ipadapter_execute(model,
|
||||
is_sdxl=is_sdxl,
|
||||
is_plus=is_plus,
|
||||
is_full=is_full,
|
||||
is_faceid=is_faceid
|
||||
is_faceid=is_faceid,
|
||||
is_portrait_unnorm=is_portrait_unnorm,
|
||||
).to(device, dtype=dtype)
|
||||
|
||||
if is_faceid and is_plus:
|
||||
@@ -365,6 +372,7 @@ def ipadapter_execute(model,
|
||||
"sigma_start": sigma_start,
|
||||
"sigma_end": sigma_end,
|
||||
"unfold_batch": unfold_batch,
|
||||
"image_schedule": image_schedule,
|
||||
"embeds_scaling": embeds_scaling,
|
||||
}
|
||||
|
||||
@@ -492,7 +500,7 @@ class IPAdapterUnifiedLoaderFaceID(IPAdapterUnifiedLoader):
|
||||
def INPUT_TYPES(s):
|
||||
return {"required": {
|
||||
"model": ("MODEL", ),
|
||||
"preset": (['FACEID', 'FACEID PLUS - SD1.5 only', 'FACEID PLUS V2', 'FACEID PORTRAIT (style transfer)'], ),
|
||||
"preset": (['FACEID', 'FACEID PLUS - SD1.5 only', 'FACEID PLUS V2', 'FACEID PORTRAIT (style transfer)', 'FACEID PORTRAIT UNNORM - SDXL only (strong)'], ),
|
||||
"lora_strength": ("FLOAT", { "default": 0.6, "min": 0, "max": 1, "step": 0.01 }),
|
||||
"provider": (["CPU", "CUDA", "ROCM", "DirectML", "OpenVINO", "CoreML"], ),
|
||||
},
|
||||
@@ -626,7 +634,7 @@ class IPAdapterAdvancedImport:
|
||||
FUNCTION = "apply_ipadapter"
|
||||
CATEGORY = "ipadapter"
|
||||
|
||||
def apply_ipadapter(self, model, ipadapter, start_at, end_at, weight = 1.0, weight_style=1.0, weight_composition=1.0, expand_style=False, weight_type="linear", combine_embeds="concat", weight_faceidv2=None, image=None, image_style=None, image_composition=None, image_negative=None, clip_vision=None, attn_mask=None, insightface=None, embeds_scaling='V only', layer_weights=None):
|
||||
def apply_ipadapter(self, model, ipadapter, start_at, end_at, weight = 1.0, weight_style=1.0, weight_composition=1.0, expand_style=False, weight_type="linear", combine_embeds="concat", weight_faceidv2=None, image=None, image_style=None, image_composition=None, image_negative=None, clip_vision=None, image_schedule=None, attn_mask=None, insightface=None, embeds_scaling='V only', layer_weights=None):
|
||||
is_sdxl = isinstance(model.model, (comfy.model_base.SDXL, comfy.model_base.SDXLRefiner, comfy.model_base.SDXL_instructpix2pix))
|
||||
|
||||
if image_style is not None: # we are doing style + composition transfer
|
||||
@@ -653,6 +661,7 @@ class IPAdapterAdvancedImport:
|
||||
"end_at": end_at,
|
||||
"attn_mask": attn_mask,
|
||||
"unfold_batch": self.unfold_batch,
|
||||
"image_schedule": image_schedule,
|
||||
"embeds_scaling": embeds_scaling,
|
||||
"insightface": insightface if insightface is not None else ipadapter['insightface']['model'] if 'insightface' in ipadapter else None,
|
||||
"layer_weights": layer_weights,
|
||||
@@ -693,6 +702,7 @@ class IPAdapterBatchImport(IPAdapterAdvancedImport):
|
||||
"image_negative": ("IMAGE",),
|
||||
"attn_mask": ("MASK",),
|
||||
"clip_vision": ("CLIP_VISION",),
|
||||
"image_schedule": ("INT", {"default": None, "forceInput": True} ),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -745,6 +755,7 @@ class IPAdapterStyleCompositionBatch(IPAdapterStyleComposition):
|
||||
"image_negative": ("IMAGE",),
|
||||
"attn_mask": ("MASK",),
|
||||
"clip_vision": ("CLIP_VISION",),
|
||||
"image_schedule": ("INT", {"default": None, "forceInput": True} ),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -809,7 +820,7 @@ class IPAdapterTiledImport:
|
||||
FUNCTION = "apply_tiled"
|
||||
CATEGORY = "ipadapter/tiled"
|
||||
|
||||
def apply_tiled(self, model, ipadapter, image, weight, weight_type, start_at, end_at, sharpening, combine_embeds="concat", image_negative=None, attn_mask=None, clip_vision=None, embeds_scaling='V only'):
|
||||
def apply_tiled(self, model, ipadapter, image, weight, weight_type, start_at, end_at, sharpening, combine_embeds="concat", image_negative=None, attn_mask=None, clip_vision=None, embeds_scaling='V only', image_schedule=None):
|
||||
# 1. Select the models
|
||||
if 'ipadapter' in ipadapter:
|
||||
ipadapter_model = ipadapter['ipadapter']['model']
|
||||
@@ -907,7 +918,9 @@ class IPAdapterTiledImport:
|
||||
"attn_mask": masks[i],
|
||||
"unfold_batch": self.unfold_batch,
|
||||
"embeds_scaling": embeds_scaling,
|
||||
"image_schedule": image_schedule,
|
||||
}
|
||||
|
||||
# apply the ipadapter to the model without cloning it
|
||||
model = ipadapter_execute(model, ipadapter_model, clip_vision, **ipa_args)
|
||||
|
||||
@@ -935,6 +948,7 @@ class IPAdapterTiledBatchImport(IPAdapterTiledImport):
|
||||
"image_negative": ("IMAGE",),
|
||||
"attn_mask": ("MASK",),
|
||||
"clip_vision": ("CLIP_VISION",),
|
||||
"image_schedule": ("INT", {"default": None, "forceInput": True} ),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1283,6 +1297,37 @@ class IPAdapterLoadEmbeds:
|
||||
path = folder_paths.get_annotated_filepath(embeds)
|
||||
return (torch.load(path).cpu(), )
|
||||
|
||||
defaultValue="""0:0,
|
||||
40:1,
|
||||
80:2,
|
||||
"""
|
||||
class IPAdapterImageSchedule:
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {"required": {"text": ("STRING", {"multiline": True, "default": defaultValue}),
|
||||
"max_frames": ("INT", {"default": 120.0, "min": 1.0, "max": 999999.0, "step": 1.0}),
|
||||
"print_output": ("BOOLEAN", {"default": False})}}
|
||||
|
||||
RETURN_TYPES = ("INT",)
|
||||
FUNCTION = "schedule"
|
||||
|
||||
CATEGORY = "ipadapter/utils"
|
||||
|
||||
def schedule(self, text, max_frames, print_output):
|
||||
frames = [0] * max_frames
|
||||
for item in text.split(","):
|
||||
item = item.strip()
|
||||
if ":" in item:
|
||||
parts = item.split(":")
|
||||
if len(parts) == 2:
|
||||
start_frame = int(parts[0])
|
||||
value = int(parts[1])
|
||||
for i in range(start_frame, max_frames):
|
||||
frames[i] = value
|
||||
if print_output is True:
|
||||
print("ValueSchedule: ", frames)
|
||||
return (frames, )
|
||||
|
||||
class IPAdapterWeights:
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
|
||||
@@ -61,7 +61,7 @@ def get_ipadapter_file(preset, is_sdxl):
|
||||
if is_sdxl:
|
||||
raise Exception("full face model is not supported for SDXL")
|
||||
pattern = 'full.face.sd15\.(safetensors|bin)$'
|
||||
elif preset.startswith("faceid portrait"):
|
||||
elif preset.startswith("faceid portrait ("):
|
||||
if is_sdxl:
|
||||
pattern = 'portrait.sdxl\.(safetensors|bin)$'
|
||||
else:
|
||||
@@ -70,6 +70,12 @@ def get_ipadapter_file(preset, is_sdxl):
|
||||
if not [e for e in ipadapter_list if re.search(pattern, e, re.IGNORECASE)]:
|
||||
pattern = 'portrait.sd15\.(safetensors|bin)$'
|
||||
is_insightface = True
|
||||
elif preset.startswith("faceid portrait unnorm"):
|
||||
if is_sdxl:
|
||||
pattern = 'portrait.sdxl.unnorm\.(safetensors|bin)$'
|
||||
else:
|
||||
raise Exception("portrait unnorm model is not supported for SD1.5")
|
||||
is_insightface = True
|
||||
elif preset == "faceid":
|
||||
if is_sdxl:
|
||||
pattern = 'faceid.sdxl\.(safetensors|bin)$'
|
||||
@@ -131,6 +137,9 @@ def ipadapter_model_loader(file):
|
||||
|
||||
if 'plusv2' in file.lower():
|
||||
model["faceidplusv2"] = True
|
||||
|
||||
if 'unnorm' in file.lower():
|
||||
model["portraitunnorm"] = True
|
||||
|
||||
return model
|
||||
|
||||
@@ -145,21 +154,34 @@ def insightface_loader(provider):
|
||||
model.prepare(ctx_id=0, det_size=(640, 640))
|
||||
return model
|
||||
|
||||
def encode_image_masked(clip_vision, image, mask=None):
|
||||
def encode_image_masked(clip_vision, images, mask=None):
|
||||
model_management.load_model_gpu(clip_vision.patcher)
|
||||
image = image.to(clip_vision.load_device)
|
||||
|
||||
pixel_values = clip_preprocess(image.to(clip_vision.load_device)).float()
|
||||
# Initialize lists to collect outputs
|
||||
last_hidden_states = []
|
||||
image_embeds = []
|
||||
penultimate_hidden_states = []
|
||||
|
||||
if mask is not None:
|
||||
pixel_values = pixel_values * mask.to(clip_vision.load_device)
|
||||
# Loop over each image in the batch
|
||||
for image in images:
|
||||
pixel_values = clip_preprocess(image.to(clip_vision.load_device).unsqueeze(0)).float()
|
||||
|
||||
out = clip_vision.model(pixel_values=pixel_values, intermediate_output=-2)
|
||||
if mask is not None:
|
||||
pixel_values *= mask.to(clip_vision.load_device)
|
||||
|
||||
out = clip_vision.model(pixel_values=pixel_values, intermediate_output=-2)
|
||||
|
||||
# Collect the outputs for each image
|
||||
last_hidden_states.append(out[0].to(model_management.intermediate_device()))
|
||||
image_embeds.append(out[2].to(model_management.intermediate_device()))
|
||||
penultimate_hidden_states.append(out[1].to(model_management.intermediate_device()))
|
||||
|
||||
# Concatenate all collected outputs across the batch
|
||||
outputs = Output()
|
||||
outputs["last_hidden_state"] = out[0].to(model_management.intermediate_device())
|
||||
outputs["image_embeds"] = out[2].to(model_management.intermediate_device())
|
||||
outputs["penultimate_hidden_states"] = out[1].to(model_management.intermediate_device())
|
||||
outputs["last_hidden_state"] = torch.cat(last_hidden_states, dim=0)
|
||||
outputs["image_embeds"] = torch.cat(image_embeds, dim=0)
|
||||
outputs["penultimate_hidden_states"] = torch.cat(penultimate_hidden_states, dim=0)
|
||||
|
||||
return outputs
|
||||
|
||||
def tensor_to_size(source, dest_size):
|
||||
|
||||
Reference in New Issue
Block a user