From f4000bf8d7335117b55ff4ee263147372d5d2f10 Mon Sep 17 00:00:00 2001 From: IDGallagher Date: Fri, 19 Apr 2024 21:19:59 +0100 Subject: [PATCH 1/7] Updated IPAdapter --- .../CrossAttentionPatchImport.py | 33 ++++++---- .../ComfyUI_IPAdapter_plus/IPAdapterPlus.py | 65 +++++++++++++++---- imports/ComfyUI_IPAdapter_plus/utils.py | 42 +++++++++--- 3 files changed, 107 insertions(+), 33 deletions(-) diff --git a/imports/ComfyUI_IPAdapter_plus/CrossAttentionPatchImport.py b/imports/ComfyUI_IPAdapter_plus/CrossAttentionPatchImport.py index 78b3601..c615970 100644 --- a/imports/ComfyUI_IPAdapter_plus/CrossAttentionPatchImport.py +++ b/imports/ComfyUI_IPAdapter_plus/CrossAttentionPatchImport.py @@ -6,7 +6,7 @@ from .utils import tensor_to_size class CrossAttentionPatchImport: # forward for patching - def __init__(self, ipadapter=None, number=0, weight=1.0, cond=None, cond_alt=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, embeds_scaling='V only'): + def __init__(self, ipadapter=None, number=0, weight=1.0, cond=None, cond_alt=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, image_schedule=None, embeds_scaling='V only'): self.weights = [weight] self.ipadapters = [ipadapter] self.conds = [cond] @@ -17,6 +17,7 @@ class CrossAttentionPatchImport: self.sigma_starts = [sigma_start] self.sigma_ends = [sigma_end] self.unfold_batch = [unfold_batch] + self.image_schedule = [image_schedule] self.embeds_scaling = [embeds_scaling] self.number = number self.layers = 11 if '101_to_k_ip' in ipadapter.ip_layers.to_kvs else 16 # TODO: check if this is a valid condition to detect all models @@ -24,7 +25,7 @@ class CrossAttentionPatchImport: self.k_key = str(self.number*2+1) + "_to_k_ip" self.v_key = str(self.number*2+1) + "_to_v_ip" - def set_new_condition(self, ipadapter=None, number=0, weight=1.0, cond=None, cond_alt=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, embeds_scaling='V only'): + def set_new_condition(self, ipadapter=None, number=0, weight=1.0, cond=None, cond_alt=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, image_schedule=None, embeds_scaling='V only'): self.weights.append(weight) self.ipadapters.append(ipadapter) self.conds.append(cond) @@ -35,6 +36,7 @@ class CrossAttentionPatchImport: self.sigma_starts.append(sigma_start) self.sigma_ends.append(sigma_end) self.unfold_batch.append(unfold_batch) + self.image_schedule.append(image_schedule) self.embeds_scaling.append(embeds_scaling) def __call__(self, q, k, v, extra_options): @@ -54,7 +56,7 @@ class CrossAttentionPatchImport: out = optimized_attention(q, k, v, extra_options["n_heads"]) _, _, oh, ow = extra_options["original_shape"] - for weight, cond, cond_alt, uncond, ipadapter, mask, weight_type, sigma_start, sigma_end, unfold_batch, embeds_scaling in zip(self.weights, self.conds, self.conds_alt, self.unconds, self.ipadapters, self.masks, self.weight_types, self.sigma_starts, self.sigma_ends, self.unfold_batch, self.embeds_scaling): + for weight, cond, cond_alt, uncond, ipadapter, mask, weight_type, sigma_start, sigma_end, unfold_batch, image_schedule, embeds_scaling in zip(self.weights, self.conds, self.conds_alt, self.unconds, self.ipadapters, self.masks, self.weight_types, self.sigma_starts, self.sigma_ends, self.unfold_batch, self.image_schedule, self.embeds_scaling): if sigma <= sigma_start and sigma >= sigma_end: if weight_type == 'ease in': weight = weight * (0.05 + 0.95 * (1 - t_idx / self.layers)) @@ -94,16 +96,23 @@ class CrossAttentionPatchImport: elif weight == 0: continue - # if image length matches or exceeds full_length get sub_idx images - if cond.shape[0] >= ad_params["full_length"]: - cond = torch.Tensor(cond[ad_params["sub_idxs"]]) - uncond = torch.Tensor(uncond[ad_params["sub_idxs"]]) - # otherwise get sub_idxs images + if image_schedule is not None: + # Use the image_schedule as a lookup table to get the embedded image corresponding to each sub_idx + # If image_schedule isn't long enough then use the last image + cond_idxs = [image_schedule[i if i < len(image_schedule) else -1] for i in ad_params["sub_idxs"]] + cond = torch.Tensor(cond[cond_idxs]) + uncond = torch.Tensor(uncond[cond_idxs]) else: - cond = tensor_to_size(cond, ad_params["full_length"]) - uncond = tensor_to_size(uncond, ad_params["full_length"]) - cond = cond[ad_params["sub_idxs"]] - uncond = uncond[ad_params["sub_idxs"]] + # if image length matches or exceeds full_length get sub_idx images + if cond.shape[0] >= ad_params["full_length"]: + cond = torch.Tensor(cond[ad_params["sub_idxs"]]) + uncond = torch.Tensor(uncond[ad_params["sub_idxs"]]) + # otherwise get sub_idxs images + else: + cond = tensor_to_size(cond, ad_params["full_length"]) + uncond = tensor_to_size(uncond, ad_params["full_length"]) + cond = cond[ad_params["sub_idxs"]] + uncond = uncond[ad_params["sub_idxs"]] else: if isinstance(weight, torch.Tensor): weight = tensor_to_size(weight, batch_prompt) diff --git a/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py b/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py index 32b5012..9413999 100644 --- a/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py +++ b/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py @@ -45,7 +45,7 @@ WEIGHT_TYPES = ["linear", "ease in", "ease out", 'ease in-out', 'reverse in-out' ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ """ class IPAdapterImport(nn.Module): - def __init__(self, ipadapter_model, cross_attention_dim=1024, output_cross_attention_dim=1024, clip_embeddings_dim=1024, clip_extra_context_tokens=4, is_sdxl=False, is_plus=False, is_full=False, is_faceid=False): + def __init__(self, ipadapter_model, cross_attention_dim=1024, output_cross_attention_dim=1024, clip_embeddings_dim=1024, clip_extra_context_tokens=4, is_sdxl=False, is_plus=False, is_full=False, is_faceid=False, is_portrait_unnorm=False): super().__init__() self.clip_embeddings_dim = clip_embeddings_dim @@ -55,12 +55,13 @@ class IPAdapterImport(nn.Module): self.is_sdxl = is_sdxl self.is_full = is_full self.is_plus = is_plus + self.is_portrait_unnorm = is_portrait_unnorm - if is_faceid: + if is_faceid and not is_portrait_unnorm: self.image_proj_model = self.init_proj_faceid() elif is_full: self.image_proj_model = self.init_proj_full() - elif is_plus: + elif is_plus or is_portrait_unnorm: self.image_proj_model = self.init_proj_plus() else: self.image_proj_model = self.init_proj() @@ -161,6 +162,7 @@ def ipadapter_execute(model, pos_embed=None, neg_embed=None, unfold_batch=False, + image_schedule=None, embeds_scaling='V only', layer_weights=None): device = model_management.get_torch_device() @@ -170,8 +172,9 @@ def ipadapter_execute(model, is_full = "proj.3.weight" in ipadapter["image_proj"] is_portrait = "proj.2.weight" in ipadapter["image_proj"] and not "proj.3.weight" in ipadapter["image_proj"] and not "0.to_q_lora.down.weight" in ipadapter["ip_adapter"] - is_faceid = is_portrait or "0.to_q_lora.down.weight" in ipadapter["ip_adapter"] - is_plus = is_full or "latents" in ipadapter["image_proj"] or "perceiver_resampler.proj_in.weight" in ipadapter["image_proj"] + is_portrait_unnorm = "portraitunnorm" in ipadapter + is_faceid = is_portrait or "0.to_q_lora.down.weight" in ipadapter["ip_adapter"] or is_portrait_unnorm + is_plus = (is_full or "latents" in ipadapter["image_proj"] or "perceiver_resampler.proj_in.weight" in ipadapter["image_proj"]) and not is_portrait_unnorm is_faceidv2 = "faceidplusv2" in ipadapter output_cross_attention_dim = ipadapter["ip_adapter"]["1.to_k_ip.weight"].shape[1] is_sdxl = output_cross_attention_dim == 2048 @@ -182,8 +185,8 @@ def ipadapter_execute(model, if is_faceidv2: weight_faceidv2 = weight_faceidv2 if weight_faceidv2 is not None else weight*2 - cross_attention_dim = 1280 if is_plus and is_sdxl and not is_faceid else output_cross_attention_dim - clip_extra_context_tokens = 16 if (is_plus and not is_faceid) or is_portrait else 4 + cross_attention_dim = 1280 if (is_plus and is_sdxl and not is_faceid) or is_portrait_unnorm else output_cross_attention_dim + clip_extra_context_tokens = 16 if (is_plus and not is_faceid) or is_portrait or is_portrait_unnorm else 4 if image is not None and image.shape[1] != image.shape[2]: print("\033[33mINFO: the IPAdapter reference image is not a square, CLIPImageProcessor will resize and crop it at the center. If the main focus of the picture is not in the middle the result might not be what you are expecting.\033[0m") @@ -233,7 +236,10 @@ def ipadapter_execute(model, insightface.det_model.input_size = size # TODO: hacky but seems to be working face = insightface.get(image_iface[i]) if face: - face_cond_embeds.append(torch.from_numpy(face[0].normed_embedding).unsqueeze(0)) + if not is_portrait_unnorm: + face_cond_embeds.append(torch.from_numpy(face[0].normed_embedding).unsqueeze(0)) + else: + face_cond_embeds.append(torch.from_numpy(face[0].embedding).unsqueeze(0)) image.append(image_to_tensor(face_align.norm_crop(image_iface[i], landmark=face[0].kps, image_size=256))) if 640 not in size: @@ -330,7 +336,8 @@ def ipadapter_execute(model, is_sdxl=is_sdxl, is_plus=is_plus, is_full=is_full, - is_faceid=is_faceid + is_faceid=is_faceid, + is_portrait_unnorm=is_portrait_unnorm, ).to(device, dtype=dtype) if is_faceid and is_plus: @@ -365,6 +372,7 @@ def ipadapter_execute(model, "sigma_start": sigma_start, "sigma_end": sigma_end, "unfold_batch": unfold_batch, + "image_schedule": image_schedule, "embeds_scaling": embeds_scaling, } @@ -492,7 +500,7 @@ class IPAdapterUnifiedLoaderFaceID(IPAdapterUnifiedLoader): def INPUT_TYPES(s): return {"required": { "model": ("MODEL", ), - "preset": (['FACEID', 'FACEID PLUS - SD1.5 only', 'FACEID PLUS V2', 'FACEID PORTRAIT (style transfer)'], ), + "preset": (['FACEID', 'FACEID PLUS - SD1.5 only', 'FACEID PLUS V2', 'FACEID PORTRAIT (style transfer)', 'FACEID PORTRAIT UNNORM - SDXL only (strong)'], ), "lora_strength": ("FLOAT", { "default": 0.6, "min": 0, "max": 1, "step": 0.01 }), "provider": (["CPU", "CUDA", "ROCM", "DirectML", "OpenVINO", "CoreML"], ), }, @@ -626,7 +634,7 @@ class IPAdapterAdvancedImport: FUNCTION = "apply_ipadapter" CATEGORY = "ipadapter" - def apply_ipadapter(self, model, ipadapter, start_at, end_at, weight = 1.0, weight_style=1.0, weight_composition=1.0, expand_style=False, weight_type="linear", combine_embeds="concat", weight_faceidv2=None, image=None, image_style=None, image_composition=None, image_negative=None, clip_vision=None, attn_mask=None, insightface=None, embeds_scaling='V only', layer_weights=None): + def apply_ipadapter(self, model, ipadapter, start_at, end_at, weight = 1.0, weight_style=1.0, weight_composition=1.0, expand_style=False, weight_type="linear", combine_embeds="concat", weight_faceidv2=None, image=None, image_style=None, image_composition=None, image_negative=None, clip_vision=None, image_schedule=None, attn_mask=None, insightface=None, embeds_scaling='V only', layer_weights=None): is_sdxl = isinstance(model.model, (comfy.model_base.SDXL, comfy.model_base.SDXLRefiner, comfy.model_base.SDXL_instructpix2pix)) if image_style is not None: # we are doing style + composition transfer @@ -653,6 +661,7 @@ class IPAdapterAdvancedImport: "end_at": end_at, "attn_mask": attn_mask, "unfold_batch": self.unfold_batch, + "image_schedule": image_schedule, "embeds_scaling": embeds_scaling, "insightface": insightface if insightface is not None else ipadapter['insightface']['model'] if 'insightface' in ipadapter else None, "layer_weights": layer_weights, @@ -693,6 +702,7 @@ class IPAdapterBatchImport(IPAdapterAdvancedImport): "image_negative": ("IMAGE",), "attn_mask": ("MASK",), "clip_vision": ("CLIP_VISION",), + "image_schedule": ("INT", {"default": None, "forceInput": True} ), } } @@ -745,6 +755,7 @@ class IPAdapterStyleCompositionBatch(IPAdapterStyleComposition): "image_negative": ("IMAGE",), "attn_mask": ("MASK",), "clip_vision": ("CLIP_VISION",), + "image_schedule": ("INT", {"default": None, "forceInput": True} ), } } @@ -935,6 +946,7 @@ class IPAdapterTiledBatchImport(IPAdapterTiledImport): "image_negative": ("IMAGE",), "attn_mask": ("MASK",), "clip_vision": ("CLIP_VISION",), + "image_schedule": ("INT", {"default": None, "forceInput": True} ), } } @@ -1283,6 +1295,37 @@ class IPAdapterLoadEmbeds: path = folder_paths.get_annotated_filepath(embeds) return (torch.load(path).cpu(), ) +defaultValue="""0:0, +40:1, +80:2, +""" +class IPAdapterImageSchedule: + @classmethod + def INPUT_TYPES(s): + return {"required": {"text": ("STRING", {"multiline": True, "default": defaultValue}), + "max_frames": ("INT", {"default": 120.0, "min": 1.0, "max": 999999.0, "step": 1.0}), + "print_output": ("BOOLEAN", {"default": False})}} + + RETURN_TYPES = ("INT",) + FUNCTION = "schedule" + + CATEGORY = "ipadapter/utils" + + def schedule(self, text, max_frames, print_output): + frames = [0] * max_frames + for item in text.split(","): + item = item.strip() + if ":" in item: + parts = item.split(":") + if len(parts) == 2: + start_frame = int(parts[0]) + value = int(parts[1]) + for i in range(start_frame, max_frames): + frames[i] = value + if print_output is True: + print("ValueSchedule: ", frames) + return (frames, ) + class IPAdapterWeights: @classmethod def INPUT_TYPES(s): diff --git a/imports/ComfyUI_IPAdapter_plus/utils.py b/imports/ComfyUI_IPAdapter_plus/utils.py index e08b1d1..01f8ff5 100644 --- a/imports/ComfyUI_IPAdapter_plus/utils.py +++ b/imports/ComfyUI_IPAdapter_plus/utils.py @@ -61,7 +61,7 @@ def get_ipadapter_file(preset, is_sdxl): if is_sdxl: raise Exception("full face model is not supported for SDXL") pattern = 'full.face.sd15\.(safetensors|bin)$' - elif preset.startswith("faceid portrait"): + elif preset.startswith("faceid portrait ("): if is_sdxl: pattern = 'portrait.sdxl\.(safetensors|bin)$' else: @@ -70,6 +70,12 @@ def get_ipadapter_file(preset, is_sdxl): if not [e for e in ipadapter_list if re.search(pattern, e, re.IGNORECASE)]: pattern = 'portrait.sd15\.(safetensors|bin)$' is_insightface = True + elif preset.startswith("faceid portrait unnorm"): + if is_sdxl: + pattern = 'portrait.sdxl.unnorm\.(safetensors|bin)$' + else: + raise Exception("portrait unnorm model is not supported for SD1.5") + is_insightface = True elif preset == "faceid": if is_sdxl: pattern = 'faceid.sdxl\.(safetensors|bin)$' @@ -131,6 +137,9 @@ def ipadapter_model_loader(file): if 'plusv2' in file.lower(): model["faceidplusv2"] = True + + if 'unnorm' in file.lower(): + model["portraitunnorm"] = True return model @@ -145,21 +154,34 @@ def insightface_loader(provider): model.prepare(ctx_id=0, det_size=(640, 640)) return model -def encode_image_masked(clip_vision, image, mask=None): +def encode_image_masked(clip_vision, images, mask=None): model_management.load_model_gpu(clip_vision.patcher) - image = image.to(clip_vision.load_device) - pixel_values = clip_preprocess(image.to(clip_vision.load_device)).float() + # Initialize lists to collect outputs + last_hidden_states = [] + image_embeds = [] + penultimate_hidden_states = [] - if mask is not None: - pixel_values = pixel_values * mask.to(clip_vision.load_device) + # Loop over each image in the batch + for image in images: + pixel_values = clip_preprocess(image.to(clip_vision.load_device).unsqueeze(0)).float() - out = clip_vision.model(pixel_values=pixel_values, intermediate_output=-2) + if mask is not None: + pixel_values *= mask.to(clip_vision.load_device) + out = clip_vision.model(pixel_values=pixel_values, intermediate_output=-2) + + # Collect the outputs for each image + last_hidden_states.append(out[0].to(model_management.intermediate_device())) + image_embeds.append(out[2].to(model_management.intermediate_device())) + penultimate_hidden_states.append(out[1].to(model_management.intermediate_device())) + + # Concatenate all collected outputs across the batch outputs = Output() - outputs["last_hidden_state"] = out[0].to(model_management.intermediate_device()) - outputs["image_embeds"] = out[2].to(model_management.intermediate_device()) - outputs["penultimate_hidden_states"] = out[1].to(model_management.intermediate_device()) + outputs["last_hidden_state"] = torch.cat(last_hidden_states, dim=0) + outputs["image_embeds"] = torch.cat(image_embeds, dim=0) + outputs["penultimate_hidden_states"] = torch.cat(penultimate_hidden_states, dim=0) + return outputs def tensor_to_size(source, dest_size): From f37626687372a8493a10dcc4c3c205522d79bfdf Mon Sep 17 00:00:00 2001 From: IDGallagher Date: Tue, 23 Apr 2024 11:29:22 +0100 Subject: [PATCH 2/7] . --- SteerableMotion.py | 73 ++++++++++++++++++- .../ComfyUI_IPAdapter_plus/IPAdapterPlus.py | 8 +- 2 files changed, 77 insertions(+), 4 deletions(-) diff --git a/SteerableMotion.py b/SteerableMotion.py index 8553e51..c63ebc0 100644 --- a/SteerableMotion.py +++ b/SteerableMotion.py @@ -10,6 +10,7 @@ import matplotlib.pyplot as plt # Local application/library specific imports from .imports.ComfyUI_IPAdapter_plus.IPAdapterPlus import IPAdapterBatchImport, IPAdapterTiledBatchImport, IPAdapterTiledImport, PrepImageForClipVisionImport, IPAdapterAdvancedImport, IPAdapterNoiseImport from .imports.AdvancedControlNet.nodes_sparsectrl import SparseIndexMethodNodeImport +import gc class BatchCreativeInterpolationNode: @@ -379,10 +380,11 @@ class BatchCreativeInterpolationNode: keyframe_position = keyframe_positions[i] interpolation = "ease-in-out" # strength_from = strength_to = 1.0 - + image_index = 0 if i == 0: # buffer image = images[0] + image_index = 0 strength_from = strength_to = strength_values[0][1] batch_index_from = 0 @@ -393,6 +395,7 @@ class BatchCreativeInterpolationNode: # GET IMAGE AND KEYFRAME INFLUENCE VALUES image = images[i-1] + image_index = i-1 key_frame_influence_from, key_frame_influence_to = key_frame_influence_values[i-1] start_strength, mid_strength, end_strength = strength_values[i-1] @@ -408,6 +411,7 @@ class BatchCreativeInterpolationNode: # GET IMAGE AND KEYFRAME INFLUENCE VALUES image = images[i-1] + image_index = i - 1 key_frame_influence_from,key_frame_influence_to = key_frame_influence_values[i-1] start_strength, mid_strength, end_strength = strength_values[i-1] if len(keyframe_positions) == 4: @@ -426,6 +430,7 @@ class BatchCreativeInterpolationNode: elif i == len(keyframe_positions) - 1: # buffer image = images[i-2] + image_index = i - 2 strength_from = strength_to = strength_values[i-2][1] if len(keyframe_positions) == 4: @@ -440,7 +445,8 @@ class BatchCreativeInterpolationNode: else: # middle images # GET IMAGE AND KEYFRAME INFLUENCE VALUES - image = images[i-1] + image = images[i-1] + image_index = i - 1 key_frame_influence_from,key_frame_influence_to = key_frame_influence_values[i-1] start_strength, mid_strength, end_strength = strength_values[i-1] keyframe_position = keyframe_positions[i] @@ -464,6 +470,69 @@ class BatchCreativeInterpolationNode: # PROCESS WEIGHTS ipa_frame_numbers, ipa_weights = process_weights(frame_numbers, weights, 1.0) + print(f'i {i} image index {image_index} ====') + # print(f"frame numbers {frame_numbers}") + # print(f"weights {weights}") + print(f"frame numbers {ipa_frame_numbers}") + print(f"weights {ipa_weights}") + print("------") + + # class IPBin: + + # def __init__(self): + # self.indicies = [] + # self.image_schedule = [] + # self.weight_schedule = [] + # self.imageBatch = [] + + # def length(self): + # return len(self.indicies) + + # def add(self, image, image_index, frame_numbers, weights): + # # Search for image index, if it isn't there add the image + # try: + # index = self.indicies.index(image_index) + # except ValueError: + # self.imageBatch.append(image) + # self.indicies.append(image_index) + # index = len(self.indicies) - 1 + # self.image_schedule.extend([index] * (frame_numbers[-1] - len(self.image_schedule))) + # self.weight_schedule.extend([0] * (frame_numbers[0] - len(self.weight_schedule))) + # self.weight_schedule.extend(weights) + + + # # Start with one bin + # bins = [IPBin()] + # active_index = -1 + # # Find a bin that we can fit the next image into + # for i, bin in bins: + # if bin.length() <= ipa_frame_numbers[0]: + # active_index = i + # break1 + # # If we didn't find a suitable bin, add a new one + # if active_index == -1: + # bins.append(IPBin()) + # active_index = len(bins) - 1 + + # bins[active_index].add(image, image_index, ipa_frame_numbers, ipa_weights) + + # active_image_schedule = image_schedule_1 + # # Choose active image_schedule + # if len(image_schedule_1) == 0 or image_index == image_indicies_1[-1]: + # active_image_schedule = image_schedule_1 + # elif image_index == image_schedule_2[-1]: + # active_image_schedule = image_schedule_2 + # elif ipa_frame_numbers[0] >= len(image_schedule_1): + # active_image_schedule = image_schedule_1 + # elif ipa_frame_numbers[0] >= len(image_schedule_2): + # active_image_schedule = image_schedule_2 + # else: + # raise ValueError("This shouldn't happen!") + # # Fill the gaps + + # # Add image and weights + + prepare_for_clip_vision = PrepImageForClipVisionImport() prepped_image, = prepare_for_clip_vision.prep_image(image=image.unsqueeze(0), interpolation="LANCZOS", crop_position="pad", sharpening=0.1) diff --git a/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py b/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py index 9413999..d228b4d 100644 --- a/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py +++ b/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py @@ -191,8 +191,12 @@ def ipadapter_execute(model, if image is not None and image.shape[1] != image.shape[2]: print("\033[33mINFO: the IPAdapter reference image is not a square, CLIPImageProcessor will resize and crop it at the center. If the main focus of the picture is not in the middle the result might not be what you are expecting.\033[0m") - if isinstance(weight, list): - weight = torch.tensor(weight).unsqueeze(-1).unsqueeze(-1).to(device, dtype=dtype) if unfold_batch else weight[0] + weight = [1, 2, 6, 7, 3, 4] + if isinstance(weight, list) and isinstance(weight[0], list): + weight = [torch.tensor(w).unsqueeze(-1).unsqueeze(-1).to(device, dtype=dtype) for w in weight] + elif isinstance(weight, list): + weight = torch.tensor(weight).unsqueeze(-1).unsqueeze(-1).to(device, dtype=dtype) if unfold_batch else weight[0] + print(weight) # special weight types if layer_weights is not None and layer_weights != '': From ffd8d740648555abc29c6fdf4df9b0729e2fe9c3 Mon Sep 17 00:00:00 2001 From: IDGallagher Date: Wed, 24 Apr 2024 09:46:59 +0100 Subject: [PATCH 3/7] Batching working for non-detailed ipadapter --- SteerableMotion.py | 148 +++++++++--------- .../ComfyUI_IPAdapter_plus/IPAdapterPlus.py | 8 +- 2 files changed, 79 insertions(+), 77 deletions(-) diff --git a/SteerableMotion.py b/SteerableMotion.py index c63ebc0..d3cc740 100644 --- a/SteerableMotion.py +++ b/SteerableMotion.py @@ -10,8 +10,6 @@ import matplotlib.pyplot as plt # Local application/library specific imports from .imports.ComfyUI_IPAdapter_plus.IPAdapterPlus import IPAdapterBatchImport, IPAdapterTiledBatchImport, IPAdapterTiledImport, PrepImageForClipVisionImport, IPAdapterAdvancedImport, IPAdapterNoiseImport from .imports.AdvancedControlNet.nodes_sparsectrl import SparseIndexMethodNodeImport -import gc - class BatchCreativeInterpolationNode: @classmethod @@ -369,11 +367,38 @@ class BatchCreativeInterpolationNode: else: last_key_frame_position = (keyframe_positions[-1]) + class IPBin: + def __init__(self): + self.indicies = [] + self.image_schedule = [] + self.weight_schedule = [] + self.imageBatch = [] + + def length(self): + return len(self.image_schedule) + + def add(self, image, image_index, frame_numbers, weights, general_weight): + # Map frames to their corresponding reversed weights for easy lookup + frame_to_weight = {frame: weights[i] for i, frame in enumerate(frame_numbers)} + # Search for image index, if it isn't there add the image + try: + index = self.indicies.index(image_index) + except ValueError: + self.imageBatch.append(image) + self.indicies.append(image_index) + index = self.indicies.index(image_index) + + self.image_schedule.extend([index] * (frame_numbers[-1] + 1 - len(self.image_schedule))) + self.weight_schedule.extend([0] * (frame_numbers[0] - len(self.weight_schedule))) + self.weight_schedule.extend(general_weight * frame_to_weight[frame] for frame in range(frame_numbers[0], frame_numbers[-1] + 1)) + # CREATE LISTS FOR WEIGHTS AND FRAME NUMBERS all_cn_frame_numbers = [] all_cn_weights = [] all_ipa_weights = [] all_ipa_frame_numbers = [] + # Start with one bin + bins = [IPBin()] for i in range(len(keyframe_positions)): @@ -477,45 +502,26 @@ class BatchCreativeInterpolationNode: print(f"weights {ipa_weights}") print("------") - # class IPBin: - - # def __init__(self): - # self.indicies = [] - # self.image_schedule = [] - # self.weight_schedule = [] - # self.imageBatch = [] + prepare_for_clip_vision = PrepImageForClipVisionImport() + prepped_image, = prepare_for_clip_vision.prep_image(image=image.unsqueeze(0), interpolation="LANCZOS", crop_position="pad", sharpening=0.1) - # def length(self): - # return len(self.indicies) - - # def add(self, image, image_index, frame_numbers, weights): - # # Search for image index, if it isn't there add the image - # try: - # index = self.indicies.index(image_index) - # except ValueError: - # self.imageBatch.append(image) - # self.indicies.append(image_index) - # index = len(self.indicies) - 1 - # self.image_schedule.extend([index] * (frame_numbers[-1] - len(self.image_schedule))) - # self.weight_schedule.extend([0] * (frame_numbers[0] - len(self.weight_schedule))) - # self.weight_schedule.extend(weights) - - - # # Start with one bin - # bins = [IPBin()] - # active_index = -1 - # # Find a bin that we can fit the next image into - # for i, bin in bins: - # if bin.length() <= ipa_frame_numbers[0]: - # active_index = i - # break1 - # # If we didn't find a suitable bin, add a new one - # if active_index == -1: - # bins.append(IPBin()) - # active_index = len(bins) - 1 - - # bins[active_index].add(image, image_index, ipa_frame_numbers, ipa_weights) + active_index = -1 + # Find a bin that we can fit the next image into + for i, bin in enumerate(bins): + if bin.length() <= ipa_frame_numbers[0]: + active_index = i + break + # If we didn't find a suitable bin, add a new one + if active_index == -1: + bins.append(IPBin()) + active_index = len(bins) - 1 + # Add the image to the bin + bins[active_index].add(prepped_image, image_index, ipa_frame_numbers, ipa_weights, base_ipa_advanced_settings["ipa_weight"]) + for i, bin in enumerate(bins): + print(f"{i} schedule {bin.image_schedule}") + print(f"{i} weights {bin.weight_schedule}") + i += 1 # active_image_schedule = image_schedule_1 # # Choose active image_schedule # if len(image_schedule_1) == 0 or image_index == image_indicies_1[-1]: @@ -531,43 +537,43 @@ class BatchCreativeInterpolationNode: # # Fill the gaps # # Add image and weights + + # weight_batch = create_weight_batch(last_key_frame_position, ipa_weights, ipa_frame_numbers) + # print(f"weight batch {weight_batch}") - - prepare_for_clip_vision = PrepImageForClipVisionImport() - prepped_image, = prepare_for_clip_vision.prep_image(image=image.unsqueeze(0), interpolation="LANCZOS", crop_position="pad", sharpening=0.1) - - weight_batch = create_weight_batch(last_key_frame_position, ipa_weights, ipa_frame_numbers) - - if base_ipa_advanced_settings["ipa_noise_strength"] > 0: - if base_ipa_advanced_settings["use_image_for_noise"]: - noise_image = prepped_image - else: - noise_image = None - ipa_noise = IPAdapterNoiseImport() - negative_noise, = ipa_noise.make_noise(type=base_ipa_advanced_settings["type_of_noise"], strength=base_ipa_advanced_settings["ipa_noise_strength"], blur=base_ipa_advanced_settings["noise_blur"], image_optional=noise_image) - else: - negative_noise = None - - ipadapter_application = IPAdapterBatchImport() - model, = ipadapter_application.apply_ipadapter(model=model, ipadapter=ipadapter, image=prepped_image, weight=weight_batch*base_ipa_advanced_settings["ipa_weight"], weight_type=base_ipa_advanced_settings["ipa_weight_type"], start_at=base_ipa_advanced_settings["ipa_starts_at"], end_at=base_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,image_negative=negative_noise,embeds_scaling=base_ipa_advanced_settings["ipa_embeds_scaling"]) - - if high_detail_mode: - if detail_ipa_advanced_settings["ipa_noise_strength"] > 0: - if detail_ipa_advanced_settings["use_image_for_noise"]: - noise_image = image.unsqueeze(0) - else: - noise_image = None - ipa_noise = IPAdapterNoiseImport() - negative_noise, = ipa_noise.make_noise(type=detail_ipa_advanced_settings["type_of_noise"], strength=detail_ipa_advanced_settings["ipa_noise_strength"], blur=detail_ipa_advanced_settings["noise_blur"], image_optional=noise_image) - else: - negative_noise = None - - tiled_ipa_application = IPAdapterTiledBatchImport() - model, *_ = tiled_ipa_application.apply_tiled(model=model, ipadapter=ipadapter, image=image.unsqueeze(0), weight=weight_batch*base_ipa_advanced_settings["ipa_weight"], weight_type=detail_ipa_advanced_settings["ipa_weight_type"], start_at=detail_ipa_advanced_settings["ipa_starts_at"], end_at=detail_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,sharpening=0.1,image_negative=negative_noise,embeds_scaling=detail_ipa_advanced_settings["ipa_embeds_scaling"]) + all_ipa_frame_numbers.append(ipa_frame_numbers) all_ipa_weights.append(ipa_weights) + # if base_ipa_advanced_settings["ipa_noise_strength"] > 0: + # if base_ipa_advanced_settings["use_image_for_noise"]: + # noise_image = prepped_image + # else: + # noise_image = None + # ipa_noise = IPAdapterNoiseImport() + # negative_noise, = ipa_noise.make_noise(type=base_ipa_advanced_settings["type_of_noise"], strength=base_ipa_advanced_settings["ipa_noise_strength"], blur=base_ipa_advanced_settings["noise_blur"], image_optional=noise_image) + # else: + # negative_noise = None + negative_noise = None + for i, bin in enumerate(bins): + ipadapter_application = IPAdapterBatchImport() + model, = ipadapter_application.apply_ipadapter(model=model, ipadapter=ipadapter, image=torch.cat(bin.imageBatch, dim=0), weight=bin.weight_schedule, weight_type=base_ipa_advanced_settings["ipa_weight_type"], start_at=base_ipa_advanced_settings["ipa_starts_at"], end_at=base_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,image_negative=negative_noise,embeds_scaling=base_ipa_advanced_settings["ipa_embeds_scaling"], image_schedule=bin.image_schedule) + + # if high_detail_mode: + # if detail_ipa_advanced_settings["ipa_noise_strength"] > 0: + # if detail_ipa_advanced_settings["use_image_for_noise"]: + # noise_image = image.unsqueeze(0) + # else: + # noise_image = None + # ipa_noise = IPAdapterNoiseImport() + # negative_noise, = ipa_noise.make_noise(type=detail_ipa_advanced_settings["type_of_noise"], strength=detail_ipa_advanced_settings["ipa_noise_strength"], blur=detail_ipa_advanced_settings["noise_blur"], image_optional=noise_image) + # else: + # negative_noise = None + + # tiled_ipa_application = IPAdapterTiledBatchImport() + # model, *_ = tiled_ipa_application.apply_tiled(model=model, ipadapter=ipadapter, image=image.unsqueeze(0), weight=weight_batch*base_ipa_advanced_settings["ipa_weight"], weight_type=detail_ipa_advanced_settings["ipa_weight_type"], start_at=detail_ipa_advanced_settings["ipa_starts_at"], end_at=detail_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,sharpening=0.1,image_negative=negative_noise,embeds_scaling=detail_ipa_advanced_settings["ipa_embeds_scaling"]) + comparison_diagram, = plot_weight_comparison(all_cn_frame_numbers, all_cn_weights, all_ipa_frame_numbers, all_ipa_weights, buffer) return comparison_diagram, positive, negative, model, sparse_indexes, last_key_frame_position, buffer diff --git a/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py b/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py index d228b4d..2ca998b 100644 --- a/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py +++ b/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py @@ -191,12 +191,8 @@ def ipadapter_execute(model, if image is not None and image.shape[1] != image.shape[2]: print("\033[33mINFO: the IPAdapter reference image is not a square, CLIPImageProcessor will resize and crop it at the center. If the main focus of the picture is not in the middle the result might not be what you are expecting.\033[0m") - weight = [1, 2, 6, 7, 3, 4] - if isinstance(weight, list) and isinstance(weight[0], list): - weight = [torch.tensor(w).unsqueeze(-1).unsqueeze(-1).to(device, dtype=dtype) for w in weight] - elif isinstance(weight, list): - weight = torch.tensor(weight).unsqueeze(-1).unsqueeze(-1).to(device, dtype=dtype) if unfold_batch else weight[0] - print(weight) + if isinstance(weight, list): + weight = torch.tensor(weight).unsqueeze(-1).unsqueeze(-1).to(device, dtype=dtype) if unfold_batch else weight[0] # special weight types if layer_weights is not None and layer_weights != '': From 8d29e7f33cc46e4a85ab6a23102a102ae08522c3 Mon Sep 17 00:00:00 2001 From: IDGallagher Date: Wed, 24 Apr 2024 15:16:22 +0100 Subject: [PATCH 4/7] Batching working for detailed images --- SteerableMotion.py | 47 +++++++------------ .../ComfyUI_IPAdapter_plus/IPAdapterPlus.py | 4 +- 2 files changed, 20 insertions(+), 31 deletions(-) diff --git a/SteerableMotion.py b/SteerableMotion.py index d3cc740..f0a17e9 100644 --- a/SteerableMotion.py +++ b/SteerableMotion.py @@ -373,11 +373,12 @@ class BatchCreativeInterpolationNode: self.image_schedule = [] self.weight_schedule = [] self.imageBatch = [] + self.bigImageBatch = [] def length(self): return len(self.image_schedule) - def add(self, image, image_index, frame_numbers, weights, general_weight): + def add(self, image, big_image, image_index, frame_numbers, weights, general_weight): # Map frames to their corresponding reversed weights for easy lookup frame_to_weight = {frame: weights[i] for i, frame in enumerate(frame_numbers)} # Search for image index, if it isn't there add the image @@ -385,6 +386,7 @@ class BatchCreativeInterpolationNode: index = self.indicies.index(image_index) except ValueError: self.imageBatch.append(image) + self.bigImageBatch.append(big_image) self.indicies.append(image_index) index = self.indicies.index(image_index) @@ -516,27 +518,12 @@ class BatchCreativeInterpolationNode: bins.append(IPBin()) active_index = len(bins) - 1 # Add the image to the bin - bins[active_index].add(prepped_image, image_index, ipa_frame_numbers, ipa_weights, base_ipa_advanced_settings["ipa_weight"]) + bins[active_index].add(prepped_image, image.unsqueeze(0), image_index, ipa_frame_numbers, ipa_weights, base_ipa_advanced_settings["ipa_weight"]) for i, bin in enumerate(bins): print(f"{i} schedule {bin.image_schedule}") print(f"{i} weights {bin.weight_schedule}") i += 1 - # active_image_schedule = image_schedule_1 - # # Choose active image_schedule - # if len(image_schedule_1) == 0 or image_index == image_indicies_1[-1]: - # active_image_schedule = image_schedule_1 - # elif image_index == image_schedule_2[-1]: - # active_image_schedule = image_schedule_2 - # elif ipa_frame_numbers[0] >= len(image_schedule_1): - # active_image_schedule = image_schedule_1 - # elif ipa_frame_numbers[0] >= len(image_schedule_2): - # active_image_schedule = image_schedule_2 - # else: - # raise ValueError("This shouldn't happen!") - # # Fill the gaps - - # # Add image and weights # weight_batch = create_weight_batch(last_key_frame_position, ipa_weights, ipa_frame_numbers) # print(f"weight batch {weight_batch}") @@ -560,19 +547,19 @@ class BatchCreativeInterpolationNode: ipadapter_application = IPAdapterBatchImport() model, = ipadapter_application.apply_ipadapter(model=model, ipadapter=ipadapter, image=torch.cat(bin.imageBatch, dim=0), weight=bin.weight_schedule, weight_type=base_ipa_advanced_settings["ipa_weight_type"], start_at=base_ipa_advanced_settings["ipa_starts_at"], end_at=base_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,image_negative=negative_noise,embeds_scaling=base_ipa_advanced_settings["ipa_embeds_scaling"], image_schedule=bin.image_schedule) - # if high_detail_mode: - # if detail_ipa_advanced_settings["ipa_noise_strength"] > 0: - # if detail_ipa_advanced_settings["use_image_for_noise"]: - # noise_image = image.unsqueeze(0) - # else: - # noise_image = None - # ipa_noise = IPAdapterNoiseImport() - # negative_noise, = ipa_noise.make_noise(type=detail_ipa_advanced_settings["type_of_noise"], strength=detail_ipa_advanced_settings["ipa_noise_strength"], blur=detail_ipa_advanced_settings["noise_blur"], image_optional=noise_image) - # else: - # negative_noise = None - - # tiled_ipa_application = IPAdapterTiledBatchImport() - # model, *_ = tiled_ipa_application.apply_tiled(model=model, ipadapter=ipadapter, image=image.unsqueeze(0), weight=weight_batch*base_ipa_advanced_settings["ipa_weight"], weight_type=detail_ipa_advanced_settings["ipa_weight_type"], start_at=detail_ipa_advanced_settings["ipa_starts_at"], end_at=detail_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,sharpening=0.1,image_negative=negative_noise,embeds_scaling=detail_ipa_advanced_settings["ipa_embeds_scaling"]) + if high_detail_mode: + # if detail_ipa_advanced_settings["ipa_noise_strength"] > 0: + # if detail_ipa_advanced_settings["use_image_for_noise"]: + # noise_image = image.unsqueeze(0) + # else: + # noise_image = None + # ipa_noise = IPAdapterNoiseImport() + # negative_noise, = ipa_noise.make_noise(type=detail_ipa_advanced_settings["type_of_noise"], strength=detail_ipa_advanced_settings["ipa_noise_strength"], blur=detail_ipa_advanced_settings["noise_blur"], image_optional=noise_image) + # else: + # negative_noise = None + negative_noise = None + tiled_ipa_application = IPAdapterTiledBatchImport() + model, *_ = tiled_ipa_application.apply_tiled(model=model, ipadapter=ipadapter, image=torch.cat(bin.bigImageBatch, dim=0), weight=bin.weight_schedule, weight_type=detail_ipa_advanced_settings["ipa_weight_type"], start_at=detail_ipa_advanced_settings["ipa_starts_at"], end_at=detail_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,sharpening=0.1,image_negative=negative_noise,embeds_scaling=detail_ipa_advanced_settings["ipa_embeds_scaling"], image_schedule=bin.image_schedule) comparison_diagram, = plot_weight_comparison(all_cn_frame_numbers, all_cn_weights, all_ipa_frame_numbers, all_ipa_weights, buffer) diff --git a/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py b/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py index 2ca998b..a6c84b8 100644 --- a/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py +++ b/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py @@ -820,7 +820,7 @@ class IPAdapterTiledImport: FUNCTION = "apply_tiled" CATEGORY = "ipadapter/tiled" - def apply_tiled(self, model, ipadapter, image, weight, weight_type, start_at, end_at, sharpening, combine_embeds="concat", image_negative=None, attn_mask=None, clip_vision=None, embeds_scaling='V only'): + def apply_tiled(self, model, ipadapter, image, weight, weight_type, start_at, end_at, sharpening, combine_embeds="concat", image_negative=None, attn_mask=None, clip_vision=None, embeds_scaling='V only', image_schedule=None): # 1. Select the models if 'ipadapter' in ipadapter: ipadapter_model = ipadapter['ipadapter']['model'] @@ -918,7 +918,9 @@ class IPAdapterTiledImport: "attn_mask": masks[i], "unfold_batch": self.unfold_batch, "embeds_scaling": embeds_scaling, + "image_schedule": image_schedule, } + # apply the ipadapter to the model without cloning it model = ipadapter_execute(model, ipadapter_model, clip_vision, **ipa_args) From 6da0c3940e253f451ea4a7826ce783186dfd4678 Mon Sep 17 00:00:00 2001 From: IDGallagher Date: Wed, 24 Apr 2024 18:26:36 +0100 Subject: [PATCH 5/7] Noise and detail fixes --- SteerableMotion.py | 67 +++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/SteerableMotion.py b/SteerableMotion.py index f0a17e9..4ecefbe 100644 --- a/SteerableMotion.py +++ b/SteerableMotion.py @@ -374,11 +374,13 @@ class BatchCreativeInterpolationNode: self.weight_schedule = [] self.imageBatch = [] self.bigImageBatch = [] + self.noiseBatch = [] + self.bigNoiseBatch = [] def length(self): return len(self.image_schedule) - def add(self, image, big_image, image_index, frame_numbers, weights, general_weight): + def add(self, image, big_image, noise, big_noise, image_index, frame_numbers, weights): # Map frames to their corresponding reversed weights for easy lookup frame_to_weight = {frame: weights[i] for i, frame in enumerate(frame_numbers)} # Search for image index, if it isn't there add the image @@ -387,12 +389,14 @@ class BatchCreativeInterpolationNode: except ValueError: self.imageBatch.append(image) self.bigImageBatch.append(big_image) + if noise is not None: self.noiseBatch.append(noise) + if big_noise is not None: self.bigNoiseBatch.append(big_noise) self.indicies.append(image_index) index = self.indicies.index(image_index) self.image_schedule.extend([index] * (frame_numbers[-1] + 1 - len(self.image_schedule))) self.weight_schedule.extend([0] * (frame_numbers[0] - len(self.weight_schedule))) - self.weight_schedule.extend(general_weight * frame_to_weight[frame] for frame in range(frame_numbers[0], frame_numbers[-1] + 1)) + self.weight_schedule.extend(frame_to_weight[frame] for frame in range(frame_numbers[0], frame_numbers[-1] + 1)) # CREATE LISTS FOR WEIGHTS AND FRAME NUMBERS all_cn_frame_numbers = [] @@ -506,6 +510,26 @@ class BatchCreativeInterpolationNode: prepare_for_clip_vision = PrepImageForClipVisionImport() prepped_image, = prepare_for_clip_vision.prep_image(image=image.unsqueeze(0), interpolation="LANCZOS", crop_position="pad", sharpening=0.1) + + if base_ipa_advanced_settings["ipa_noise_strength"] > 0: + if base_ipa_advanced_settings["use_image_for_noise"]: + noise_image = prepped_image + else: + noise_image = None + ipa_noise = IPAdapterNoiseImport() + negative_noise, = ipa_noise.make_noise(type=base_ipa_advanced_settings["type_of_noise"], strength=base_ipa_advanced_settings["ipa_noise_strength"], blur=base_ipa_advanced_settings["noise_blur"], image_optional=noise_image) + else: + negative_noise = None + + if high_detail_mode and detail_ipa_advanced_settings["ipa_noise_strength"] > 0: + if detail_ipa_advanced_settings["use_image_for_noise"]: + noise_image = image.unsqueeze(0) + else: + noise_image = None + ipa_noise = IPAdapterNoiseImport() + big_negative_noise, = ipa_noise.make_noise(type=detail_ipa_advanced_settings["type_of_noise"], strength=detail_ipa_advanced_settings["ipa_noise_strength"], blur=detail_ipa_advanced_settings["noise_blur"], image_optional=noise_image) + else: + big_negative_noise = None active_index = -1 # Find a bin that we can fit the next image into @@ -518,51 +542,26 @@ class BatchCreativeInterpolationNode: bins.append(IPBin()) active_index = len(bins) - 1 # Add the image to the bin - bins[active_index].add(prepped_image, image.unsqueeze(0), image_index, ipa_frame_numbers, ipa_weights, base_ipa_advanced_settings["ipa_weight"]) + bins[active_index].add(prepped_image, image.unsqueeze(0), negative_noise, big_negative_noise, image_index, ipa_frame_numbers, ipa_weights) for i, bin in enumerate(bins): print(f"{i} schedule {bin.image_schedule}") print(f"{i} weights {bin.weight_schedule}") i += 1 - # weight_batch = create_weight_batch(last_key_frame_position, ipa_weights, ipa_frame_numbers) - # print(f"weight batch {weight_batch}") - - - all_ipa_frame_numbers.append(ipa_frame_numbers) all_ipa_weights.append(ipa_weights) - # if base_ipa_advanced_settings["ipa_noise_strength"] > 0: - # if base_ipa_advanced_settings["use_image_for_noise"]: - # noise_image = prepped_image - # else: - # noise_image = None - # ipa_noise = IPAdapterNoiseImport() - # negative_noise, = ipa_noise.make_noise(type=base_ipa_advanced_settings["type_of_noise"], strength=base_ipa_advanced_settings["ipa_noise_strength"], blur=base_ipa_advanced_settings["noise_blur"], image_optional=noise_image) - # else: - # negative_noise = None - negative_noise = None for i, bin in enumerate(bins): ipadapter_application = IPAdapterBatchImport() - model, = ipadapter_application.apply_ipadapter(model=model, ipadapter=ipadapter, image=torch.cat(bin.imageBatch, dim=0), weight=bin.weight_schedule, weight_type=base_ipa_advanced_settings["ipa_weight_type"], start_at=base_ipa_advanced_settings["ipa_starts_at"], end_at=base_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,image_negative=negative_noise,embeds_scaling=base_ipa_advanced_settings["ipa_embeds_scaling"], image_schedule=bin.image_schedule) - - if high_detail_mode: - # if detail_ipa_advanced_settings["ipa_noise_strength"] > 0: - # if detail_ipa_advanced_settings["use_image_for_noise"]: - # noise_image = image.unsqueeze(0) - # else: - # noise_image = None - # ipa_noise = IPAdapterNoiseImport() - # negative_noise, = ipa_noise.make_noise(type=detail_ipa_advanced_settings["type_of_noise"], strength=detail_ipa_advanced_settings["ipa_noise_strength"], blur=detail_ipa_advanced_settings["noise_blur"], image_optional=noise_image) - # else: - # negative_noise = None - negative_noise = None - tiled_ipa_application = IPAdapterTiledBatchImport() - model, *_ = tiled_ipa_application.apply_tiled(model=model, ipadapter=ipadapter, image=torch.cat(bin.bigImageBatch, dim=0), weight=bin.weight_schedule, weight_type=detail_ipa_advanced_settings["ipa_weight_type"], start_at=detail_ipa_advanced_settings["ipa_starts_at"], end_at=detail_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,sharpening=0.1,image_negative=negative_noise,embeds_scaling=detail_ipa_advanced_settings["ipa_embeds_scaling"], image_schedule=bin.image_schedule) + negative_noise = torch.cat(bin.noiseBatch, dim=0) if len(bin.noiseBatch) > 0 else None + model, = ipadapter_application.apply_ipadapter(model=model, ipadapter=ipadapter, image=torch.cat(bin.imageBatch, dim=0), weight=[x * base_ipa_advanced_settings["ipa_weight"] for x in bin.weight_schedule], weight_type=base_ipa_advanced_settings["ipa_weight_type"], start_at=base_ipa_advanced_settings["ipa_starts_at"], end_at=base_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,image_negative=negative_noise,embeds_scaling=base_ipa_advanced_settings["ipa_embeds_scaling"], image_schedule=bin.image_schedule) + if high_detail_mode: + tiled_ipa_application = IPAdapterTiledBatchImport() + negative_noise = torch.cat(bin.bigNoiseBatch, dim=0) if len(bin.bigNoiseBatch) > 0 else None + model, *_ = tiled_ipa_application.apply_tiled(model=model, ipadapter=ipadapter, image=torch.cat(bin.bigImageBatch, dim=0), weight=[x * detail_ipa_advanced_settings["ipa_weight"] for x in bin.weight_schedule], weight_type=detail_ipa_advanced_settings["ipa_weight_type"], start_at=detail_ipa_advanced_settings["ipa_starts_at"], end_at=detail_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,sharpening=0.1,image_negative=negative_noise,embeds_scaling=detail_ipa_advanced_settings["ipa_embeds_scaling"], image_schedule=bin.image_schedule) comparison_diagram, = plot_weight_comparison(all_cn_frame_numbers, all_cn_weights, all_ipa_frame_numbers, all_ipa_weights, buffer) - return comparison_diagram, positive, negative, model, sparse_indexes, last_key_frame_position, buffer class IpaConfigurationNode: From dfe4d11197c1600f6de7bf04816fd7a06a52eafa Mon Sep 17 00:00:00 2001 From: IDGallagher Date: Thu, 25 Apr 2024 12:20:11 +0100 Subject: [PATCH 6/7] Comments --- SteerableMotion.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/SteerableMotion.py b/SteerableMotion.py index 4ecefbe..4f28e31 100644 --- a/SteerableMotion.py +++ b/SteerableMotion.py @@ -508,6 +508,7 @@ class BatchCreativeInterpolationNode: print(f"weights {ipa_weights}") print("------") + # Prepare images and noise prepare_for_clip_vision = PrepImageForClipVisionImport() prepped_image, = prepare_for_clip_vision.prep_image(image=image.unsqueeze(0), interpolation="LANCZOS", crop_position="pad", sharpening=0.1) @@ -531,6 +532,7 @@ class BatchCreativeInterpolationNode: else: big_negative_noise = None + # Fill up bins with image frames. Bins will automatically be created when needed but all the frames should be able to be packed into two bins active_index = -1 # Find a bin that we can fit the next image into for i, bin in enumerate(bins): @@ -541,6 +543,7 @@ class BatchCreativeInterpolationNode: if active_index == -1: bins.append(IPBin()) active_index = len(bins) - 1 + # Add the image to the bin bins[active_index].add(prepped_image, image.unsqueeze(0), negative_noise, big_negative_noise, image_index, ipa_frame_numbers, ipa_weights) @@ -552,6 +555,7 @@ class BatchCreativeInterpolationNode: all_ipa_frame_numbers.append(ipa_frame_numbers) all_ipa_weights.append(ipa_weights) + # Go through the bins and create IPAdapters for them for i, bin in enumerate(bins): ipadapter_application = IPAdapterBatchImport() negative_noise = torch.cat(bin.noiseBatch, dim=0) if len(bin.noiseBatch) > 0 else None From e741e8ee631dc44a8aee85a962e1bc896296758f Mon Sep 17 00:00:00 2001 From: IDGallagher Date: Thu, 25 Apr 2024 19:48:48 +0100 Subject: [PATCH 7/7] Remove print statements --- SteerableMotion.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/SteerableMotion.py b/SteerableMotion.py index 696b126..7fdac30 100644 --- a/SteerableMotion.py +++ b/SteerableMotion.py @@ -502,12 +502,12 @@ class BatchCreativeInterpolationNode: # PROCESS WEIGHTS ipa_frame_numbers, ipa_weights = process_weights(frame_numbers, weights, base_ipa_advanced_settings["ipa_weight"]) - print(f'i {i} image index {image_index} ====') - # print(f"frame numbers {frame_numbers}") - # print(f"weights {weights}") - print(f"frame numbers {ipa_frame_numbers}") - print(f"weights {ipa_weights}") - print("------") + # print(f'i {i} image index {image_index} ====') + # # print(f"frame numbers {frame_numbers}") + # # print(f"weights {weights}") + # print(f"frame numbers {ipa_frame_numbers}") + # print(f"weights {ipa_weights}") + # print("------") # Prepare images and noise prepare_for_clip_vision = PrepImageForClipVisionImport() @@ -548,10 +548,10 @@ class BatchCreativeInterpolationNode: # Add the image to the bin bins[active_index].add(prepped_image, image.unsqueeze(0), negative_noise, big_negative_noise, image_index, ipa_frame_numbers, ipa_weights) - for i, bin in enumerate(bins): - print(f"{i} schedule {bin.image_schedule}") - print(f"{i} weights {bin.weight_schedule}") - i += 1 + # for i, bin in enumerate(bins): + # print(f"{i} schedule {bin.image_schedule}") + # print(f"{i} weights {bin.weight_schedule}") + # i += 1 all_ipa_frame_numbers.append(ipa_frame_numbers) all_ipa_weights.append(ipa_weights)