diff --git a/SteerableMotion.py b/SteerableMotion.py index 4df7ab6..7fdac30 100644 --- a/SteerableMotion.py +++ b/SteerableMotion.py @@ -11,7 +11,6 @@ import matplotlib.pyplot as plt from .imports.ComfyUI_IPAdapter_plus.IPAdapterPlus import IPAdapterBatchImport, IPAdapterTiledBatchImport, IPAdapterTiledImport, PrepImageForClipVisionImport, IPAdapterAdvancedImport, IPAdapterNoiseImport from .imports.AdvancedControlNet.nodes_sparsectrl import SparseIndexMethodNodeImport - class BatchCreativeInterpolationNode: @classmethod def IS_CHANGED(cls, **kwargs): @@ -369,21 +368,55 @@ class BatchCreativeInterpolationNode: else: last_key_frame_position = (keyframe_positions[-1]) + class IPBin: + def __init__(self): + self.indicies = [] + self.image_schedule = [] + self.weight_schedule = [] + self.imageBatch = [] + self.bigImageBatch = [] + self.noiseBatch = [] + self.bigNoiseBatch = [] + + def length(self): + return len(self.image_schedule) + + def add(self, image, big_image, noise, big_noise, image_index, frame_numbers, weights): + # Map frames to their corresponding reversed weights for easy lookup + frame_to_weight = {frame: weights[i] for i, frame in enumerate(frame_numbers)} + # Search for image index, if it isn't there add the image + try: + index = self.indicies.index(image_index) + except ValueError: + self.imageBatch.append(image) + self.bigImageBatch.append(big_image) + if noise is not None: self.noiseBatch.append(noise) + if big_noise is not None: self.bigNoiseBatch.append(big_noise) + self.indicies.append(image_index) + index = self.indicies.index(image_index) + + self.image_schedule.extend([index] * (frame_numbers[-1] + 1 - len(self.image_schedule))) + self.weight_schedule.extend([0] * (frame_numbers[0] - len(self.weight_schedule))) + self.weight_schedule.extend(frame_to_weight[frame] for frame in range(frame_numbers[0], frame_numbers[-1] + 1)) + # CREATE LISTS FOR WEIGHTS AND FRAME NUMBERS all_cn_frame_numbers = [] all_cn_weights = [] all_ipa_weights = [] all_ipa_frame_numbers = [] + # Start with one bin + bins = [IPBin()] for i in range(len(keyframe_positions)): keyframe_position = keyframe_positions[i] interpolation = "ease-in-out" # strength_from = strength_to = 1.0 - + image_index = 0 if i == 0: # buffer image = images[0] + image_index = 0 strength_from = strength_to = strength_values[0][1] batch_index_from = 0 @@ -394,6 +427,7 @@ class BatchCreativeInterpolationNode: # GET IMAGE AND KEYFRAME INFLUENCE VALUES image = images[i-1] + image_index = i-1 key_frame_influence_from, key_frame_influence_to = key_frame_influence_values[i-1] start_strength, mid_strength, end_strength = strength_values[i-1] @@ -409,6 +443,7 @@ class BatchCreativeInterpolationNode: # GET IMAGE AND KEYFRAME INFLUENCE VALUES image = images[i-1] + image_index = i - 1 key_frame_influence_from,key_frame_influence_to = key_frame_influence_values[i-1] start_strength, mid_strength, end_strength = strength_values[i-1] if len(keyframe_positions) == 4: @@ -427,6 +462,7 @@ class BatchCreativeInterpolationNode: elif i == len(keyframe_positions) - 1: # buffer image = images[i-2] + image_index = i - 2 strength_from = strength_to = strength_values[i-2][1] if len(keyframe_positions) == 4: @@ -441,7 +477,8 @@ class BatchCreativeInterpolationNode: else: # middle images # GET IMAGE AND KEYFRAME INFLUENCE VALUES - image = images[i-1] + image = images[i-1] + image_index = i - 1 key_frame_influence_from,key_frame_influence_to = key_frame_influence_values[i-1] start_strength, mid_strength, end_strength = strength_values[i-1] keyframe_position = keyframe_positions[i] @@ -465,11 +502,17 @@ class BatchCreativeInterpolationNode: # PROCESS WEIGHTS ipa_frame_numbers, ipa_weights = process_weights(frame_numbers, weights, base_ipa_advanced_settings["ipa_weight"]) + # print(f'i {i} image index {image_index} ====') + # # print(f"frame numbers {frame_numbers}") + # # print(f"weights {weights}") + # print(f"frame numbers {ipa_frame_numbers}") + # print(f"weights {ipa_weights}") + # print("------") + + # Prepare images and noise prepare_for_clip_vision = PrepImageForClipVisionImport() prepped_image, = prepare_for_clip_vision.prep_image(image=image.unsqueeze(0), interpolation="LANCZOS", crop_position="pad", sharpening=0.1) - - weight_batch = create_weight_batch(last_key_frame_position, ipa_weights, ipa_frame_numbers) - + if base_ipa_advanced_settings["ipa_noise_strength"] > 0: if base_ipa_advanced_settings["use_image_for_noise"]: noise_image = prepped_image @@ -480,28 +523,50 @@ class BatchCreativeInterpolationNode: else: negative_noise = None - ipadapter_application = IPAdapterBatchImport() - model, = ipadapter_application.apply_ipadapter(model=model, ipadapter=ipadapter, image=prepped_image, weight=weight_batch, weight_type=base_ipa_advanced_settings["ipa_weight_type"], start_at=base_ipa_advanced_settings["ipa_starts_at"], end_at=base_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,image_negative=negative_noise,embeds_scaling=base_ipa_advanced_settings["ipa_embeds_scaling"]) - - if high_detail_mode: - if detail_ipa_advanced_settings["ipa_noise_strength"] > 0: - if detail_ipa_advanced_settings["use_image_for_noise"]: - noise_image = image.unsqueeze(0) - else: - noise_image = None - ipa_noise = IPAdapterNoiseImport() - negative_noise, = ipa_noise.make_noise(type=detail_ipa_advanced_settings["type_of_noise"], strength=detail_ipa_advanced_settings["ipa_noise_strength"], blur=detail_ipa_advanced_settings["noise_blur"], image_optional=noise_image) + if high_detail_mode and detail_ipa_advanced_settings["ipa_noise_strength"] > 0: + if detail_ipa_advanced_settings["use_image_for_noise"]: + noise_image = image.unsqueeze(0) else: - negative_noise = None - - tiled_ipa_application = IPAdapterTiledBatchImport() - model, *_ = tiled_ipa_application.apply_tiled(model=model, ipadapter=ipadapter, image=image.unsqueeze(0), weight=weight_batch, weight_type=detail_ipa_advanced_settings["ipa_weight_type"], start_at=detail_ipa_advanced_settings["ipa_starts_at"], end_at=detail_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,sharpening=0.1,image_negative=negative_noise,embeds_scaling=detail_ipa_advanced_settings["ipa_embeds_scaling"]) + noise_image = None + ipa_noise = IPAdapterNoiseImport() + big_negative_noise, = ipa_noise.make_noise(type=detail_ipa_advanced_settings["type_of_noise"], strength=detail_ipa_advanced_settings["ipa_noise_strength"], blur=detail_ipa_advanced_settings["noise_blur"], image_optional=noise_image) + else: + big_negative_noise = None + # Fill up bins with image frames. Bins will automatically be created when needed but all the frames should be able to be packed into two bins + active_index = -1 + # Find a bin that we can fit the next image into + for i, bin in enumerate(bins): + if bin.length() <= ipa_frame_numbers[0]: + active_index = i + break + # If we didn't find a suitable bin, add a new one + if active_index == -1: + bins.append(IPBin()) + active_index = len(bins) - 1 + + # Add the image to the bin + bins[active_index].add(prepped_image, image.unsqueeze(0), negative_noise, big_negative_noise, image_index, ipa_frame_numbers, ipa_weights) + + # for i, bin in enumerate(bins): + # print(f"{i} schedule {bin.image_schedule}") + # print(f"{i} weights {bin.weight_schedule}") + # i += 1 + all_ipa_frame_numbers.append(ipa_frame_numbers) all_ipa_weights.append(ipa_weights) - comparison_diagram, = plot_weight_comparison(all_cn_frame_numbers, all_cn_weights, all_ipa_frame_numbers, all_ipa_weights, buffer) + # Go through the bins and create IPAdapters for them + for i, bin in enumerate(bins): + ipadapter_application = IPAdapterBatchImport() + negative_noise = torch.cat(bin.noiseBatch, dim=0) if len(bin.noiseBatch) > 0 else None + model, = ipadapter_application.apply_ipadapter(model=model, ipadapter=ipadapter, image=torch.cat(bin.imageBatch, dim=0), weight=[x * base_ipa_advanced_settings["ipa_weight"] for x in bin.weight_schedule], weight_type=base_ipa_advanced_settings["ipa_weight_type"], start_at=base_ipa_advanced_settings["ipa_starts_at"], end_at=base_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,image_negative=negative_noise,embeds_scaling=base_ipa_advanced_settings["ipa_embeds_scaling"], image_schedule=bin.image_schedule) + if high_detail_mode: + tiled_ipa_application = IPAdapterTiledBatchImport() + negative_noise = torch.cat(bin.bigNoiseBatch, dim=0) if len(bin.bigNoiseBatch) > 0 else None + model, *_ = tiled_ipa_application.apply_tiled(model=model, ipadapter=ipadapter, image=torch.cat(bin.bigImageBatch, dim=0), weight=[x * detail_ipa_advanced_settings["ipa_weight"] for x in bin.weight_schedule], weight_type=detail_ipa_advanced_settings["ipa_weight_type"], start_at=detail_ipa_advanced_settings["ipa_starts_at"], end_at=detail_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,sharpening=0.1,image_negative=negative_noise,embeds_scaling=detail_ipa_advanced_settings["ipa_embeds_scaling"], image_schedule=bin.image_schedule) + comparison_diagram, = plot_weight_comparison(all_cn_frame_numbers, all_cn_weights, all_ipa_frame_numbers, all_ipa_weights, buffer) return comparison_diagram, positive, negative, model, sparse_indexes, last_key_frame_position, buffer class IpaConfigurationNode: diff --git a/imports/ComfyUI_IPAdapter_plus/CrossAttentionPatchImport.py b/imports/ComfyUI_IPAdapter_plus/CrossAttentionPatchImport.py index 78b3601..c615970 100644 --- a/imports/ComfyUI_IPAdapter_plus/CrossAttentionPatchImport.py +++ b/imports/ComfyUI_IPAdapter_plus/CrossAttentionPatchImport.py @@ -6,7 +6,7 @@ from .utils import tensor_to_size class CrossAttentionPatchImport: # forward for patching - def __init__(self, ipadapter=None, number=0, weight=1.0, cond=None, cond_alt=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, embeds_scaling='V only'): + def __init__(self, ipadapter=None, number=0, weight=1.0, cond=None, cond_alt=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, image_schedule=None, embeds_scaling='V only'): self.weights = [weight] self.ipadapters = [ipadapter] self.conds = [cond] @@ -17,6 +17,7 @@ class CrossAttentionPatchImport: self.sigma_starts = [sigma_start] self.sigma_ends = [sigma_end] self.unfold_batch = [unfold_batch] + self.image_schedule = [image_schedule] self.embeds_scaling = [embeds_scaling] self.number = number self.layers = 11 if '101_to_k_ip' in ipadapter.ip_layers.to_kvs else 16 # TODO: check if this is a valid condition to detect all models @@ -24,7 +25,7 @@ class CrossAttentionPatchImport: self.k_key = str(self.number*2+1) + "_to_k_ip" self.v_key = str(self.number*2+1) + "_to_v_ip" - def set_new_condition(self, ipadapter=None, number=0, weight=1.0, cond=None, cond_alt=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, embeds_scaling='V only'): + def set_new_condition(self, ipadapter=None, number=0, weight=1.0, cond=None, cond_alt=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, image_schedule=None, embeds_scaling='V only'): self.weights.append(weight) self.ipadapters.append(ipadapter) self.conds.append(cond) @@ -35,6 +36,7 @@ class CrossAttentionPatchImport: self.sigma_starts.append(sigma_start) self.sigma_ends.append(sigma_end) self.unfold_batch.append(unfold_batch) + self.image_schedule.append(image_schedule) self.embeds_scaling.append(embeds_scaling) def __call__(self, q, k, v, extra_options): @@ -54,7 +56,7 @@ class CrossAttentionPatchImport: out = optimized_attention(q, k, v, extra_options["n_heads"]) _, _, oh, ow = extra_options["original_shape"] - for weight, cond, cond_alt, uncond, ipadapter, mask, weight_type, sigma_start, sigma_end, unfold_batch, embeds_scaling in zip(self.weights, self.conds, self.conds_alt, self.unconds, self.ipadapters, self.masks, self.weight_types, self.sigma_starts, self.sigma_ends, self.unfold_batch, self.embeds_scaling): + for weight, cond, cond_alt, uncond, ipadapter, mask, weight_type, sigma_start, sigma_end, unfold_batch, image_schedule, embeds_scaling in zip(self.weights, self.conds, self.conds_alt, self.unconds, self.ipadapters, self.masks, self.weight_types, self.sigma_starts, self.sigma_ends, self.unfold_batch, self.image_schedule, self.embeds_scaling): if sigma <= sigma_start and sigma >= sigma_end: if weight_type == 'ease in': weight = weight * (0.05 + 0.95 * (1 - t_idx / self.layers)) @@ -94,16 +96,23 @@ class CrossAttentionPatchImport: elif weight == 0: continue - # if image length matches or exceeds full_length get sub_idx images - if cond.shape[0] >= ad_params["full_length"]: - cond = torch.Tensor(cond[ad_params["sub_idxs"]]) - uncond = torch.Tensor(uncond[ad_params["sub_idxs"]]) - # otherwise get sub_idxs images + if image_schedule is not None: + # Use the image_schedule as a lookup table to get the embedded image corresponding to each sub_idx + # If image_schedule isn't long enough then use the last image + cond_idxs = [image_schedule[i if i < len(image_schedule) else -1] for i in ad_params["sub_idxs"]] + cond = torch.Tensor(cond[cond_idxs]) + uncond = torch.Tensor(uncond[cond_idxs]) else: - cond = tensor_to_size(cond, ad_params["full_length"]) - uncond = tensor_to_size(uncond, ad_params["full_length"]) - cond = cond[ad_params["sub_idxs"]] - uncond = uncond[ad_params["sub_idxs"]] + # if image length matches or exceeds full_length get sub_idx images + if cond.shape[0] >= ad_params["full_length"]: + cond = torch.Tensor(cond[ad_params["sub_idxs"]]) + uncond = torch.Tensor(uncond[ad_params["sub_idxs"]]) + # otherwise get sub_idxs images + else: + cond = tensor_to_size(cond, ad_params["full_length"]) + uncond = tensor_to_size(uncond, ad_params["full_length"]) + cond = cond[ad_params["sub_idxs"]] + uncond = uncond[ad_params["sub_idxs"]] else: if isinstance(weight, torch.Tensor): weight = tensor_to_size(weight, batch_prompt) diff --git a/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py b/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py index 32b5012..a6c84b8 100644 --- a/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py +++ b/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py @@ -45,7 +45,7 @@ WEIGHT_TYPES = ["linear", "ease in", "ease out", 'ease in-out', 'reverse in-out' ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ """ class IPAdapterImport(nn.Module): - def __init__(self, ipadapter_model, cross_attention_dim=1024, output_cross_attention_dim=1024, clip_embeddings_dim=1024, clip_extra_context_tokens=4, is_sdxl=False, is_plus=False, is_full=False, is_faceid=False): + def __init__(self, ipadapter_model, cross_attention_dim=1024, output_cross_attention_dim=1024, clip_embeddings_dim=1024, clip_extra_context_tokens=4, is_sdxl=False, is_plus=False, is_full=False, is_faceid=False, is_portrait_unnorm=False): super().__init__() self.clip_embeddings_dim = clip_embeddings_dim @@ -55,12 +55,13 @@ class IPAdapterImport(nn.Module): self.is_sdxl = is_sdxl self.is_full = is_full self.is_plus = is_plus + self.is_portrait_unnorm = is_portrait_unnorm - if is_faceid: + if is_faceid and not is_portrait_unnorm: self.image_proj_model = self.init_proj_faceid() elif is_full: self.image_proj_model = self.init_proj_full() - elif is_plus: + elif is_plus or is_portrait_unnorm: self.image_proj_model = self.init_proj_plus() else: self.image_proj_model = self.init_proj() @@ -161,6 +162,7 @@ def ipadapter_execute(model, pos_embed=None, neg_embed=None, unfold_batch=False, + image_schedule=None, embeds_scaling='V only', layer_weights=None): device = model_management.get_torch_device() @@ -170,8 +172,9 @@ def ipadapter_execute(model, is_full = "proj.3.weight" in ipadapter["image_proj"] is_portrait = "proj.2.weight" in ipadapter["image_proj"] and not "proj.3.weight" in ipadapter["image_proj"] and not "0.to_q_lora.down.weight" in ipadapter["ip_adapter"] - is_faceid = is_portrait or "0.to_q_lora.down.weight" in ipadapter["ip_adapter"] - is_plus = is_full or "latents" in ipadapter["image_proj"] or "perceiver_resampler.proj_in.weight" in ipadapter["image_proj"] + is_portrait_unnorm = "portraitunnorm" in ipadapter + is_faceid = is_portrait or "0.to_q_lora.down.weight" in ipadapter["ip_adapter"] or is_portrait_unnorm + is_plus = (is_full or "latents" in ipadapter["image_proj"] or "perceiver_resampler.proj_in.weight" in ipadapter["image_proj"]) and not is_portrait_unnorm is_faceidv2 = "faceidplusv2" in ipadapter output_cross_attention_dim = ipadapter["ip_adapter"]["1.to_k_ip.weight"].shape[1] is_sdxl = output_cross_attention_dim == 2048 @@ -182,14 +185,14 @@ def ipadapter_execute(model, if is_faceidv2: weight_faceidv2 = weight_faceidv2 if weight_faceidv2 is not None else weight*2 - cross_attention_dim = 1280 if is_plus and is_sdxl and not is_faceid else output_cross_attention_dim - clip_extra_context_tokens = 16 if (is_plus and not is_faceid) or is_portrait else 4 + cross_attention_dim = 1280 if (is_plus and is_sdxl and not is_faceid) or is_portrait_unnorm else output_cross_attention_dim + clip_extra_context_tokens = 16 if (is_plus and not is_faceid) or is_portrait or is_portrait_unnorm else 4 if image is not None and image.shape[1] != image.shape[2]: print("\033[33mINFO: the IPAdapter reference image is not a square, CLIPImageProcessor will resize and crop it at the center. If the main focus of the picture is not in the middle the result might not be what you are expecting.\033[0m") if isinstance(weight, list): - weight = torch.tensor(weight).unsqueeze(-1).unsqueeze(-1).to(device, dtype=dtype) if unfold_batch else weight[0] + weight = torch.tensor(weight).unsqueeze(-1).unsqueeze(-1).to(device, dtype=dtype) if unfold_batch else weight[0] # special weight types if layer_weights is not None and layer_weights != '': @@ -233,7 +236,10 @@ def ipadapter_execute(model, insightface.det_model.input_size = size # TODO: hacky but seems to be working face = insightface.get(image_iface[i]) if face: - face_cond_embeds.append(torch.from_numpy(face[0].normed_embedding).unsqueeze(0)) + if not is_portrait_unnorm: + face_cond_embeds.append(torch.from_numpy(face[0].normed_embedding).unsqueeze(0)) + else: + face_cond_embeds.append(torch.from_numpy(face[0].embedding).unsqueeze(0)) image.append(image_to_tensor(face_align.norm_crop(image_iface[i], landmark=face[0].kps, image_size=256))) if 640 not in size: @@ -330,7 +336,8 @@ def ipadapter_execute(model, is_sdxl=is_sdxl, is_plus=is_plus, is_full=is_full, - is_faceid=is_faceid + is_faceid=is_faceid, + is_portrait_unnorm=is_portrait_unnorm, ).to(device, dtype=dtype) if is_faceid and is_plus: @@ -365,6 +372,7 @@ def ipadapter_execute(model, "sigma_start": sigma_start, "sigma_end": sigma_end, "unfold_batch": unfold_batch, + "image_schedule": image_schedule, "embeds_scaling": embeds_scaling, } @@ -492,7 +500,7 @@ class IPAdapterUnifiedLoaderFaceID(IPAdapterUnifiedLoader): def INPUT_TYPES(s): return {"required": { "model": ("MODEL", ), - "preset": (['FACEID', 'FACEID PLUS - SD1.5 only', 'FACEID PLUS V2', 'FACEID PORTRAIT (style transfer)'], ), + "preset": (['FACEID', 'FACEID PLUS - SD1.5 only', 'FACEID PLUS V2', 'FACEID PORTRAIT (style transfer)', 'FACEID PORTRAIT UNNORM - SDXL only (strong)'], ), "lora_strength": ("FLOAT", { "default": 0.6, "min": 0, "max": 1, "step": 0.01 }), "provider": (["CPU", "CUDA", "ROCM", "DirectML", "OpenVINO", "CoreML"], ), }, @@ -626,7 +634,7 @@ class IPAdapterAdvancedImport: FUNCTION = "apply_ipadapter" CATEGORY = "ipadapter" - def apply_ipadapter(self, model, ipadapter, start_at, end_at, weight = 1.0, weight_style=1.0, weight_composition=1.0, expand_style=False, weight_type="linear", combine_embeds="concat", weight_faceidv2=None, image=None, image_style=None, image_composition=None, image_negative=None, clip_vision=None, attn_mask=None, insightface=None, embeds_scaling='V only', layer_weights=None): + def apply_ipadapter(self, model, ipadapter, start_at, end_at, weight = 1.0, weight_style=1.0, weight_composition=1.0, expand_style=False, weight_type="linear", combine_embeds="concat", weight_faceidv2=None, image=None, image_style=None, image_composition=None, image_negative=None, clip_vision=None, image_schedule=None, attn_mask=None, insightface=None, embeds_scaling='V only', layer_weights=None): is_sdxl = isinstance(model.model, (comfy.model_base.SDXL, comfy.model_base.SDXLRefiner, comfy.model_base.SDXL_instructpix2pix)) if image_style is not None: # we are doing style + composition transfer @@ -653,6 +661,7 @@ class IPAdapterAdvancedImport: "end_at": end_at, "attn_mask": attn_mask, "unfold_batch": self.unfold_batch, + "image_schedule": image_schedule, "embeds_scaling": embeds_scaling, "insightface": insightface if insightface is not None else ipadapter['insightface']['model'] if 'insightface' in ipadapter else None, "layer_weights": layer_weights, @@ -693,6 +702,7 @@ class IPAdapterBatchImport(IPAdapterAdvancedImport): "image_negative": ("IMAGE",), "attn_mask": ("MASK",), "clip_vision": ("CLIP_VISION",), + "image_schedule": ("INT", {"default": None, "forceInput": True} ), } } @@ -745,6 +755,7 @@ class IPAdapterStyleCompositionBatch(IPAdapterStyleComposition): "image_negative": ("IMAGE",), "attn_mask": ("MASK",), "clip_vision": ("CLIP_VISION",), + "image_schedule": ("INT", {"default": None, "forceInput": True} ), } } @@ -809,7 +820,7 @@ class IPAdapterTiledImport: FUNCTION = "apply_tiled" CATEGORY = "ipadapter/tiled" - def apply_tiled(self, model, ipadapter, image, weight, weight_type, start_at, end_at, sharpening, combine_embeds="concat", image_negative=None, attn_mask=None, clip_vision=None, embeds_scaling='V only'): + def apply_tiled(self, model, ipadapter, image, weight, weight_type, start_at, end_at, sharpening, combine_embeds="concat", image_negative=None, attn_mask=None, clip_vision=None, embeds_scaling='V only', image_schedule=None): # 1. Select the models if 'ipadapter' in ipadapter: ipadapter_model = ipadapter['ipadapter']['model'] @@ -907,7 +918,9 @@ class IPAdapterTiledImport: "attn_mask": masks[i], "unfold_batch": self.unfold_batch, "embeds_scaling": embeds_scaling, + "image_schedule": image_schedule, } + # apply the ipadapter to the model without cloning it model = ipadapter_execute(model, ipadapter_model, clip_vision, **ipa_args) @@ -935,6 +948,7 @@ class IPAdapterTiledBatchImport(IPAdapterTiledImport): "image_negative": ("IMAGE",), "attn_mask": ("MASK",), "clip_vision": ("CLIP_VISION",), + "image_schedule": ("INT", {"default": None, "forceInput": True} ), } } @@ -1283,6 +1297,37 @@ class IPAdapterLoadEmbeds: path = folder_paths.get_annotated_filepath(embeds) return (torch.load(path).cpu(), ) +defaultValue="""0:0, +40:1, +80:2, +""" +class IPAdapterImageSchedule: + @classmethod + def INPUT_TYPES(s): + return {"required": {"text": ("STRING", {"multiline": True, "default": defaultValue}), + "max_frames": ("INT", {"default": 120.0, "min": 1.0, "max": 999999.0, "step": 1.0}), + "print_output": ("BOOLEAN", {"default": False})}} + + RETURN_TYPES = ("INT",) + FUNCTION = "schedule" + + CATEGORY = "ipadapter/utils" + + def schedule(self, text, max_frames, print_output): + frames = [0] * max_frames + for item in text.split(","): + item = item.strip() + if ":" in item: + parts = item.split(":") + if len(parts) == 2: + start_frame = int(parts[0]) + value = int(parts[1]) + for i in range(start_frame, max_frames): + frames[i] = value + if print_output is True: + print("ValueSchedule: ", frames) + return (frames, ) + class IPAdapterWeights: @classmethod def INPUT_TYPES(s): diff --git a/imports/ComfyUI_IPAdapter_plus/utils.py b/imports/ComfyUI_IPAdapter_plus/utils.py index e08b1d1..01f8ff5 100644 --- a/imports/ComfyUI_IPAdapter_plus/utils.py +++ b/imports/ComfyUI_IPAdapter_plus/utils.py @@ -61,7 +61,7 @@ def get_ipadapter_file(preset, is_sdxl): if is_sdxl: raise Exception("full face model is not supported for SDXL") pattern = 'full.face.sd15\.(safetensors|bin)$' - elif preset.startswith("faceid portrait"): + elif preset.startswith("faceid portrait ("): if is_sdxl: pattern = 'portrait.sdxl\.(safetensors|bin)$' else: @@ -70,6 +70,12 @@ def get_ipadapter_file(preset, is_sdxl): if not [e for e in ipadapter_list if re.search(pattern, e, re.IGNORECASE)]: pattern = 'portrait.sd15\.(safetensors|bin)$' is_insightface = True + elif preset.startswith("faceid portrait unnorm"): + if is_sdxl: + pattern = 'portrait.sdxl.unnorm\.(safetensors|bin)$' + else: + raise Exception("portrait unnorm model is not supported for SD1.5") + is_insightface = True elif preset == "faceid": if is_sdxl: pattern = 'faceid.sdxl\.(safetensors|bin)$' @@ -131,6 +137,9 @@ def ipadapter_model_loader(file): if 'plusv2' in file.lower(): model["faceidplusv2"] = True + + if 'unnorm' in file.lower(): + model["portraitunnorm"] = True return model @@ -145,21 +154,34 @@ def insightface_loader(provider): model.prepare(ctx_id=0, det_size=(640, 640)) return model -def encode_image_masked(clip_vision, image, mask=None): +def encode_image_masked(clip_vision, images, mask=None): model_management.load_model_gpu(clip_vision.patcher) - image = image.to(clip_vision.load_device) - pixel_values = clip_preprocess(image.to(clip_vision.load_device)).float() + # Initialize lists to collect outputs + last_hidden_states = [] + image_embeds = [] + penultimate_hidden_states = [] - if mask is not None: - pixel_values = pixel_values * mask.to(clip_vision.load_device) + # Loop over each image in the batch + for image in images: + pixel_values = clip_preprocess(image.to(clip_vision.load_device).unsqueeze(0)).float() - out = clip_vision.model(pixel_values=pixel_values, intermediate_output=-2) + if mask is not None: + pixel_values *= mask.to(clip_vision.load_device) + out = clip_vision.model(pixel_values=pixel_values, intermediate_output=-2) + + # Collect the outputs for each image + last_hidden_states.append(out[0].to(model_management.intermediate_device())) + image_embeds.append(out[2].to(model_management.intermediate_device())) + penultimate_hidden_states.append(out[1].to(model_management.intermediate_device())) + + # Concatenate all collected outputs across the batch outputs = Output() - outputs["last_hidden_state"] = out[0].to(model_management.intermediate_device()) - outputs["image_embeds"] = out[2].to(model_management.intermediate_device()) - outputs["penultimate_hidden_states"] = out[1].to(model_management.intermediate_device()) + outputs["last_hidden_state"] = torch.cat(last_hidden_states, dim=0) + outputs["image_embeds"] = torch.cat(image_embeds, dim=0) + outputs["penultimate_hidden_states"] = torch.cat(penultimate_hidden_states, dim=0) + return outputs def tensor_to_size(source, dest_size):