From ffd8d740648555abc29c6fdf4df9b0729e2fe9c3 Mon Sep 17 00:00:00 2001 From: IDGallagher Date: Wed, 24 Apr 2024 09:46:59 +0100 Subject: [PATCH] Batching working for non-detailed ipadapter --- SteerableMotion.py | 148 +++++++++--------- .../ComfyUI_IPAdapter_plus/IPAdapterPlus.py | 8 +- 2 files changed, 79 insertions(+), 77 deletions(-) diff --git a/SteerableMotion.py b/SteerableMotion.py index c63ebc0..d3cc740 100644 --- a/SteerableMotion.py +++ b/SteerableMotion.py @@ -10,8 +10,6 @@ import matplotlib.pyplot as plt # Local application/library specific imports from .imports.ComfyUI_IPAdapter_plus.IPAdapterPlus import IPAdapterBatchImport, IPAdapterTiledBatchImport, IPAdapterTiledImport, PrepImageForClipVisionImport, IPAdapterAdvancedImport, IPAdapterNoiseImport from .imports.AdvancedControlNet.nodes_sparsectrl import SparseIndexMethodNodeImport -import gc - class BatchCreativeInterpolationNode: @classmethod @@ -369,11 +367,38 @@ class BatchCreativeInterpolationNode: else: last_key_frame_position = (keyframe_positions[-1]) + class IPBin: + def __init__(self): + self.indicies = [] + self.image_schedule = [] + self.weight_schedule = [] + self.imageBatch = [] + + def length(self): + return len(self.image_schedule) + + def add(self, image, image_index, frame_numbers, weights, general_weight): + # Map frames to their corresponding reversed weights for easy lookup + frame_to_weight = {frame: weights[i] for i, frame in enumerate(frame_numbers)} + # Search for image index, if it isn't there add the image + try: + index = self.indicies.index(image_index) + except ValueError: + self.imageBatch.append(image) + self.indicies.append(image_index) + index = self.indicies.index(image_index) + + self.image_schedule.extend([index] * (frame_numbers[-1] + 1 - len(self.image_schedule))) + self.weight_schedule.extend([0] * (frame_numbers[0] - len(self.weight_schedule))) + self.weight_schedule.extend(general_weight * frame_to_weight[frame] for frame in range(frame_numbers[0], frame_numbers[-1] + 1)) + # CREATE LISTS FOR WEIGHTS AND FRAME NUMBERS all_cn_frame_numbers = [] all_cn_weights = [] all_ipa_weights = [] all_ipa_frame_numbers = [] + # Start with one bin + bins = [IPBin()] for i in range(len(keyframe_positions)): @@ -477,45 +502,26 @@ class BatchCreativeInterpolationNode: print(f"weights {ipa_weights}") print("------") - # class IPBin: - - # def __init__(self): - # self.indicies = [] - # self.image_schedule = [] - # self.weight_schedule = [] - # self.imageBatch = [] + prepare_for_clip_vision = PrepImageForClipVisionImport() + prepped_image, = prepare_for_clip_vision.prep_image(image=image.unsqueeze(0), interpolation="LANCZOS", crop_position="pad", sharpening=0.1) - # def length(self): - # return len(self.indicies) - - # def add(self, image, image_index, frame_numbers, weights): - # # Search for image index, if it isn't there add the image - # try: - # index = self.indicies.index(image_index) - # except ValueError: - # self.imageBatch.append(image) - # self.indicies.append(image_index) - # index = len(self.indicies) - 1 - # self.image_schedule.extend([index] * (frame_numbers[-1] - len(self.image_schedule))) - # self.weight_schedule.extend([0] * (frame_numbers[0] - len(self.weight_schedule))) - # self.weight_schedule.extend(weights) - - - # # Start with one bin - # bins = [IPBin()] - # active_index = -1 - # # Find a bin that we can fit the next image into - # for i, bin in bins: - # if bin.length() <= ipa_frame_numbers[0]: - # active_index = i - # break1 - # # If we didn't find a suitable bin, add a new one - # if active_index == -1: - # bins.append(IPBin()) - # active_index = len(bins) - 1 - - # bins[active_index].add(image, image_index, ipa_frame_numbers, ipa_weights) + active_index = -1 + # Find a bin that we can fit the next image into + for i, bin in enumerate(bins): + if bin.length() <= ipa_frame_numbers[0]: + active_index = i + break + # If we didn't find a suitable bin, add a new one + if active_index == -1: + bins.append(IPBin()) + active_index = len(bins) - 1 + # Add the image to the bin + bins[active_index].add(prepped_image, image_index, ipa_frame_numbers, ipa_weights, base_ipa_advanced_settings["ipa_weight"]) + for i, bin in enumerate(bins): + print(f"{i} schedule {bin.image_schedule}") + print(f"{i} weights {bin.weight_schedule}") + i += 1 # active_image_schedule = image_schedule_1 # # Choose active image_schedule # if len(image_schedule_1) == 0 or image_index == image_indicies_1[-1]: @@ -531,43 +537,43 @@ class BatchCreativeInterpolationNode: # # Fill the gaps # # Add image and weights + + # weight_batch = create_weight_batch(last_key_frame_position, ipa_weights, ipa_frame_numbers) + # print(f"weight batch {weight_batch}") - - prepare_for_clip_vision = PrepImageForClipVisionImport() - prepped_image, = prepare_for_clip_vision.prep_image(image=image.unsqueeze(0), interpolation="LANCZOS", crop_position="pad", sharpening=0.1) - - weight_batch = create_weight_batch(last_key_frame_position, ipa_weights, ipa_frame_numbers) - - if base_ipa_advanced_settings["ipa_noise_strength"] > 0: - if base_ipa_advanced_settings["use_image_for_noise"]: - noise_image = prepped_image - else: - noise_image = None - ipa_noise = IPAdapterNoiseImport() - negative_noise, = ipa_noise.make_noise(type=base_ipa_advanced_settings["type_of_noise"], strength=base_ipa_advanced_settings["ipa_noise_strength"], blur=base_ipa_advanced_settings["noise_blur"], image_optional=noise_image) - else: - negative_noise = None - - ipadapter_application = IPAdapterBatchImport() - model, = ipadapter_application.apply_ipadapter(model=model, ipadapter=ipadapter, image=prepped_image, weight=weight_batch*base_ipa_advanced_settings["ipa_weight"], weight_type=base_ipa_advanced_settings["ipa_weight_type"], start_at=base_ipa_advanced_settings["ipa_starts_at"], end_at=base_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,image_negative=negative_noise,embeds_scaling=base_ipa_advanced_settings["ipa_embeds_scaling"]) - - if high_detail_mode: - if detail_ipa_advanced_settings["ipa_noise_strength"] > 0: - if detail_ipa_advanced_settings["use_image_for_noise"]: - noise_image = image.unsqueeze(0) - else: - noise_image = None - ipa_noise = IPAdapterNoiseImport() - negative_noise, = ipa_noise.make_noise(type=detail_ipa_advanced_settings["type_of_noise"], strength=detail_ipa_advanced_settings["ipa_noise_strength"], blur=detail_ipa_advanced_settings["noise_blur"], image_optional=noise_image) - else: - negative_noise = None - - tiled_ipa_application = IPAdapterTiledBatchImport() - model, *_ = tiled_ipa_application.apply_tiled(model=model, ipadapter=ipadapter, image=image.unsqueeze(0), weight=weight_batch*base_ipa_advanced_settings["ipa_weight"], weight_type=detail_ipa_advanced_settings["ipa_weight_type"], start_at=detail_ipa_advanced_settings["ipa_starts_at"], end_at=detail_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,sharpening=0.1,image_negative=negative_noise,embeds_scaling=detail_ipa_advanced_settings["ipa_embeds_scaling"]) + all_ipa_frame_numbers.append(ipa_frame_numbers) all_ipa_weights.append(ipa_weights) + # if base_ipa_advanced_settings["ipa_noise_strength"] > 0: + # if base_ipa_advanced_settings["use_image_for_noise"]: + # noise_image = prepped_image + # else: + # noise_image = None + # ipa_noise = IPAdapterNoiseImport() + # negative_noise, = ipa_noise.make_noise(type=base_ipa_advanced_settings["type_of_noise"], strength=base_ipa_advanced_settings["ipa_noise_strength"], blur=base_ipa_advanced_settings["noise_blur"], image_optional=noise_image) + # else: + # negative_noise = None + negative_noise = None + for i, bin in enumerate(bins): + ipadapter_application = IPAdapterBatchImport() + model, = ipadapter_application.apply_ipadapter(model=model, ipadapter=ipadapter, image=torch.cat(bin.imageBatch, dim=0), weight=bin.weight_schedule, weight_type=base_ipa_advanced_settings["ipa_weight_type"], start_at=base_ipa_advanced_settings["ipa_starts_at"], end_at=base_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,image_negative=negative_noise,embeds_scaling=base_ipa_advanced_settings["ipa_embeds_scaling"], image_schedule=bin.image_schedule) + + # if high_detail_mode: + # if detail_ipa_advanced_settings["ipa_noise_strength"] > 0: + # if detail_ipa_advanced_settings["use_image_for_noise"]: + # noise_image = image.unsqueeze(0) + # else: + # noise_image = None + # ipa_noise = IPAdapterNoiseImport() + # negative_noise, = ipa_noise.make_noise(type=detail_ipa_advanced_settings["type_of_noise"], strength=detail_ipa_advanced_settings["ipa_noise_strength"], blur=detail_ipa_advanced_settings["noise_blur"], image_optional=noise_image) + # else: + # negative_noise = None + + # tiled_ipa_application = IPAdapterTiledBatchImport() + # model, *_ = tiled_ipa_application.apply_tiled(model=model, ipadapter=ipadapter, image=image.unsqueeze(0), weight=weight_batch*base_ipa_advanced_settings["ipa_weight"], weight_type=detail_ipa_advanced_settings["ipa_weight_type"], start_at=detail_ipa_advanced_settings["ipa_starts_at"], end_at=detail_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,sharpening=0.1,image_negative=negative_noise,embeds_scaling=detail_ipa_advanced_settings["ipa_embeds_scaling"]) + comparison_diagram, = plot_weight_comparison(all_cn_frame_numbers, all_cn_weights, all_ipa_frame_numbers, all_ipa_weights, buffer) return comparison_diagram, positive, negative, model, sparse_indexes, last_key_frame_position, buffer diff --git a/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py b/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py index d228b4d..2ca998b 100644 --- a/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py +++ b/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py @@ -191,12 +191,8 @@ def ipadapter_execute(model, if image is not None and image.shape[1] != image.shape[2]: print("\033[33mINFO: the IPAdapter reference image is not a square, CLIPImageProcessor will resize and crop it at the center. If the main focus of the picture is not in the middle the result might not be what you are expecting.\033[0m") - weight = [1, 2, 6, 7, 3, 4] - if isinstance(weight, list) and isinstance(weight[0], list): - weight = [torch.tensor(w).unsqueeze(-1).unsqueeze(-1).to(device, dtype=dtype) for w in weight] - elif isinstance(weight, list): - weight = torch.tensor(weight).unsqueeze(-1).unsqueeze(-1).to(device, dtype=dtype) if unfold_batch else weight[0] - print(weight) + if isinstance(weight, list): + weight = torch.tensor(weight).unsqueeze(-1).unsqueeze(-1).to(device, dtype=dtype) if unfold_batch else weight[0] # special weight types if layer_weights is not None and layer_weights != '':