Batching working for non-detailed ipadapter
This commit is contained in:
IDGallagher
+77
-71
@@ -10,8 +10,6 @@ import matplotlib.pyplot as plt
|
||||
# Local application/library specific imports
|
||||
from .imports.ComfyUI_IPAdapter_plus.IPAdapterPlus import IPAdapterBatchImport, IPAdapterTiledBatchImport, IPAdapterTiledImport, PrepImageForClipVisionImport, IPAdapterAdvancedImport, IPAdapterNoiseImport
|
||||
from .imports.AdvancedControlNet.nodes_sparsectrl import SparseIndexMethodNodeImport
|
||||
import gc
|
||||
|
||||
|
||||
class BatchCreativeInterpolationNode:
|
||||
@classmethod
|
||||
@@ -369,11 +367,38 @@ class BatchCreativeInterpolationNode:
|
||||
else:
|
||||
last_key_frame_position = (keyframe_positions[-1])
|
||||
|
||||
class IPBin:
|
||||
def __init__(self):
|
||||
self.indicies = []
|
||||
self.image_schedule = []
|
||||
self.weight_schedule = []
|
||||
self.imageBatch = []
|
||||
|
||||
def length(self):
|
||||
return len(self.image_schedule)
|
||||
|
||||
def add(self, image, image_index, frame_numbers, weights, general_weight):
|
||||
# Map frames to their corresponding reversed weights for easy lookup
|
||||
frame_to_weight = {frame: weights[i] for i, frame in enumerate(frame_numbers)}
|
||||
# Search for image index, if it isn't there add the image
|
||||
try:
|
||||
index = self.indicies.index(image_index)
|
||||
except ValueError:
|
||||
self.imageBatch.append(image)
|
||||
self.indicies.append(image_index)
|
||||
index = self.indicies.index(image_index)
|
||||
|
||||
self.image_schedule.extend([index] * (frame_numbers[-1] + 1 - len(self.image_schedule)))
|
||||
self.weight_schedule.extend([0] * (frame_numbers[0] - len(self.weight_schedule)))
|
||||
self.weight_schedule.extend(general_weight * frame_to_weight[frame] for frame in range(frame_numbers[0], frame_numbers[-1] + 1))
|
||||
|
||||
# CREATE LISTS FOR WEIGHTS AND FRAME NUMBERS
|
||||
all_cn_frame_numbers = []
|
||||
all_cn_weights = []
|
||||
all_ipa_weights = []
|
||||
all_ipa_frame_numbers = []
|
||||
# Start with one bin
|
||||
bins = [IPBin()]
|
||||
|
||||
for i in range(len(keyframe_positions)):
|
||||
|
||||
@@ -477,45 +502,26 @@ class BatchCreativeInterpolationNode:
|
||||
print(f"weights {ipa_weights}")
|
||||
print("------")
|
||||
|
||||
# class IPBin:
|
||||
|
||||
# def __init__(self):
|
||||
# self.indicies = []
|
||||
# self.image_schedule = []
|
||||
# self.weight_schedule = []
|
||||
# self.imageBatch = []
|
||||
prepare_for_clip_vision = PrepImageForClipVisionImport()
|
||||
prepped_image, = prepare_for_clip_vision.prep_image(image=image.unsqueeze(0), interpolation="LANCZOS", crop_position="pad", sharpening=0.1)
|
||||
|
||||
# def length(self):
|
||||
# return len(self.indicies)
|
||||
|
||||
# def add(self, image, image_index, frame_numbers, weights):
|
||||
# # Search for image index, if it isn't there add the image
|
||||
# try:
|
||||
# index = self.indicies.index(image_index)
|
||||
# except ValueError:
|
||||
# self.imageBatch.append(image)
|
||||
# self.indicies.append(image_index)
|
||||
# index = len(self.indicies) - 1
|
||||
# self.image_schedule.extend([index] * (frame_numbers[-1] - len(self.image_schedule)))
|
||||
# self.weight_schedule.extend([0] * (frame_numbers[0] - len(self.weight_schedule)))
|
||||
# self.weight_schedule.extend(weights)
|
||||
|
||||
|
||||
# # Start with one bin
|
||||
# bins = [IPBin()]
|
||||
# active_index = -1
|
||||
# # Find a bin that we can fit the next image into
|
||||
# for i, bin in bins:
|
||||
# if bin.length() <= ipa_frame_numbers[0]:
|
||||
# active_index = i
|
||||
# break1
|
||||
# # If we didn't find a suitable bin, add a new one
|
||||
# if active_index == -1:
|
||||
# bins.append(IPBin())
|
||||
# active_index = len(bins) - 1
|
||||
|
||||
# bins[active_index].add(image, image_index, ipa_frame_numbers, ipa_weights)
|
||||
active_index = -1
|
||||
# Find a bin that we can fit the next image into
|
||||
for i, bin in enumerate(bins):
|
||||
if bin.length() <= ipa_frame_numbers[0]:
|
||||
active_index = i
|
||||
break
|
||||
# If we didn't find a suitable bin, add a new one
|
||||
if active_index == -1:
|
||||
bins.append(IPBin())
|
||||
active_index = len(bins) - 1
|
||||
# Add the image to the bin
|
||||
bins[active_index].add(prepped_image, image_index, ipa_frame_numbers, ipa_weights, base_ipa_advanced_settings["ipa_weight"])
|
||||
|
||||
for i, bin in enumerate(bins):
|
||||
print(f"{i} schedule {bin.image_schedule}")
|
||||
print(f"{i} weights {bin.weight_schedule}")
|
||||
i += 1
|
||||
# active_image_schedule = image_schedule_1
|
||||
# # Choose active image_schedule
|
||||
# if len(image_schedule_1) == 0 or image_index == image_indicies_1[-1]:
|
||||
@@ -531,43 +537,43 @@ class BatchCreativeInterpolationNode:
|
||||
# # Fill the gaps
|
||||
|
||||
# # Add image and weights
|
||||
|
||||
# weight_batch = create_weight_batch(last_key_frame_position, ipa_weights, ipa_frame_numbers)
|
||||
# print(f"weight batch {weight_batch}")
|
||||
|
||||
|
||||
prepare_for_clip_vision = PrepImageForClipVisionImport()
|
||||
prepped_image, = prepare_for_clip_vision.prep_image(image=image.unsqueeze(0), interpolation="LANCZOS", crop_position="pad", sharpening=0.1)
|
||||
|
||||
weight_batch = create_weight_batch(last_key_frame_position, ipa_weights, ipa_frame_numbers)
|
||||
|
||||
if base_ipa_advanced_settings["ipa_noise_strength"] > 0:
|
||||
if base_ipa_advanced_settings["use_image_for_noise"]:
|
||||
noise_image = prepped_image
|
||||
else:
|
||||
noise_image = None
|
||||
ipa_noise = IPAdapterNoiseImport()
|
||||
negative_noise, = ipa_noise.make_noise(type=base_ipa_advanced_settings["type_of_noise"], strength=base_ipa_advanced_settings["ipa_noise_strength"], blur=base_ipa_advanced_settings["noise_blur"], image_optional=noise_image)
|
||||
else:
|
||||
negative_noise = None
|
||||
|
||||
ipadapter_application = IPAdapterBatchImport()
|
||||
model, = ipadapter_application.apply_ipadapter(model=model, ipadapter=ipadapter, image=prepped_image, weight=weight_batch*base_ipa_advanced_settings["ipa_weight"], weight_type=base_ipa_advanced_settings["ipa_weight_type"], start_at=base_ipa_advanced_settings["ipa_starts_at"], end_at=base_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,image_negative=negative_noise,embeds_scaling=base_ipa_advanced_settings["ipa_embeds_scaling"])
|
||||
|
||||
if high_detail_mode:
|
||||
if detail_ipa_advanced_settings["ipa_noise_strength"] > 0:
|
||||
if detail_ipa_advanced_settings["use_image_for_noise"]:
|
||||
noise_image = image.unsqueeze(0)
|
||||
else:
|
||||
noise_image = None
|
||||
ipa_noise = IPAdapterNoiseImport()
|
||||
negative_noise, = ipa_noise.make_noise(type=detail_ipa_advanced_settings["type_of_noise"], strength=detail_ipa_advanced_settings["ipa_noise_strength"], blur=detail_ipa_advanced_settings["noise_blur"], image_optional=noise_image)
|
||||
else:
|
||||
negative_noise = None
|
||||
|
||||
tiled_ipa_application = IPAdapterTiledBatchImport()
|
||||
model, *_ = tiled_ipa_application.apply_tiled(model=model, ipadapter=ipadapter, image=image.unsqueeze(0), weight=weight_batch*base_ipa_advanced_settings["ipa_weight"], weight_type=detail_ipa_advanced_settings["ipa_weight_type"], start_at=detail_ipa_advanced_settings["ipa_starts_at"], end_at=detail_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,sharpening=0.1,image_negative=negative_noise,embeds_scaling=detail_ipa_advanced_settings["ipa_embeds_scaling"])
|
||||
|
||||
|
||||
all_ipa_frame_numbers.append(ipa_frame_numbers)
|
||||
all_ipa_weights.append(ipa_weights)
|
||||
|
||||
# if base_ipa_advanced_settings["ipa_noise_strength"] > 0:
|
||||
# if base_ipa_advanced_settings["use_image_for_noise"]:
|
||||
# noise_image = prepped_image
|
||||
# else:
|
||||
# noise_image = None
|
||||
# ipa_noise = IPAdapterNoiseImport()
|
||||
# negative_noise, = ipa_noise.make_noise(type=base_ipa_advanced_settings["type_of_noise"], strength=base_ipa_advanced_settings["ipa_noise_strength"], blur=base_ipa_advanced_settings["noise_blur"], image_optional=noise_image)
|
||||
# else:
|
||||
# negative_noise = None
|
||||
negative_noise = None
|
||||
for i, bin in enumerate(bins):
|
||||
ipadapter_application = IPAdapterBatchImport()
|
||||
model, = ipadapter_application.apply_ipadapter(model=model, ipadapter=ipadapter, image=torch.cat(bin.imageBatch, dim=0), weight=bin.weight_schedule, weight_type=base_ipa_advanced_settings["ipa_weight_type"], start_at=base_ipa_advanced_settings["ipa_starts_at"], end_at=base_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,image_negative=negative_noise,embeds_scaling=base_ipa_advanced_settings["ipa_embeds_scaling"], image_schedule=bin.image_schedule)
|
||||
|
||||
# if high_detail_mode:
|
||||
# if detail_ipa_advanced_settings["ipa_noise_strength"] > 0:
|
||||
# if detail_ipa_advanced_settings["use_image_for_noise"]:
|
||||
# noise_image = image.unsqueeze(0)
|
||||
# else:
|
||||
# noise_image = None
|
||||
# ipa_noise = IPAdapterNoiseImport()
|
||||
# negative_noise, = ipa_noise.make_noise(type=detail_ipa_advanced_settings["type_of_noise"], strength=detail_ipa_advanced_settings["ipa_noise_strength"], blur=detail_ipa_advanced_settings["noise_blur"], image_optional=noise_image)
|
||||
# else:
|
||||
# negative_noise = None
|
||||
|
||||
# tiled_ipa_application = IPAdapterTiledBatchImport()
|
||||
# model, *_ = tiled_ipa_application.apply_tiled(model=model, ipadapter=ipadapter, image=image.unsqueeze(0), weight=weight_batch*base_ipa_advanced_settings["ipa_weight"], weight_type=detail_ipa_advanced_settings["ipa_weight_type"], start_at=detail_ipa_advanced_settings["ipa_starts_at"], end_at=detail_ipa_advanced_settings["ipa_ends_at"], clip_vision=clip_vision,sharpening=0.1,image_negative=negative_noise,embeds_scaling=detail_ipa_advanced_settings["ipa_embeds_scaling"])
|
||||
|
||||
comparison_diagram, = plot_weight_comparison(all_cn_frame_numbers, all_cn_weights, all_ipa_frame_numbers, all_ipa_weights, buffer)
|
||||
|
||||
return comparison_diagram, positive, negative, model, sparse_indexes, last_key_frame_position, buffer
|
||||
|
||||
@@ -191,12 +191,8 @@ def ipadapter_execute(model,
|
||||
if image is not None and image.shape[1] != image.shape[2]:
|
||||
print("\033[33mINFO: the IPAdapter reference image is not a square, CLIPImageProcessor will resize and crop it at the center. If the main focus of the picture is not in the middle the result might not be what you are expecting.\033[0m")
|
||||
|
||||
weight = [1, 2, 6, 7, 3, 4]
|
||||
if isinstance(weight, list) and isinstance(weight[0], list):
|
||||
weight = [torch.tensor(w).unsqueeze(-1).unsqueeze(-1).to(device, dtype=dtype) for w in weight]
|
||||
elif isinstance(weight, list):
|
||||
weight = torch.tensor(weight).unsqueeze(-1).unsqueeze(-1).to(device, dtype=dtype) if unfold_batch else weight[0]
|
||||
print(weight)
|
||||
if isinstance(weight, list):
|
||||
weight = torch.tensor(weight).unsqueeze(-1).unsqueeze(-1).to(device, dtype=dtype) if unfold_batch else weight[0]
|
||||
|
||||
# special weight types
|
||||
if layer_weights is not None and layer_weights != '':
|
||||
|
||||
Reference in New Issue
Block a user