diff --git a/SteerableMotion.py b/SteerableMotion.py index dacaefd..bda24c3 100644 --- a/SteerableMotion.py +++ b/SteerableMotion.py @@ -16,8 +16,12 @@ from .imports.AdvancedControlNet.latent_keyframe_nodes import ( LatentKeyframeInterpolationNodeImport ) from .imports.AdvancedControlNet.weight_nodes import ScaledSoftUniversalWeightsImport +from .imports.AdvancedControlNet.nodes_sparsectrl import SparseIndexMethodNodeImport +from .imports.AdvancedControlNet.control_sparsectrl import SparseIndexMethodImport from .imports.AdvancedControlNet.nodes import ControlNetLoaderAdvancedImport, AdvancedControlNetApplyImport,TimestepKeyframeNodeImport +from abc import ABC, abstractmethod + class BatchCreativeInterpolationNode: @classmethod def IS_CHANGED(cls, **kwargs): @@ -27,44 +31,37 @@ class BatchCreativeInterpolationNode: def INPUT_TYPES(s): return { "required": { - "positive": ("CONDITIONING", ), - "negative": ("CONDITIONING", ), "images": ("IMAGE", ), "model": ("MODEL", ), "ipadapter": ("IPADAPTER", ), "clip_vision": ("CLIP_VISION",), - "control_net_name": (folder_paths.get_filename_list("controlnet"), ), "type_of_frame_distribution": (["linear", "dynamic"],), - "linear_frame_distribution_value": ("INT", {"default": 16, "min": 4, "max": 64, "step": 1}), - "dynamic_frame_distribution_values": ("STRING", {"multiline": True, "default": "0,10,26,40"}), + "linear_frame_distribution_value": ("INT", {"default": 16, "min": 4, "max": 64, "step": 1}), + "dynamic_frame_distribution_values": ("STRING", {"multiline": True, "default": "0,10,26,40"}), "type_of_key_frame_influence": (["linear", "dynamic"],), "linear_key_frame_influence_value": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.1}), - "dynamic_key_frame_influence_values": ("STRING", {"multiline": True, "default": "1.0,1.0,1.0,0.5"}), + "dynamic_key_frame_influence_values": ("STRING", {"multiline": True, "default": "1.0,1.0,1.0,0.5"}), "type_of_cn_strength_distribution": (["linear", "dynamic"],), - "linear_cn_strength_value": ("STRING", {"multiline": False, "default": "(0.0,0.4)"}), + "linear_cn_strength_value": ("STRING", {"multiline": False, "default": "(0.3,0.4)"}), "dynamic_cn_strength_values": ("STRING", {"multiline": True, "default": "(0.0,1.0),(0.0,1.0),(0.0,1.0),(0.0,1.0)"}), - "soft_scaled_cn_weights_multiplier": ("FLOAT", {"default": 0.85, "min": 0.0, "max": 10.0, "step": 0.1}), - "buffer": ("INT", {"default": 4, "min": 0, "max": 16, "step": 1}), - "relative_ipadapter_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 5.0, "step": 0.1}), - "relative_ipadapter_influence": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 5.0, "step": 0.1}), + "buffer": ("INT", {"default": 4, "min": 0, "max": 16, "step": 1}), "ipadapter_noise": ("FLOAT", {"default": 0.3, "min": 0.0, "max": 1.0, "step": 0.01}), }, "optional": { } } - RETURN_TYPES = ("IMAGE","CONDITIONING","CONDITIONING","MODEL",) - RETURN_NAMES = ("GRAPH","POSITIVE", "NEGATIVE","MODEL") + RETURN_TYPES = ("IMAGE","MODEL","SPARSE_METHOD","INT") + RETURN_NAMES = ("GRAPH","MODEL","KEYFRAME_POSITIONS", "BATCH_SIZE") FUNCTION = "combined_function" CATEGORY = "Steerable-Motion/Interpolation" - def combined_function(self, positive, negative, images,model,ipadapter,clip_vision,control_net_name, - type_of_frame_distribution,linear_frame_distribution_value,dynamic_frame_distribution_values, - type_of_key_frame_influence,linear_key_frame_influence_value,dynamic_key_frame_influence_values, - type_of_cn_strength_distribution,linear_cn_strength_value,dynamic_cn_strength_values, - soft_scaled_cn_weights_multiplier,buffer,relative_ipadapter_strength, - relative_ipadapter_influence,ipadapter_noise): + def combined_function(self,images,model,ipadapter,clip_vision,type_of_frame_distribution, + linear_frame_distribution_value, dynamic_frame_distribution_values, + type_of_key_frame_influence,linear_key_frame_influence_value, + dynamic_key_frame_influence_values,type_of_cn_strength_distribution, + linear_cn_strength_value,dynamic_cn_strength_values,buffer,ipadapter_noise): def calculate_dynamic_influence_ranges(keyframe_positions, key_frame_influence_values, allow_extension=True): if len(keyframe_positions) < 2 or len(keyframe_positions) != len(key_frame_influence_values): @@ -74,14 +71,22 @@ class BatchCreativeInterpolationNode: for i, position in enumerate(keyframe_positions): influence_factor = key_frame_influence_values[i] - # Calculate the base range size - range_size = influence_factor * (keyframe_positions[-1] - keyframe_positions[0]) / (len(keyframe_positions) - 1) / 2 + if i == 0: + # Special handling for the first keyframe (half the distance to the second keyframe) + range_size = influence_factor * (keyframe_positions[1] - keyframe_positions[0]) / 2 + start_influence = position # Start from the first keyframe position + end_influence = position + range_size + elif i == len(keyframe_positions) - 1: + # Special handling for the last keyframe (half the distance from the penultimate keyframe) + range_size = influence_factor * (keyframe_positions[-1] - keyframe_positions[-2]) / 2 + start_influence = position - range_size + end_influence = position # End at the last keyframe position + else: + # Regular calculation for other keyframes + range_size = influence_factor * (keyframe_positions[-1] - keyframe_positions[0]) / (len(keyframe_positions) - 1) / 2 + start_influence = position - range_size + end_influence = position + range_size - # Calculate symmetric start and end influence - start_influence = position - range_size - end_influence = position + range_size - - # Adjust start and end influence to not exceed previous and next keyframes if not allow_extension: start_influence = max(start_influence, keyframe_positions[i - 1] if i > 0 else 0) end_influence = min(end_influence, keyframe_positions[i + 1] if i < len(keyframe_positions) - 1 else keyframe_positions[-1]) @@ -89,7 +94,8 @@ class BatchCreativeInterpolationNode: influence_ranges.append((round(start_influence), round(end_influence))) return influence_ranges - + + def add_starting_buffer(influence_ranges, buffer=4): shifted_ranges = [(0, buffer)] for start, end in influence_ranges: @@ -126,21 +132,6 @@ class BatchCreativeInterpolationNode: # Create a list with the linear_key_frame_influence_value for each keyframe return [linear_key_frame_influence_value for _ in keyframe_positions] - def extract_start_and_endpoint_values(type_of_key_frame_influence, dynamic_key_frame_influence_values, keyframe_positions, linear_key_frame_influence_value): - if type_of_key_frame_influence == "dynamic": - # If dynamic_key_frame_influence_values is a list of characters representing tuples, process it - if isinstance(dynamic_key_frame_influence_values[0], str) and dynamic_key_frame_influence_values[0] == "(": - # Join the characters to form a single string and evaluate to convert into a list of tuples - string_representation = ''.join(dynamic_key_frame_influence_values) - dynamic_values = eval(f'[{string_representation}]') - else: - # If it's already a list of tuples or a single tuple, use it directly - dynamic_values = dynamic_key_frame_influence_values if isinstance(dynamic_key_frame_influence_values, list) else [dynamic_key_frame_influence_values] - return dynamic_values - else: - # Return a list of tuples with the linear_key_frame_influence_value as a tuple repeated for each position - return [linear_key_frame_influence_value for _ in keyframe_positions] - def create_mask_batch(last_key_frame_position, weights, frames): # Hardcoded dimensions width, height = 512, 512 @@ -162,68 +153,17 @@ class BatchCreativeInterpolationNode: masks_tensor = torch.stack(masks, dim=0) return masks_tensor - - def adjust_influence_range(batch_index_from, batch_index_to_excl, last_key_frame_position, scale_factor, buffer): - # Calculate the midpoint of the current range - midpoint = (batch_index_from + batch_index_to_excl) // 2 - - # Calculate the new range length - new_range_length = int((batch_index_to_excl - batch_index_from) * scale_factor) - - # Adjusting both sides of the range - if batch_index_from == 0: - # Start is anchored at 0 - new_batch_index_from = 0 - new_batch_index_to_excl = batch_index_from + new_range_length - elif batch_index_to_excl == last_key_frame_position: - # End is anchored at last_key_frame_position - new_batch_index_from = batch_index_to_excl - new_range_length - new_batch_index_to_excl = last_key_frame_position - else: - # No anchoring, adjust both sides around the midpoint - new_batch_index_from = midpoint - new_range_length // 2 - new_batch_index_to_excl = midpoint + new_range_length // 2 - - # Remove minimum and maximum constraints - - return new_batch_index_from, new_batch_index_to_excl - - def adjust_strength_values(strength_from, strength_to, multiplier): - mid_point = (strength_from + strength_to) / 2 - range_half = abs(strength_to - strength_from) / 2 - - # Adjust the range with the multiplier - new_range_half = min(range_half * multiplier, 0.5) - - # Calculate new strength values, ensuring they stay within [0.0, 1.0] - new_strength_from = max(mid_point - new_range_half, 0.0) - new_strength_to = min(mid_point + new_range_half, 1.0) - - # Preserve the order of the original strength values - if strength_from > strength_to: - new_strength_from, new_strength_to = new_strength_to, new_strength_from - - return (new_strength_from, new_strength_to) - - def plot_weight_comparison(cn_frame_numbers, cn_weights, ipadapter_frame_numbers, ipadapter_weights, buffer): + + def plot_weight_comparison(ipadapter_frame_numbers, ipadapter_weights, buffer): plt.figure(figsize=(12, 8)) # Defining colors for each set of data colors = ['b', 'g', 'r', 'c', 'm', 'y', 'k'] - # Alternating the data sets with labels and colors - max_length = max(len(cn_frame_numbers), len(ipadapter_frame_numbers)) + # Plotting data for ipadapter + max_length = len(ipadapter_frame_numbers) label_counter = 1 if buffer < 0 else 0 # Start from 1 if buffer < 0, else start from 0 for i in range(max_length): - # Label for cn_strength - if i < len(cn_frame_numbers): - if i == 0 and buffer > 0: - label = 'cn_strength_buffer' - else: - label = f'cn_strength_{label_counter}' - plt.plot(cn_frame_numbers[i], cn_weights[i], marker='o', color=colors[i % len(colors)], label=label) - - # Label for ipa_strength if i < len(ipadapter_frame_numbers): if i == 0 and buffer > 0: label = 'ipa_strength_buffer' @@ -235,7 +175,7 @@ class BatchCreativeInterpolationNode: label_counter += 1 plt.legend() - max_weight = max([weight.max() for weight in cn_weights + ipadapter_weights]) * 1.5 + max_weight = max([weight.max() for weight in ipadapter_weights]) * 1.5 plt.ylim(0, max_weight) buffer_io = BytesIO() @@ -246,33 +186,49 @@ class BatchCreativeInterpolationNode: img = Image.open(buffer_io) img_tensor = TT.ToTensor()(img) - + img_tensor = img_tensor.unsqueeze(0) img_tensor = img_tensor.permute([0, 2, 3, 1]) return (img_tensor,) - - keyframe_positions = get_keyframe_positions(type_of_frame_distribution, dynamic_frame_distribution_values, images, linear_frame_distribution_value) - cn_strength_values = extract_start_and_endpoint_values(type_of_cn_strength_distribution, dynamic_cn_strength_values, keyframe_positions, linear_cn_strength_value) - key_frame_influence_values = extract_keyframe_values(type_of_key_frame_influence, dynamic_key_frame_influence_values, keyframe_positions, linear_key_frame_influence_value) - influence_ranges = calculate_dynamic_influence_ranges(keyframe_positions,key_frame_influence_values) - influence_ranges = add_starting_buffer(influence_ranges, buffer) - cn_strength_values = [literal_eval(val) if isinstance(val, str) else val for val in cn_strength_values] - cn_frame_numbers, cn_weights, ipadapter_frame_numbers, ipadapter_weights = [], [], [], [] - last_key_frame_position = (keyframe_positions[-1]) + buffer - - embeds = [] - masks = [] - existing_embeds = [] + def extract_start_and_endpoint_values(type_of_key_frame_influence, dynamic_key_frame_influence_values, keyframe_positions, linear_key_frame_influence_value): + if type_of_key_frame_influence == "dynamic": + # If dynamic_key_frame_influence_values is a list of characters representing tuples, process it + if isinstance(dynamic_key_frame_influence_values[0], str) and dynamic_key_frame_influence_values[0] == "(": + # Join the characters to form a single string and evaluate to convert into a list of tuples + string_representation = ''.join(dynamic_key_frame_influence_values) + dynamic_values = eval(f'[{string_representation}]') + else: + # If it's already a list of tuples or a single tuple, use it directly + dynamic_values = dynamic_key_frame_influence_values if isinstance(dynamic_key_frame_influence_values, list) else [dynamic_key_frame_influence_values] + return dynamic_values + else: + # Return a list of tuples with the linear_key_frame_influence_value as a tuple repeated for each position + return [linear_key_frame_influence_value for _ in keyframe_positions] + + + keyframe_positions = get_keyframe_positions(type_of_frame_distribution, dynamic_frame_distribution_values, images, linear_frame_distribution_value) + cn_strength_values = extract_start_and_endpoint_values(type_of_cn_strength_distribution, dynamic_cn_strength_values, keyframe_positions, linear_cn_strength_value) + cn_strength_values = [literal_eval(val) if isinstance(val, str) else val for val in cn_strength_values] - for i, (start, end) in enumerate(influence_ranges): - # set basic values - batch_index_from, batch_index_to_excl = influence_ranges[i] - ipadapter_strength_multiplier = relative_ipadapter_strength - ipadapter_influence_multiplier = relative_ipadapter_influence + keyframe_positions_string = ','.join(str(pos) for pos in keyframe_positions) + + sparseindexmethod = SparseIndexMethodNodeImport() + sparse_indexes, = sparseindexmethod.get_method(keyframe_positions_string) + + key_frame_influence_values = extract_keyframe_values(type_of_key_frame_influence, dynamic_key_frame_influence_values, keyframe_positions, linear_key_frame_influence_value) + influence_ranges = calculate_dynamic_influence_ranges(keyframe_positions, key_frame_influence_values) + influence_ranges = add_starting_buffer(influence_ranges, buffer) + last_key_frame_position = (keyframe_positions[-1]) + buffer + + all_frame_numbers = [] + all_weights = [] + + for i, (batch_index_from, batch_index_to_excl) in enumerate(influence_ranges): + # Default values revert_direction_at_midpoint = False interpolation = "ease-in-out" @@ -281,86 +237,44 @@ class BatchCreativeInterpolationNode: if i == 0: if buffer > 0: # First image with buffer image = images[0] - strength_from = strength_to = cn_strength_values[0][1] if len(cn_strength_values) > 0 else (1.0, 1.0) - ipadapter_influence_multiplier = 1.0 + strength_from = strength_to = cn_strength_values[0][1] if len(cn_strength_values) > 0 else (1.0, 1.0) interpolation = "ease-in-out" else: continue # Skip first image without buffer elif i == 1: # First image image = images[0] strength_to, strength_from = cn_strength_values[0] if len(cn_strength_values) > 0 else (0.0, 1.0) - interpolation = "ease-in" + # interpolation = "ease-in" elif i == len(images): # Last image image = images[i-1] strength_from, strength_to = cn_strength_values[i-1] if i-1 < len(cn_strength_values) else (0.0, 1.0) - interpolation = "ease-out" + # interpolation = "ease-out" else: # Middle images image = images[i-1] strength_from, strength_to = cn_strength_values[i-1] if i-1 < len(cn_strength_values) else (0.0, 1.0) revert_direction_at_midpoint = True - # Import necessary modules - latent_keyframe_interpolation_node = LatentKeyframeInterpolationNodeImport() - scaled_soft_control_net_weights = ScaledSoftUniversalWeightsImport() - timestep_keyframe_node = TimestepKeyframeNodeImport() - control_net_loader = ControlNetLoaderAdvancedImport() - apply_advanced_control_net = AdvancedControlNetApplyImport() ipadapter_application = IPAdapterApplyImport() ipadapter_encoder = IPAdapterEncoderImport() - # ipadapter_batcher = IPAdapterBatchEmbedsImport() - - # Load keyframe and append frame numbers and weights - weights, frame_numbers, latent_keyframe = latent_keyframe_interpolation_node.load_keyframe( - batch_index_from, strength_from, batch_index_to_excl, strength_to, interpolation, revert_direction_at_midpoint, last_key_frame_position, i, len(influence_ranges), buffer) - cn_frame_numbers.append(frame_numbers) - cn_weights.append(weights) - - # Load weights and keyframe - control_net_weights, _ = scaled_soft_control_net_weights.load_weights(soft_scaled_cn_weights_multiplier, False) - timestep_keyframe = timestep_keyframe_node.load_keyframe(start_percent=0.0, control_net_weights=control_net_weights, latent_keyframe=latent_keyframe, prev_timestep_keyframe=None)[0] - - # Load and apply control net - control_net = control_net_loader.load_controlnet(control_net_name, timestep_keyframe)[0] - positive, negative = apply_advanced_control_net.apply_controlnet(positive, negative, control_net, image.unsqueeze(0), 1.0, 0.0, 1.0) - - # Prepare image - prepped_image = prep_image(image=image.unsqueeze(0), interpolation="LANCZOS", crop_position="pad", sharpening=0.0)[0] - - # Adjust strength values and influence range - ipa_strength_from, ipa_strength_to = adjust_strength_values(strength_from, strength_to, ipadapter_strength_multiplier) - ipa_batch_index_from, ipa_batch_index_to_excl = adjust_influence_range(batch_index_from, batch_index_to_excl, last_key_frame_position, ipadapter_influence_multiplier, buffer) - - # Calculate weights and append frame numbers and weights - ipa_weights, ipa_frame_numbers = calculate_weights(ipa_batch_index_from, ipa_batch_index_to_excl, ipa_strength_from, ipa_strength_to, interpolation, revert_direction_at_midpoint, last_key_frame_position, i, len(influence_ranges), buffer) - ipadapter_frame_numbers.append(ipa_frame_numbers) - ipadapter_weights.append(ipa_weights) - - - mask = create_mask_batch(last_key_frame_position, ipa_weights, frame_numbers) - # add mask to masks list - masks.append(mask) + prepped_image = prep_image(image=image.unsqueeze(0), interpolation="LANCZOS", crop_position="pad", sharpening=0.0)[0] + + weights, frame_numbers = calculate_weights(batch_index_from, batch_index_to_excl, strength_from, strength_to, interpolation, revert_direction_at_midpoint, last_key_frame_position, i, len(influence_ranges), buffer) + + mask = create_mask_batch(last_key_frame_position, weights, frame_numbers) + + # add mask to masks list embed, = ipadapter_encoder.preprocess(clip_vision, prepped_image, True, 0.0, 1.0) # add embeds to current batch - embeds.append(embed) - model, = ipadapter_application.apply_ipadapter(ipadapter=ipadapter, model=model, weight=1.0, image=None, weight_type="original", noise=ipadapter_noise, embeds=embed, attn_mask=mask, start_at=0.0, end_at=1.0, unfold_batch=True) - - # print out the format for the embeds - - # merged_embeds = torch.cat(embeds, dim=1) + all_frame_numbers.append(frame_numbers) + all_weights.append(weights) + + weights_diagram, = plot_weight_comparison(all_frame_numbers, all_weights, buffer) - # stacked_masks = torch.stack(masks) - - # merged_masks = torch.cat(masks, dim=1) - - - - comparison_diagram, = plot_weight_comparison(cn_frame_numbers, cn_weights, ipadapter_frame_numbers, ipadapter_weights, buffer) - - return comparison_diagram, positive, negative, model + return weights_diagram, model,sparse_indexes, last_key_frame_position # NODE MAPPING diff --git a/demo/creative_interpolation_example.json b/demo/creative_interpolation_example.json index 28c2e0f..367351a 100644 --- a/demo/creative_interpolation_example.json +++ b/demo/creative_interpolation_example.json @@ -1,52 +1,319 @@ { - "last_node_id": 456, - "last_link_id": 860, + "last_node_id": 472, + "last_link_id": 895, "nodes": [ { - "id": 436, - "type": "PreviewImage", + "id": 401, + "type": "VHS_LoadImagesPath", "pos": [ - 458.4848111817443, - -591.2252734560072 + -1431.1200500262726, + -1538.0194254767262 + ], + "size": [ + 315, + 194 ], - "size": { - "0": 820.0277099609375, - "1": 511.5301818847656 - }, "flags": {}, - "order": 20, + "order": 0, "mode": 0, - "inputs": [ + "outputs": [ { - "name": "images", + "name": "IMAGE", "type": "IMAGE", - "link": 841 + "links": [ + 737, + 861, + 881 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + }, + { + "name": "INT", + "type": "INT", + "links": [], + "shape": 3, + "slot_index": 2 } ], "properties": { - "Node name for S&R": "PreviewImage" - } + "Node name for S&R": "VHS_LoadImagesPath" + }, + "widgets_values": { + "directory": "input/input/", + "image_load_cap": 0, + "skip_first_images": 0, + "select_every_nth": 1, + "choose folder to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_images": 0, + "filename": "input/input/", + "type": "path", + "format": "folder", + "select_every_nth": 1 + } + } + }, + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 455, + "type": "Note Plus (mtb)", + "pos": { + "0": -835.8253173828125, + "1": -1130.6640625, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 454.0640869140625, + "1": 388.03802490234375 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "## Understanding the settings\n\n
\nThere are 3 main settings:\n\n
\n\n- Key frame position\n- Length of influence\n- Strength of influence\n\n
\n\nThe type_of fields decide whether each of settings are the same for each frame or different.\n\n
\n\nWhen the type has been set to 'linear', you can adjust the value for every in with the _value fields.\n\n
\n\nWhen the type is set to 'dynamic', the text box below each setting are for the individual values - follow the format provided to set values for each frame.\n", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, + { + "id": 468, + "type": "ACN_AdvancedControlNetApply", + "pos": [ + -282.54006698250515, + -1871.1889539350007 + ], + "size": { + "0": 314.1339111328125, + "1": 246 + }, + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 885, + "slot_index": 0 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 886, + "slot_index": 1 + }, + { + "name": "control_net", + "type": "CONTROL_NET", + "link": 879 + }, + { + "name": "image", + "type": "IMAGE", + "link": 880 + }, + { + "name": "mask_optional", + "type": "MASK", + "link": null + }, + { + "name": "timestep_kf", + "type": "TIMESTEP_KEYFRAME", + "link": null + }, + { + "name": "latent_kf_override", + "type": "LATENT_KEYFRAME", + "link": null + }, + { + "name": "weights_override", + "type": "CONTROL_NET_WEIGHTS", + "link": null + } + ], + "outputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 889 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "negative", + "type": "CONDITIONING", + "links": [ + 890 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "ACN_AdvancedControlNetApply" + }, + "widgets_values": [ + 0.8, + 0, + 0.7000000000000001 + ] + }, + { + "id": 450, + "type": "Note Plus (mtb)", + "pos": { + "0": -850, + "1": -709, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": [ + 492.59745980047865, + 149.39620638116412 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "## Setting frame position ⮕\n\n
\n\nSetting linear frame distribution makes the frames spaced out by the linear value - for example, if it's set to 16, the frames will be at positions 0, 16, 32, etc. If you set type to dynamic, you'll need to enter the values in the text box below.", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, + { + "id": 451, + "type": "Note Plus (mtb)", + "pos": { + "0": -849, + "1": -540, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": [ + 497.6794418497627, + 117.36078188815816 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "## Set duration of frame influence ⮕\n
\n1.0 equals around the distance between this and the next frames, 2.0 equals twice this. You can set these values for each frame individually in the field below.", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, + { + "id": 452, + "type": "Note Plus (mtb)", + "pos": { + "0": -852, + "1": -395, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": [ + 502.86638402601, + 132.24178721194846 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "## Set high and low point of influence strength ⮕\n
\n\nThe number are the range the strength runs from and to - the first value is the low point, the last is the high.", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 }, { "id": 370, "type": "CLIPVisionLoader", "pos": [ - -278.51518881825587, - -598.2252734560072 + -332, + -1014 ], "size": { "0": 210, "1": 91.58576202392578 }, "flags": {}, - "order": 0, + "order": 5, "mode": 0, "outputs": [ { "name": "CLIP_VISION", "type": "CLIP_VISION", "links": [ - 826 + 863 ], "shape": 3, "slot_index": 0 @@ -59,19 +326,768 @@ "SD1.5/pytorch_model.bin" ] }, + { + "id": 369, + "type": "IPAdapterModelLoader", + "pos": [ + -75, + -1010 + ], + "size": { + "0": 210, + "1": 92.0848388671875 + }, + "flags": {}, + "order": 6, + "mode": 0, + "outputs": [ + { + "name": "IPADAPTER", + "type": "IPADAPTER", + "links": [ + 864 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterModelLoader" + }, + "widgets_values": [ + "ip-adapter-plus_sd15.bin" + ] + }, + { + "id": 469, + "type": "ACN_SparseCtrlRGBPreprocessor", + "pos": [ + -738.9073483674799, + -1911.8438099709022 + ], + "size": { + "0": 312.9380187988281, + "1": 66 + }, + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 881 + }, + { + "name": "vae", + "type": "VAE", + "link": 882 + }, + { + "name": "latent_size", + "type": "LATENT", + "link": 883 + } + ], + "outputs": [ + { + "name": "proc_IMAGE", + "type": "IMAGE", + "links": [ + 880 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ACN_SparseCtrlRGBPreprocessor" + } + }, + { + "id": 466, + "type": "PrimitiveNode", + "pos": [ + 893.0571671339583, + -1671.1880809802162 + ], + "size": { + "0": 210, + "1": 82 + }, + "flags": {}, + "order": 7, + "mode": 0, + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 875 + ], + "widget": { + "name": "height" + }, + "slot_index": 0 + } + ], + "title": "Height", + "properties": { + "Run widget replace on values": false + }, + "widgets_values": [ + 512, + "fixed" + ] + }, + { + "id": 462, + "type": "CLIPTextEncode", + "pos": [ + 363.05716713395856, + -1644.1880809802162 + ], + "size": [ + 435.7196270870246, + 105.19292433739406 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 894 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 463, + "type": "CLIPTextEncode", + "pos": [ + 362.05716713395856, + -1453.1880809802162 + ], + "size": [ + 438.03718528028674, + 112.69035875077475 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 895 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 885, + 886 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "hand, embedding:EasyNegative" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 458, + "type": "VAELoader", + "pos": [ + 888.0571671339583, + -1472.1880809802162 + ], + "size": [ + 280.3557676400221, + 95.60184877164215 + ], + "flags": {}, + "order": 8, + "mode": 0, + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 867, + 882 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAELoader" + }, + "widgets_values": [ + "vae-ft-mse-840000-ema-pruned.safetensors" + ] + }, + { + "id": 467, + "type": "ACN_SparseCtrlLoaderAdvanced", + "pos": [ + -759.9073483674799, + -1757.8438099709022 + ], + "size": { + "0": 314.50390625, + "1": 150 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "sparse_method", + "type": "SPARSE_METHOD", + "link": null, + "slot_index": 0 + }, + { + "name": "tk_optional", + "type": "TIMESTEP_KEYFRAME", + "link": null + } + ], + "outputs": [ + { + "name": "CONTROL_NET", + "type": "CONTROL_NET", + "links": [ + 879 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ACN_SparseCtrlLoaderAdvanced" + }, + "widgets_values": [ + "v3_sd15_sparsectrl_rgb.ckpt", + true, + 1, + 1 + ] + }, + { + "id": 470, + "type": "LoraLoader", + "pos": [ + -305, + -1546 + ], + "size": { + "0": 256.37469482421875, + "1": 126.03836059570312 + }, + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 891 + }, + { + "name": "clip", + "type": "CLIP", + "link": 893 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 892 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 894, + 895 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LoraLoader" + }, + "widgets_values": [ + "v3_sd15_adapter.ckpt", + 1, + 1 + ] + }, + { + "id": 472, + "type": "Note Plus (mtb)", + "pos": { + "0": -723, + "1": -1564, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": [ + 292.25772635512976, + 176.61475031810005 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "## ⬆ Download these models from Hugging Face\n
\nGo [here](https://huggingface.co/guoyww/animatediff/tree/main) and download the models with the same name into your models/controlnet folder.", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, + { + "id": 457, + "type": "BatchCreativeInterpolation", + "pos": [ + -309, + -793 + ], + "size": { + "0": 400, + "1": 452 + }, + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 861 + }, + { + "name": "model", + "type": "MODEL", + "link": 870 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 864 + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 863 + } + ], + "outputs": [ + { + "name": "GRAPH", + "type": "IMAGE", + "links": [ + 865 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 891 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "KEYFRAME_POSITIONS", + "type": "SPARSE_METHOD", + "links": [], + "shape": 3, + "slot_index": 2 + }, + { + "name": "BATCH_SIZE", + "type": "INT", + "links": [ + 873 + ], + "shape": 3, + "slot_index": 3 + } + ], + "properties": { + "Node name for S&R": "BatchCreativeInterpolation" + }, + "widgets_values": [ + "linear", + 24, + "0,24,40,56", + "linear", + 1.4000000000000001, + "1.15,1.2,1.2,1.0", + "linear", + "(0.3,0.7)", + "0.3", + 4, + 0.3 + ] + }, + { + "id": 464, + "type": "ADE_EmptyLatentImageLarge", + "pos": [ + -182, + -229 + ], + "size": { + "0": 310.79998779296875, + "1": 74 + }, + "flags": { + "collapsed": false + }, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "width", + "type": "INT", + "link": 874, + "widget": { + "name": "width" + }, + "slot_index": 0 + }, + { + "name": "height", + "type": "INT", + "link": 875, + "widget": { + "name": "height" + } + }, + { + "name": "batch_size", + "type": "INT", + "link": 873, + "widget": { + "name": "batch_size" + }, + "slot_index": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 876, + 883 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ADE_EmptyLatentImageLarge" + }, + "widgets_values": [ + 512, + 512, + 52 + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 389, + "type": "PreviewImage", + "pos": [ + 271, + -384 + ], + "size": { + "0": 969.0651245117188, + "1": 246 + }, + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 737 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 454, + "type": "Note Plus (mtb)", + "pos": { + "0": 153, + "1": -735, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": [ + 231.76398170739776, + 258.793973300541 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "## Observe the graph to understand the the motion ⮕\n
\n As you tweak the settings, observe the impact on the graph. You can disconnect the positive, negative and model connections to just observe the graph without creating videos.", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, + { + "id": 456, + "type": "Note Plus (mtb)", + "pos": { + "0": 174, + "1": -1051, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 266.1289367675781 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "## ⬅ Download the models from Comfy Manager\n
\nSearch the names and download the relevant models that are associated with the IPAdapter_plus node.", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, + { + "id": 445, + "type": "Note Plus (mtb)", + "pos": { + "0": -1575, + "1": -1140, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 645.7035522460938, + "1": 1039.391357421875 + }, + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "# A brief guide\n\n
\n\n## Philosophy for getting the most from this:\n\n
\n\nThis isn’t a tool like text to video that will perform well out of the box - it’s more like a paint brush, an artistic tool that you need to figure out how to get the best from. \n\n
\n\nThrough trial and error, you'll need to build an understanding of how the motion and settings work, what its limitations are, which inputs work best with it, etc.\n\n
\n\nIf you can figure out how to wield it, this approach can provide enough control for you to make beautiful things that match your imagination precisely.\n\n
\n\n## How to use this:\n
\n1.Select a folder with the images in them above. The most important thing that will influence the quality of your creation is the input images.\n\n
\n\n2.Try the basic settings to start. Based on the resulting generation, try to understand what happened - you can learn about the settings and observe the graphs to the right.\n\n
\n\n3.Tweak the settings until you achieve your desired effect. \n\n
\n\n## What this will & won't be suitable for:\n\nI believe that this will be good for a wide range of storytelling-driven and abstract motion.\n\nHowever, it probably won't be good for complex motion - other approaches like vid2vid are far more suitable for this.\n\n## If you're having problems:\n\n
\n\nFirst, download the latest version of this workflow from [here](https://raw.githubusercontent.com/peteromallet/steerable-motion/main/demo/creative_interpolation_example.json) and get the latest version of the node from Comfy Manager.\n\n
\n\nIf that doesn't work, drop into [our Discord](https://discord.gg/V72vk8T67n) and share your problem and I'll try to help ASAP.\n\n
\n\n\n## Want to join a community of people who are pushing open source models to their technical and artistic limits?\n\n
\n\nYou're welcome to join [our Discord](https://discord.gg/UJf6aum7WZ).\n\n
\n\nPlease share anything you generate in the steerable-motion channel.", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, + { + "id": 446, + "type": "Note Plus (mtb)", + "pos": { + "0": 2272.6748046875, + "1": -1806.122802734375, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": [ + 210, + 271.6663084559218 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "## ⬅ Tweak motion_scale to change amount of movement\n
\nYou can tweak motion_scale on the left to increase or decrease the amount of motion in the generation.", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, + { + "id": 281, + "type": "VHS_VideoCombine", + "pos": [ + 1504.8259476658664, + -812.8196982341881 + ], + "size": [ + 807.1564331054688, + 966.1549482345581 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 656 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 14, + "loop_count": 0, + "filename_prefix": "steerable-motion/AD_", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 20, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AD__00088.mp4", + "subfolder": "steerable-motion", + "type": "output", + "format": "video/h264-mp4" + } + } + } + }, { "id": 354, "type": "VHS_SplitImages", "pos": [ - 2672.245315377938, - -953.1334206496155 + 1621.9066556707467, + -958.6076030790747 ], "size": { "0": 231.97152709960938, "1": 152.94261169433594 }, - "flags": {}, - "order": 23, + "flags": { + "collapsed": true + }, + "order": 29, "mode": 0, "inputs": [ { @@ -122,15 +1138,17 @@ "id": 292, "type": "STMFNet VFI", "pos": [ - 2609.245315377938, - -645.1334206496156 + 2025.9066556707476, + -957.6076030790747 ], "size": { - "0": 343.60247802734375, - "1": 150 + "0": 295.6000061035156, + "1": 174 }, - "flags": {}, - "order": 24, + "flags": { + "collapsed": true + }, + "order": 30, "mode": 0, "inputs": [ { @@ -162,24 +1180,155 @@ "stmfnet.pth", 15, 2, + true, true ] }, + { + "id": 436, + "type": "PreviewImage", + "pos": [ + 416, + -989 + ], + "size": { + "0": 820.0277099609375, + "1": 511.5301818847656 + }, + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 865 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 342, + "type": "ADE_AnimateDiffUniformContextOptions", + "pos": [ + 1998, + -1357 + ], + "size": [ + 417.6433386151225, + 189.57285879987285 + ], + "flags": {}, + "order": 15, + "mode": 0, + "outputs": [ + { + "name": "CONTEXT_OPTIONS", + "type": "CONTEXT_OPTIONS", + "links": [ + 627 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ADE_AnimateDiffUniformContextOptions" + }, + "widgets_values": [ + 16, + 2, + 2, + "uniform", + false + ] + }, + { + "id": 187, + "type": "ADE_AnimateDiffLoaderWithContext", + "pos": [ + 1974, + -1781 + ], + "size": [ + 261.3230001098409, + 320.7744965769075 + ], + "flags": { + "pinned": false + }, + "order": 23, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 892, + "slot_index": 0 + }, + { + "name": "context_options", + "type": "CONTEXT_OPTIONS", + "link": 627, + "slot_index": 1 + }, + { + "name": "motion_lora", + "type": "MOTION_LORA", + "link": null + }, + { + "name": "motion_model_settings", + "type": "MOTION_MODEL_SETTINGS", + "link": null, + "slot_index": 3 + }, + { + "name": "sample_settings", + "type": "sample_settings", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 815 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ADE_AnimateDiffLoaderWithContext" + }, + "widgets_values": [ + "v3_sd15_mm.ckpt", + "sqrt_linear (AnimateDiff)", + 1, + false + ], + "color": "#232", + "bgcolor": "#353" + }, { "id": 207, "type": "KSampler Adv. (Efficient)", "pos": [ - 1603, - -1280 + 1480, + -1824 ], "size": { - "0": 440, - "1": 634 + "0": 420.55169677734375, + "1": 693.31201171875 }, "flags": { "pinned": false }, - "order": 22, + "order": 28, "mode": 0, "inputs": [ { @@ -190,22 +1339,22 @@ { "name": "positive", "type": "CONDITIONING", - "link": 858 + "link": 889 }, { "name": "negative", "type": "CONDITIONING", - "link": 859 + "link": 890 }, { "name": "latent_image", "type": "LATENT", - "link": 633 + "link": 876 }, { "name": "optional_vae", "type": "VAE", - "link": 632 + "link": 867 }, { "name": "script", @@ -283,959 +1432,86 @@ "shape": 1 }, { - "id": 187, - "type": "ADE_AnimateDiffLoaderWithContext", + "id": 461, + "type": "CheckpointLoaderSimple", "pos": [ - 1649, - -522 - ], - "size": { - "0": 382.3793029785156, - "1": 263.21368408203125 - }, - "flags": { - "pinned": false - }, - "order": 21, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 860, - "slot_index": 0 - }, - { - "name": "context_options", - "type": "CONTEXT_OPTIONS", - "link": 627, - "slot_index": 1 - }, - { - "name": "motion_lora", - "type": "MOTION_LORA", - "link": null - }, - { - "name": "motion_model_settings", - "type": "MOTION_MODEL_SETTINGS", - "link": null, - "slot_index": 3 - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 815 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "ADE_AnimateDiffLoaderWithContext" - }, - "widgets_values": [ - "mm_sd_v15_v2.ckpt", - "sqrt_linear (AnimateDiff)", - 1, - false - ], - "color": "#232", - "bgcolor": "#353" - }, - { - "id": 446, - "type": "Note Plus (mtb)", - "pos": { - "0": 2092, - "1": -488, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 268.31817626953125, - "1": 202 - }, - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "## ⬅ Tweak motion_scale to change amount of movement\n
\nYou can tweak motion_scale on the left to increase or decrease the amount of motion in the generation.", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, - { - "id": 342, - "type": "ADE_AnimateDiffUniformContextOptions", - "pos": [ - 2128, - -1128 - ], - "size": { - "0": 252, - "1": 418.359619140625 - }, - "flags": {}, - "order": 2, - "mode": 0, - "outputs": [ - { - "name": "CONTEXT_OPTIONS", - "type": "CONTEXT_OPTIONS", - "links": [ - 627 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "ADE_AnimateDiffUniformContextOptions" - }, - "widgets_values": [ - 16, - 2, - 2, - "uniform", - false - ] - }, - { - "id": 369, - "type": "IPAdapterModelLoader", - "pos": [ - -7.515188818256069, - -593.2252734560072 - ], - "size": { - "0": 210, - "1": 92.0848388671875 - }, - "flags": {}, - "order": 3, - "mode": 0, - "outputs": [ - { - "name": "IPADAPTER", - "type": "IPADAPTER", - "links": [ - 825 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "IPAdapterModelLoader" - }, - "widgets_values": [ - "ip-adapter-plus_sd15.bin" - ] - }, - { - "id": 455, - "type": "Note Plus (mtb)", - "pos": { - "0": -801.515380859375, - "1": -708.2255249023438, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 454.0640869140625, - "1": 388.03802490234375 - }, - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "## Understanding the settings\n\n
\nThere are 3 main settings:\n\n
\n\n- Key frame position\n- Length of influence\n- Strength of influence\n\n
\n\nThe type_of fields decide whether each of settings are the same for each frame or different.\n\n
\n\nWhen the type has been set to 'linear', you can adjust the value for every in with the _value fields.\n\n
\n\nWhen the type is set to 'dynamic', the text box below each setting are for the individual values - follow the format provided to set values for each frame.\n", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, - { - "id": 450, - "type": "Note Plus (mtb)", - "pos": { - "0": -804.515380859375, - "1": -287.2254638671875, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 473.6533203125, - "1": 166.67845153808594 - }, - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "## Setting frame position ⮕\n\n
\n\nSetting linear frame distribution makes the frames spaced out by the linear value - for example, if it's set to 16, the frames will be at positions 0, 16, 32, etc. If you set type to dynamic, you'll need to enter the values in the text box below.", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, - { - "id": 451, - "type": "Note Plus (mtb)", - "pos": { - "0": -808.515380859375, - "1": -95.22518157958984, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 482.83489990234375, - "1": 124.60875701904297 - }, - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "## Set duration of frame influence ⮕\n
\n1.0 equals around the distance between this and the next frames, 2.0 equals twice this. You can set these values for each frame individually in the field below.", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, - { - "id": 452, - "type": "Note Plus (mtb)", - "pos": { - "0": -830.515380859375, - "1": 50.77476119995117, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 531.0975341796875, - "1": 130.3391571044922 - }, - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "## Set high and low point of influence strength ⮕\n
\n\nThe number are the range the strength runs from and to - the first value is the low point, the last is the high.", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, - { - "id": 453, - "type": "Note Plus (mtb)", - "pos": { - "0": -829.515380859375, - "1": 200.7747039794922, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 553.7985229492188, - "1": 145 - }, - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "## Relative ipadapter settings ⮕\n\n
\n Ipadapter settings are relative to the cn settings - they act like a multiplier on influence and strength. For example, a relative ipadapter influence of 1.3 will make the length of the ipadapter influence 1.3 times longer than the influence of the controlnet.\n \n\n", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, - { - "id": 447, - "type": "Note Plus (mtb)", - "pos": { - "0": -185.51510620117188, - "1": -1143.225341796875, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 229.45303344726562, - "1": 208.07003784179688 - }, - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "## ⬅ Set prompts for each frame to improve adherence\n\n
\n\nSetting prompts to the left for each frame wil increase adherence to the input images.", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, - { - "id": 401, - "type": "VHS_LoadImagesPath", - "pos": [ - -1466.5106715235622, - -1337.934232018649 + 391, + -1828 ], "size": { "0": 315, - "1": 170 - }, - "flags": {}, - "order": 10, - "mode": 0, - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 737, - 823 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null, - "shape": 3 - }, - { - "name": "INT", - "type": "INT", - "links": [], - "shape": 3, - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "VHS_LoadImagesPath" - }, - "widgets_values": { - "directory": "input", - "image_load_cap": 0, - "skip_first_images": 0, - "select_every_nth": 1 - }, - "color": "#332922", - "bgcolor": "#593930" - }, - { - "id": 448, - "type": "Note Plus (mtb)", - "pos": { - "0": 741.4848022460938, - "1": -794.2255859375, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 338.8983154296875, - "1": 120.41668701171875 - }, - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "## ⬅ Batch size\n
\nBatch size should be the number of you want to generate + the buffer size - currently 4 by default.", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, - { - "id": 454, - "type": "Note Plus (mtb)", - "pos": { - "0": 223.4849090576172, - "1": -309.2254638671875, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 210, - "1": 297.015625 - }, - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "## Observe the graph to understand the the motion ⮕\n
\n As you tweak the settings, observe the impact on the graph. You can disconnect the positive, negative and model connections to just observe the graph without creating videos.", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, - { - "id": 456, - "type": "Note Plus (mtb)", - "pos": { - "0": 221.4849090576172, - "1": -613.2255249023438, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 210, - "1": 266.1289367675781 - }, - "flags": {}, - "order": 13, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "## ⬅ Download the models from Comfy Manager\n
\nMake sure to download the correct models for IPAdapter and ControlNet Tile from Comfy Manager - the names should match the ones here or you should be sure they're the same.", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, - { - "id": 367, - "type": "BatchPromptSchedule", - "pos": [ - -769, - -1232 - ], - "size": { - "0": 538.1281127929688, - "1": 476.44915771484375 - }, - "flags": {}, - "order": 18, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 669 - } - ], - "outputs": [ - { - "name": "POS", - "type": "CONDITIONING", - "links": [ - 821 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "NEG", - "type": "CONDITIONING", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "BatchPromptSchedule" - }, - "widgets_values": [ - "\"0\" :\"\",\n\"12\" :\"\",\n\"24\" :\"\",\n\"36\" :\"\",\n\"48\" :\"\",\n\"60\" :\"\",\n\"72\" :\"\",\n\"84\" :\"\",\n\"96\" :\"\",\n\"108\" :\"\",\n\"120\" :\"\"\n", - 120, - false, - "PRE", - "APP", - 0, - 0, - 0, - 0, - 0.2 - ] - }, - { - "id": 445, - "type": "Note Plus (mtb)", - "pos": { - "0": -1647, - "1": -914, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 645.7035522460938, - "1": 1039.391357421875 - }, - "flags": {}, - "order": 14, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "# A brief guide\n\n
\n\n## Philosophy for getting the most from this:\n\n
\n\nThis isn’t a tool like text to video that will perform well out of the box - it’s more like a paint brush, an artistic tool that you need to figure out how to get the best from. \n\n
\n\nThrough trial and error, you'll need to build an understanding of how the motion and settings work, what its limitations are, which inputs work best with it, etc.\n\n
\n\nIf you can figure out how to wield it, this approach can provide enough control for you to make beautiful things that match your imagination precisely.\n\n
\n\n## How to use this:\n
\n1.Put the images in inputs/steerable-motion. The most important thing that will influence the quality of your creation is the input images.\n\n
\n\n2.Try the basic settings to start. Based on the resulting generation, try to understand what happened - you can learn about the settings and observe the graphs to the right.\n\n
\n\n3.Tweak the settings until you achieve your desired effect. \n\n
\n\n## What this will & won't be suitable for:\n\nI believe that this will be good for a wide range of storytelling-driven and abstract motion.\n\nHowever, it probably won't be good for complex motion - other approaches like vid2vid are far more suitable for this.\n\n## If you're having problems:\n\n
\n\nFirst, download the latest version of this workflow from [here](https://raw.githubusercontent.com/peteromallet/steerable-motion/main/demo/creative_interpolation_example.json) and get the latest version of the node from Comfy Manager.\n\n
\n\nIf that doesn't work, drop into [our Discord](https://discord.gg/V72vk8T67n) and share your problem and I'll try to help ASAP.\n\n
\n\n\n## Want to join a community of people who are pushing open source models to their technical and artistic limits?\n\n
\n\nYou're welcome to join [our Discord](https://discord.gg/UJf6aum7WZ).\n\n
\n\nPlease share anything you generate in the steerable-motion channel.", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, - { - "id": 389, - "type": "PreviewImage", - "pos": [ - 237, - 61 - ], - "size": { - "0": 969.0651245117188, - "1": 246 + "1": 98 }, "flags": {}, "order": 16, "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 737 - } - ], - "properties": { - "Node name for S&R": "PreviewImage" - } - }, - { - "id": 281, - "type": "VHS_VideoCombine", - "pos": [ - 3195, - -979 - ], - "size": [ - 320, - 544 - ], - "flags": {}, - "order": 25, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 656 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "VHS_VideoCombine" - }, - "widgets_values": { - "frame_rate": 14, - "loop_count": 0, - "filename_prefix": "steerable-motion/AD_", - "format": "video/h264-mp4", - "pingpong": false, - "save_image": true, - "crf": 20, - "save_metadata": true, - "audio_file": "", - "videopreview": { - "hidden": false, - "paused": false, - "params": { - "filename": "AD__00057.mp4", - "subfolder": "steerable-motion", - "type": "output", - "format": "video/h264-mp4" - } - } - } - }, - { - "id": 189, - "type": "Efficient Loader", - "pos": [ - 165, - -1241 - ], - "size": { - "0": 558.8270874023438, - "1": 523.2349853515625 - }, - "flags": { - "pinned": false - }, - "order": 15, - "mode": 0, - "inputs": [ - { - "name": "lora_stack", - "type": "LORA_STACK", - "link": null - }, - { - "name": "cnet_stack", - "type": "CONTROL_NET_STACK", - "link": null, - "slot_index": 1 - } - ], "outputs": [ { "name": "MODEL", "type": "MODEL", "links": [ - 824 + 870 ], "shape": 3, "slot_index": 0 }, - { - "name": "CONDITIONING+", - "type": "CONDITIONING", - "links": [], - "shape": 3, - "slot_index": 1 - }, - { - "name": "CONDITIONING-", - "type": "CONDITIONING", - "links": [], - "shape": 3, - "slot_index": 2 - }, - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 633 - ], - "shape": 3, - "slot_index": 3 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 632 - ], - "shape": 3, - "slot_index": 4 - }, { "name": "CLIP", "type": "CLIP", "links": [ - 644, - 669 - ], - "shape": 3, - "slot_index": 5 - }, - { - "name": "DEPENDENCIES", - "type": "DEPENDENCIES", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "Efficient Loader" - }, - "widgets_values": [ - "Realistic_Vision_V5.0.safetensors", - "vae-ft-mse-840000-ema-pruned.safetensors", - -2, - "None", - 1, - 1, - "", - "", - "none", - "comfy", - 512, - 512, - 36 - ], - "color": "#223333", - "bgcolor": "#335555", - "shape": 1 - }, - { - "id": 352, - "type": "CLIPTextEncode", - "pos": [ - 782.4848111817444, - -1197.2252734560077 - ], - "size": { - "0": 332.93072509765625, - "1": 323.7524108886719 - }, - "flags": { - "pinned": false - }, - "order": 17, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 644 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 822 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "hands, hand, (worst quality, low quality:1.2)" - ], - "color": "#322", - "bgcolor": "#533" - }, - { - "id": 437, - "type": "BatchCreativeInterpolation", - "pos": [ - -252, - -378 - ], - "size": { - "0": 444.7484436035156, - "1": 709.2474975585938 - }, - "flags": {}, - "order": 19, - "mode": 0, - "inputs": [ - { - "name": "positive", - "type": "CONDITIONING", - "link": 821 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 822 - }, - { - "name": "images", - "type": "IMAGE", - "link": 823 - }, - { - "name": "model", - "type": "MODEL", - "link": 824 - }, - { - "name": "ipadapter", - "type": "IPADAPTER", - "link": 825 - }, - { - "name": "clip_vision", - "type": "CLIP_VISION", - "link": 826 - }, - { - "name": "control_net", - "type": "CONTROL_NET", - "link": null - } - ], - "outputs": [ - { - "name": "graph", - "type": "IMAGE", - "links": [ - 841 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "positive", - "type": "CONDITIONING", - "links": [ - 858 + 893 ], "shape": 3, "slot_index": 1 }, { - "name": "negative", - "type": "CONDITIONING", - "links": [ - 859 - ], - "shape": 3, - "slot_index": 2 - }, - { - "name": "model", - "type": "MODEL", - "links": [ - 860 - ], - "shape": 3, - "slot_index": 3 + "name": "VAE", + "type": "VAE", + "links": null, + "shape": 3 } ], "properties": { - "Node name for S&R": "BatchCreativeInterpolation" + "Node name for S&R": "CheckpointLoaderSimple" }, "widgets_values": [ - "control_v11f1e_sd15_tile_fp16.safetensors", - "linear", - "16", - "0,20,36,52", - "linear", - 1.5, - "1.0,1.9,1.0", - "linear", - "(0.225,0.35)", - "(0.2,0.4),(0.0,0.25),(0.2,0.4)", - 0.85, - 4, - 1.3, - 1, - 0.3 + "Realistic_Vision_V5.0.safetensors" + ] + }, + { + "id": 465, + "type": "PrimitiveNode", + "pos": [ + 893, + -1826 + ], + "size": { + "0": 210, + "1": 82 + }, + "flags": {}, + "order": 17, + "mode": 0, + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 874 + ], + "widget": { + "name": "width" + }, + "slot_index": 0 + } + ], + "title": "Width", + "properties": { + "Run widget replace on values": false + }, + "widgets_values": [ + 512, + "fixed" ] } ], @@ -1248,30 +1524,6 @@ 1, "CONTEXT_OPTIONS" ], - [ - 632, - 189, - 4, - 207, - 4, - "VAE" - ], - [ - 633, - 189, - 3, - 207, - 3, - "LATENT" - ], - [ - 644, - 189, - 5, - 352, - 0, - "CLIP" - ], [ 656, 292, @@ -1280,14 +1532,6 @@ 0, "IMAGE" ], - [ - 669, - 189, - 5, - 367, - 0, - "CLIP" - ], [ 737, 401, @@ -1321,94 +1565,206 @@ "MODEL" ], [ - 821, - 367, - 0, - 437, - 0, - "CONDITIONING" - ], - [ - 822, - 352, - 0, - 437, - 1, - "CONDITIONING" - ], - [ - 823, + 861, 401, 0, - 437, - 2, + 457, + 0, "IMAGE" ], [ - 824, - 189, - 0, - 437, - 3, - "MODEL" - ], - [ - 825, - 369, - 0, - 437, - 4, - "IPADAPTER" - ], - [ - 826, + 863, 370, 0, - 437, - 5, + 457, + 3, "CLIP_VISION" ], [ - 841, - 437, + 864, + 369, + 0, + 457, + 2, + "IPADAPTER" + ], + [ + 865, + 457, 0, 436, 0, "IMAGE" ], [ - 858, - 437, - 1, + 867, + 458, + 0, 207, - 1, - "CONDITIONING" + 4, + "VAE" ], [ - 859, - 437, - 2, - 207, - 2, - "CONDITIONING" + 870, + 461, + 0, + 457, + 1, + "MODEL" ], [ - 860, - 437, + 873, + 457, 3, + 464, + 2, + "INT" + ], + [ + 874, + 465, + 0, + 464, + 0, + "INT" + ], + [ + 875, + 466, + 0, + 464, + 1, + "INT" + ], + [ + 876, + 464, + 0, + 207, + 3, + "LATENT" + ], + [ + 879, + 467, + 0, + 468, + 2, + "CONTROL_NET" + ], + [ + 880, + 469, + 0, + 468, + 3, + "IMAGE" + ], + [ + 881, + 401, + 0, + 469, + 0, + "IMAGE" + ], + [ + 882, + 458, + 0, + 469, + 1, + "VAE" + ], + [ + 883, + 464, + 0, + 469, + 2, + "LATENT" + ], + [ + 885, + 463, + 0, + 468, + 0, + "CONDITIONING" + ], + [ + 886, + 463, + 0, + 468, + 1, + "CONDITIONING" + ], + [ + 889, + 468, + 0, + 207, + 1, + "CONDITIONING" + ], + [ + 890, + 468, + 1, + 207, + 2, + "CONDITIONING" + ], + [ + 891, + 457, + 1, + 470, + 0, + "MODEL" + ], + [ + 892, + 470, + 0, 187, 0, "MODEL" + ], + [ + 893, + 461, + 1, + 470, + 1, + "CLIP" + ], + [ + 894, + 470, + 1, + 462, + 0, + "CLIP" + ], + [ + 895, + 470, + 1, + 463, + 0, + "CLIP" ] ], "groups": [ { "title": "First sampler, interpolation", "bounding": [ - 1523, - -1443, - 929, - 1307 + 1377, + -1935, + 1163, + 860 ], "color": "#8A8", "font_size": 24 @@ -1416,10 +1772,10 @@ { "title": "Increase Framerate", "bounding": [ - 2548, - -1104, - 496, - 694 + 1400, + -1041, + 1079, + 100 ], "color": "#8A8", "font_size": 24 @@ -1427,10 +1783,10 @@ { "title": "Saving", "bounding": [ - 3166, - -1071, - 375, - 657 + 1407, + -906, + 1008, + 1123 ], "color": "#a1309b", "font_size": 24 @@ -1438,10 +1794,10 @@ { "title": "Group", "bounding": [ - -865, - -1332, - 2173, - 1713 + -872, + -1228, + 2194, + 1190 ], "color": "#3f789e", "font_size": 24 @@ -1449,13 +1805,35 @@ { "title": "Group", "bounding": [ - -1573, - -1515, + -1537, + -1714, 580, 467 ], "color": "#3f789e", "font_size": 24 + }, + { + "title": "Basic Node", + "bounding": [ + 273, + -1948, + 996, + 678 + ], + "color": "#3f789e", + "font_size": 24 + }, + { + "title": "Group", + "bounding": [ + -846, + -2032, + 943, + 704 + ], + "color": "#3f789e", + "font_size": 24 } ], "config": {}, diff --git a/imports/AdvancedControlNet/control_sparsectrl.py b/imports/AdvancedControlNet/control_sparsectrl.py new file mode 100644 index 0000000..cb99e0f --- /dev/null +++ b/imports/AdvancedControlNet/control_sparsectrl.py @@ -0,0 +1,79 @@ +#taken from: https://github.com/lllyasviel/ControlNet +#and modified +#and then taken from comfy/cldm/cldm.py and modified again + +from abc import ABC, abstractmethod +import math +import numpy as np +from typing import Iterable, Union +import torch +import torch as th +import torch.nn as nn +from torch import Tensor +from einops import rearrange, repeat + +from comfy.ldm.modules.diffusionmodules.util import ( + zero_module, + timestep_embedding, +) + +from comfy.cldm.cldm import ControlNet as ControlNetCLDM +from comfy.ldm.modules.attention import SpatialTransformer +from comfy.ldm.modules.diffusionmodules.openaimodel import TimestepEmbedSequential, ResBlock, Downsample +from comfy.ldm.util import exists +from comfy.ldm.modules.attention import default, optimized_attention +from comfy.ldm.modules.attention import FeedForward, SpatialTransformer +from comfy.controlnet import broadcast_image_to +from comfy.utils import repeat_to_batch_size +import comfy.ops + +# from .utils import TimestepKeyframeGroup, disable_weight_init_clean_groupnorm, prepare_mask_batch + + + + + +class SparseMethodImport(ABC): + SPREAD = "spread" + INDEX = "index" + def __init__(self, method: str): + self.method = method + + @abstractmethod + def get_indexes(self, hint_length: int, full_length: int) -> list[int]: + pass + + + +class SparseIndexMethodImport(SparseMethodImport): + def __init__(self, idxs: list[int]): + super().__init__(self.INDEX) + self.idxs = idxs + + def get_indexes(self, hint_length: int, full_length: int) -> list[int]: + orig_hint_length = hint_length + if hint_length > full_length: + hint_length = full_length + # if idxs is less than hint_length, throw error + if len(self.idxs) < hint_length: + err_msg = f"There are not enough indexes ({len(self.idxs)}) provided to fit the usable {hint_length} input images." + if orig_hint_length != hint_length: + err_msg = f"{err_msg} (original input images: {orig_hint_length})" + raise ValueError(err_msg) + # cap idxs to hint_length + idxs = self.idxs[:hint_length] + new_idxs = [] + real_idxs = set() + for idx in idxs: + if idx < 0: + real_idx = full_length+idx + if real_idx in real_idxs: + raise ValueError(f"Index '{idx}' maps to '{real_idx}' and is duplicate - indexes in Sparse Index Method must be unique.") + else: + real_idx = idx + if real_idx in real_idxs: + raise ValueError(f"Index '{idx}' is duplicate (or a negative index is equivalent) - indexes in Sparse Index Method must be unique.") + real_idxs.add(real_idx) + new_idxs.append(real_idx) + return new_idxs + diff --git a/imports/AdvancedControlNet/latent_keyframe_nodes.py b/imports/AdvancedControlNet/latent_keyframe_nodes.py index 0d6ee5e..207e4e7 100644 --- a/imports/AdvancedControlNet/latent_keyframe_nodes.py +++ b/imports/AdvancedControlNet/latent_keyframe_nodes.py @@ -247,11 +247,19 @@ def calculate_weights(batch_index_from, batch_index_to, strength_from, strength_ weights = diff * (1 - np.power(1 - index, 2)) + strength_from elif interpolation == "ease-in-out": weights = diff * ((1 - np.cos(index * np.pi)) / 2) + strength_from - - # If it's a middle keyframe, mirror the weights + if revert_direction_at_midpoint: weights = np.concatenate([weights, weights[::-1]]) - + + ''' + peak_reduction = 2 + if peak_reduction > 0: + mid_point = len(weights) // 2 + start = mid_point - peak_reduction // 2 + end = mid_point + peak_reduction // 2 + weights = np.concatenate([weights[:start], weights[end:]]) + ''' + # Generate frame numbers frame_numbers = np.arange(range_start, range_start + len(weights)) diff --git a/imports/AdvancedControlNet/nodes_sparsectrl.py b/imports/AdvancedControlNet/nodes_sparsectrl.py new file mode 100644 index 0000000..012f9ec --- /dev/null +++ b/imports/AdvancedControlNet/nodes_sparsectrl.py @@ -0,0 +1,44 @@ +from torch import Tensor + +import folder_paths +from nodes import VAEEncode +import comfy.utils + +# from .utils import TimestepKeyframeGroup +from .control_sparsectrl import SparseIndexMethodImport +# from .control import load_sparsectrl, load_controlnet, ControlNetAdvanced, SparseCtrlAdvanced + + + +class SparseIndexMethodNodeImport: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "indexes": ("STRING", {"default": "0"}), + } + } + + RETURN_TYPES = ("SPARSE_METHOD",) + FUNCTION = "get_method" + + CATEGORY = "Adv-ControlNet 🛂🅐🅒🅝/SparseCtrl" + + def get_method(self, indexes: str): + idxs = [] + unique_idxs = set() + # get indeces from string + str_idxs = [x.strip() for x in indexes.strip().split(",")] + for str_idx in str_idxs: + try: + idx = int(str_idx) + if idx in unique_idxs: + raise ValueError(f"'{idx}' is duplicated; indexes must be unique.") + idxs.append(idx) + unique_idxs.add(idx) + except ValueError: + raise ValueError(f"'{str_idx}' is not a valid integer index.") + if len(idxs) == 0: + raise ValueError(f"No indexes were listed in Sparse Index Method.") + return (SparseIndexMethodImport(idxs),) +