diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..5008ddf Binary files /dev/null and b/.DS_Store differ diff --git a/README.md b/README.md index ed3607b..eb9c7e2 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,56 @@ -# ComfyUI-Advanced-ControlNet -These custom nodes allow for scheduling ControlNet strength across latents in the same batch (WORKING) and across timesteps (IN PROGRESS). +# ComfyUI-Creative-Interpolation (Beta) -Custom weights can also be applied to ControlNets and T2IAdapters to mimic the "My prompt is more important" functionality in AUTOMATIC1111's ControlNet extension. +This a ComfyUI node for batch creative interpolation. The goal is to allow you to input a batch of images, and to provide a range of simple settings to control how the images are interpolated between. -TODO: -- Other handy nodes -- Finish and update this README for other workflows +## Installation -## Workflows +1. If you haven't already, download [ComfyUI](https://github.com/comfyanonymous/ComfyUI) and [Comfy Manager](https://github.com/ltdrdata/ComfyUI-Manager). +1. go to your custom_nodes folder and run: git clone https://github.com/peteromallet/ComfyUI-Creative-Interpolation.git +2. Download Controlnet tile from Comfy Manager: control_v11f1e_sd15_tile_fp16.safetensors -### AnimateDiff Workflows -***Latent Keyframes*** identify which latents in a batch the ControlNet should apply to, and at what strength. They connect to a ***Timestep Keyframe*** to identify at what point in the generation to kick in (for basic use, start_percent on the Timestep Keyframe should be 0.0). Latent Keyframe nodes can be chained to apply the ControlNet to multiple keyframes at various strengths. +## Usage +Here's a workflow to get started with: https://banodoco.s3.amazonaws.com/creative_interpolation_example.json + +You'll need to drop the input images into the 'creative_interpolation_input' folder. + +The main settings are: + +- frames_per_key_frame: How many frames to generate between each main key frame you provide. +- length_of_key_frame_influence: How many frames to apply the ControlNet for after each key frame - the larger the number, the the wider range the input images will influence. +- cn_strength: How strong the control of the ControlNet should overall. + +The **batch_size should be equal to the frames_per_key_frame * number_of_key_frames + 4** - the additional 4 is to accommodate for buffer frames that improve consistency. + +Also, **there's currently a bug where it needs to be restarted after each batch**. This will be fixed soon. + +As an example, here's a batch of input images: + +![Batch of input images](https://banodoco.s3.amazonaws.com/interpolation_input.png) + +And here's what it looks like when the length_of_key_frame_influence is set to 1.1: + +![1.1 Interpolation](https://banodoco.s3.amazonaws.com/1.1.gif)https://github.com/Fannovel16/ComfyUI-Frame-Interpolation + +While here's while that looks like at 0.8: + +![0.8 Interpolation](https://banodoco.s3.amazonaws.com/0.8.gif) + +These settings can be so powerful I believe - please share what works for you and your results! + +## Coming Soon + +- Fix restarting bug +- Clean up code +- Simplify settings +- Nuanced settings for each key frame +- Improvements to structure and consistency +- Better control over style + +## Want to give feedback, share creations, or join our community? + +You can drop into our Discord here: https://discord.com/invite/8Wx9dFu5tP + +## Credits + +This code draws heavily from [Kosinkadink's ComfyUI-Advanced-ControlNet](https://github.com/Kosinkadink/ComfyUI-Advanced-ControlNet), while the workflows uses [Kosinkadink's Animatediff Evolved](https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved), Cubiq's [IPAdapter_plus](https://github.com/cubiq/ComfyUI_IPAdapter_plus), Fizzledorf's [Fizznodes](https://github.com/FizzleDorf/ComfyUI_FizzNodes), Fannovel16's [Frame Interpolation](https://github.com/Fannovel16/ComfyUI-Frame-Interpolation) and more. Thanks to all and of course the Animatediff team, Controlnet, others, and of course our supportive community! \ No newline at end of file diff --git a/control/latent_keyframe_nodes.py b/control/latent_keyframe_nodes.py index c7c4d1d..81e00c6 100644 --- a/control/latent_keyframe_nodes.py +++ b/control/latent_keyframe_nodes.py @@ -144,7 +144,7 @@ class LatentKeyframeInterpolationNode: "strength_from": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.0001}, ), "strength_to": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.0001}, ), "interpolation": (["linear", "ease-in", "ease-out", "ease-in-out"], ), - "return_at_midpoint": ("BOOLEAN", {"default": False}), + "revert_direction_at_midpoint": ("BOOLEAN", {"default": False}), }, "optional": { "prev_latent_keyframe": ("LATENT_KEYFRAME", ), diff --git a/control/nodes.py b/control/nodes.py index 19796c6..e2a04ac 100644 --- a/control/nodes.py +++ b/control/nodes.py @@ -11,9 +11,6 @@ from .deprecated_nodes import LoadImagesFromDirectory from .logger import logger - - - class TimestepKeyframeNode: @classmethod def INPUT_TYPES(s): @@ -165,9 +162,9 @@ class BatchCreativeInterpolationNode: "negative": ("CONDITIONING", ), "control_net_name": (folder_paths.get_filename_list("controlnet"), ), "images": ("IMAGE", ), - "length_of_key_frame_influence": ("FLOAT", {"default": 1.1, "min": 0.0, "max": 2.0, "step": 0.001}), - "cn_strength": ("FLOAT", {"default": 0.9, "min": 0.0, "max": 1.0, "step": 0.01}), "frames_per_keyframe": ("INT", {"default": 16, "min": 4, "max": 64, "step": 1}), + "length_of_key_frame_influence": ("FLOAT", {"default": 1.1, "min": 0.0, "max": 2.0, "step": 0.001}), + "cn_strength": ("FLOAT", {"default": 0.9, "min": 0.0, "max": 1.0, "step": 0.01}), "interpolation": (["ease-in", "ease-out", "ease-in-out"],), }, "optional": { @@ -182,8 +179,7 @@ class BatchCreativeInterpolationNode: def combined_function(self, positive, negative, control_net_name, images, length_of_key_frame_influence,cn_strength,frames_per_keyframe,interpolation): - def calculate_keyframe_peaks_and_influence(frames_per_keyframe, number_of_keyframes, length_of_influence): - + def calculate_keyframe_peaks_and_influence(frames_per_keyframe, number_of_keyframes, length_of_influence, new_influence_range=(0,4)): number_of_frames = frames_per_keyframe * number_of_keyframes # Calculate the interval between keyframes interval = (number_of_frames - 1) // (number_of_keyframes - 1) @@ -198,7 +194,7 @@ class BatchCreativeInterpolationNode: if i <= adjustment: peak += 1 peaks.append(peak) - peaks.append(number_of_frames) # The last keyframe is always at the last frame + peaks.append(number_of_frames - 1) # The last keyframe is always at the last frame # Calculate the full interval between keyframes full_interval = (number_of_frames - 1) / (number_of_keyframes - 1) @@ -206,13 +202,16 @@ class BatchCreativeInterpolationNode: scaled_interval = full_interval * length_of_influence # Initialize the list to store the influence range for each keyframe - influence_ranges = [] + influence_ranges = [new_influence_range] - # Loop through each keyframe to calculate its influence range + # Shift the subsequent influence ranges + shift = new_influence_range[1] + + # Loop through each keyframe to calculate its shifted influence range for i, peak in enumerate(peaks): - # Calculate the start and end influence around the peak - start_influence = max(0, int(peak - scaled_interval / 2.0)) - end_influence = min(number_of_frames, int(peak + scaled_interval / 2.0)) + # Calculate the start and end influence around the peak, shifted by the new range's end + start_influence = max(shift, int(peak - scaled_interval / 2.0) + shift) + end_influence = min(number_of_frames + shift, int(peak + scaled_interval / 2.0) + shift) # Add the influence range as a tuple (start, end) to the list influence_ranges.append((start_influence, end_influence)) @@ -220,21 +219,27 @@ class BatchCreativeInterpolationNode: influence_ranges = calculate_keyframe_peaks_and_influence(frames_per_keyframe, len(images), length_of_key_frame_influence) - for i, image in enumerate(images): + for i, (start, end) in enumerate(influence_ranges): batch_index_from, batch_index_to_excl = influence_ranges[i] - if i == 0: # First image + if i == 0: # buffer image + image = images[0] strength_from = 1.0 - strength_to = 0.0 - return_at_midpoint = False - + strength_to = 1.0 + return_at_midpoint = False + elif i == 1: # First image + image = images[0] + strength_from = 1.0 + strength_to = 0.0 + return_at_midpoint = False elif i == len(images) - 1: # Last image + image = images[i-1] strength_from = 0.0 strength_to = 1.0 - return_at_midpoint = False - + return_at_midpoint = False else: # Middle images + image = images[i-1] strength_from = 0.0 strength_to = 1.0 return_at_midpoint = True @@ -242,11 +247,6 @@ class BatchCreativeInterpolationNode: latent_keyframe_interpolation_node = LatentKeyframeInterpolationNode() latent_keyframe, = latent_keyframe_interpolation_node.load_keyframe(batch_index_from, strength_from, batch_index_to_excl, strength_to, interpolation,return_at_midpoint) - if i == len(images) - 1: - for i in range(10): - keyframe = TimestepKeyframe(batch_index_to_excl + i, 1.0) - timestep_keyframe.add(keyframe) - scaled_soft_control_net_weights = ScaledSoftControlNetWeights() control_net_weights, _ = scaled_soft_control_net_weights.load_weights(0.85, False) @@ -261,8 +261,7 @@ class BatchCreativeInterpolationNode: control_net_loader = ControlNetLoaderAdvanced() control_net, = control_net_loader.load_controlnet(control_net_name, timestep_keyframe) - - + apply_advanced_control_net = AdvancedControlNetApply() positive, negative = apply_advanced_control_net.apply_controlnet(positive, negative, control_net, image.unsqueeze(0), cn_strength, 0.0, 1.0)