From 13c04eda4ad6eaa969decc727785108de72d7177 Mon Sep 17 00:00:00 2001
From: peter942 <peter@omalley.io>
Date: Mon, 13 Nov 2023 13:20:31 +0100
Subject: [PATCH] Fixed starting issue + added README

---
 .DS_Store                        | Bin 0 -> 6148 bytes
 README.md                        |  60 ++++++++++++++++++++++++++-----
 control/latent_keyframe_nodes.py |   2 +-
 control/nodes.py                 |  53 ++++++++++++++-------------
 4 files changed, 78 insertions(+), 37 deletions(-)
 create mode 100644 .DS_Store

diff --git a/.DS_Store b/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6
GIT binary patch
literal 6148
zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3
zem<@ulZcFPQ@L2!n>{z**<q8>++&mCkOWA81W14cNZ<zv;LbK1Poaz?KmsK2CSc!(
z0ynLxE!0092;Krf2c+FF_Fe*7ECH>lEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ
zLs35+`xjp>T0<F0fCPF1$Cyrb|F7^5{eNG?83~ZUUlGt@xh*qZDeu<Z%US-OSsOPv
j)R!Z4KLME7ReXlK;d!wEw5GODWMKRea10D2@KpjYNUI8I

literal 0
HcmV?d00001

diff --git a/README.md b/README.md
index ed3607b..eb9c7e2 100644
--- a/README.md
+++ b/README.md
@@ -1,14 +1,56 @@
-# ComfyUI-Advanced-ControlNet
-These custom nodes allow for scheduling ControlNet strength across latents in the same batch (WORKING) and across timesteps (IN PROGRESS).
+# ComfyUI-Creative-Interpolation (Beta)
 
-Custom weights can also be applied to ControlNets and T2IAdapters to mimic the "My prompt is more important" functionality in AUTOMATIC1111's ControlNet extension.
+This a ComfyUI node for batch creative interpolation. The goal is to allow you to input a batch of images, and to provide a range of simple settings to control how the images are interpolated between. 
 
-TODO:
-- Other handy nodes
-- Finish and update this README for other workflows
+## Installation
 
-## Workflows
+1. If you haven't already, download [ComfyUI](https://github.com/comfyanonymous/ComfyUI) and [Comfy Manager](https://github.com/ltdrdata/ComfyUI-Manager).
+1. go to your custom_nodes folder and run: git clone https://github.com/peteromallet/ComfyUI-Creative-Interpolation.git
+2. Download Controlnet tile from Comfy Manager: control_v11f1e_sd15_tile_fp16.safetensors
 
-### AnimateDiff Workflows
-***Latent Keyframes*** identify which latents in a batch the ControlNet should apply to, and at what strength. They connect to a ***Timestep Keyframe*** to identify at what point in the generation to kick in (for basic use, start_percent on the Timestep Keyframe should be 0.0). Latent Keyframe nodes can be chained to apply the ControlNet to multiple keyframes at various strengths.
+## Usage
 
+Here's a workflow to get started with: https://banodoco.s3.amazonaws.com/creative_interpolation_example.json
+
+You'll need to drop the input images into the 'creative_interpolation_input' folder.
+
+The main settings are:
+
+- frames_per_key_frame: How many frames to generate between each main key frame you provide.
+- length_of_key_frame_influence: How many frames to apply the ControlNet for after each key frame - the larger the number, the the wider range the input images will influence.
+- cn_strength: How strong the control of the ControlNet should overall.
+
+The **batch_size should be equal to the frames_per_key_frame * number_of_key_frames + 4** - the additional 4 is to accommodate for buffer frames that improve consistency. 
+
+Also, **there's currently a bug where it needs to be restarted after each batch**. This will be fixed soon.
+
+As an example, here's a batch of input images:
+
+![Batch of input images](https://banodoco.s3.amazonaws.com/interpolation_input.png)
+
+And here's what it looks like when the length_of_key_frame_influence is set to 1.1:
+
+![1.1 Interpolation](https://banodoco.s3.amazonaws.com/1.1.gif)https://github.com/Fannovel16/ComfyUI-Frame-Interpolation
+
+While here's while that looks like at 0.8:
+
+![0.8 Interpolation](https://banodoco.s3.amazonaws.com/0.8.gif)
+
+These settings can be so powerful I believe - please share what works for you and your results!
+
+## Coming Soon
+
+- Fix restarting bug
+- Clean up code
+- Simplify settings
+- Nuanced settings for each key frame
+- Improvements to structure and consistency
+- Better control over style
+
+## Want to give feedback, share creations, or join our community?
+
+You can drop into our Discord here: https://discord.com/invite/8Wx9dFu5tP
+
+## Credits
+
+This code draws heavily from [Kosinkadink's ComfyUI-Advanced-ControlNet](https://github.com/Kosinkadink/ComfyUI-Advanced-ControlNet), while the workflows uses [Kosinkadink's Animatediff Evolved](https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved), Cubiq's [IPAdapter_plus](https://github.com/cubiq/ComfyUI_IPAdapter_plus), Fizzledorf's [Fizznodes](https://github.com/FizzleDorf/ComfyUI_FizzNodes), Fannovel16's [Frame Interpolation](https://github.com/Fannovel16/ComfyUI-Frame-Interpolation) and more. Thanks to all and of course the Animatediff team, Controlnet, others, and of course our supportive community!
\ No newline at end of file
diff --git a/control/latent_keyframe_nodes.py b/control/latent_keyframe_nodes.py
index c7c4d1d..81e00c6 100644
--- a/control/latent_keyframe_nodes.py
+++ b/control/latent_keyframe_nodes.py
@@ -144,7 +144,7 @@ class LatentKeyframeInterpolationNode:
                 "strength_from": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.0001}, ),
                 "strength_to": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.0001}, ),
                 "interpolation": (["linear", "ease-in", "ease-out", "ease-in-out"], ),
-                "return_at_midpoint": ("BOOLEAN", {"default": False}),
+                "revert_direction_at_midpoint": ("BOOLEAN", {"default": False}),
             },
             "optional": {
                 "prev_latent_keyframe": ("LATENT_KEYFRAME", ),
diff --git a/control/nodes.py b/control/nodes.py
index 19796c6..e2a04ac 100644
--- a/control/nodes.py
+++ b/control/nodes.py
@@ -11,9 +11,6 @@ from .deprecated_nodes import LoadImagesFromDirectory
 from .logger import logger
 
 
-
-
-
 class TimestepKeyframeNode:
     @classmethod
     def INPUT_TYPES(s):
@@ -165,9 +162,9 @@ class BatchCreativeInterpolationNode:
                 "negative": ("CONDITIONING", ),
                 "control_net_name": (folder_paths.get_filename_list("controlnet"), ),
                 "images": ("IMAGE", ),
-                "length_of_key_frame_influence": ("FLOAT", {"default": 1.1, "min": 0.0, "max": 2.0, "step": 0.001}),
-                "cn_strength": ("FLOAT", {"default": 0.9, "min": 0.0, "max": 1.0, "step": 0.01}),
                 "frames_per_keyframe": ("INT", {"default": 16, "min": 4, "max": 64, "step": 1}),
+                "length_of_key_frame_influence": ("FLOAT", {"default": 1.1, "min": 0.0, "max": 2.0, "step": 0.001}),
+                "cn_strength": ("FLOAT", {"default": 0.9, "min": 0.0, "max": 1.0, "step": 0.01}),                
                 "interpolation": (["ease-in", "ease-out", "ease-in-out"],),
             },
             "optional": {
@@ -182,8 +179,7 @@ class BatchCreativeInterpolationNode:
 
     def combined_function(self, positive, negative, control_net_name, images, length_of_key_frame_influence,cn_strength,frames_per_keyframe,interpolation):
         
-        def calculate_keyframe_peaks_and_influence(frames_per_keyframe, number_of_keyframes, length_of_influence):
-
+        def calculate_keyframe_peaks_and_influence(frames_per_keyframe, number_of_keyframes, length_of_influence, new_influence_range=(0,4)):
             number_of_frames = frames_per_keyframe * number_of_keyframes
             # Calculate the interval between keyframes
             interval = (number_of_frames - 1) // (number_of_keyframes - 1)
@@ -198,7 +194,7 @@ class BatchCreativeInterpolationNode:
                 if i <= adjustment:
                     peak += 1
                 peaks.append(peak)
-            peaks.append(number_of_frames)  # The last keyframe is always at the last frame
+            peaks.append(number_of_frames - 1)  # The last keyframe is always at the last frame
 
             # Calculate the full interval between keyframes
             full_interval = (number_of_frames - 1) / (number_of_keyframes - 1)
@@ -206,13 +202,16 @@ class BatchCreativeInterpolationNode:
             scaled_interval = full_interval * length_of_influence
 
             # Initialize the list to store the influence range for each keyframe
-            influence_ranges = []
+            influence_ranges = [new_influence_range]
 
-            # Loop through each keyframe to calculate its influence range
+            # Shift the subsequent influence ranges
+            shift = new_influence_range[1]
+
+            # Loop through each keyframe to calculate its shifted influence range
             for i, peak in enumerate(peaks):
-                # Calculate the start and end influence around the peak
-                start_influence = max(0, int(peak - scaled_interval / 2.0))
-                end_influence = min(number_of_frames, int(peak + scaled_interval / 2.0))
+                # Calculate the start and end influence around the peak, shifted by the new range's end
+                start_influence = max(shift, int(peak - scaled_interval / 2.0) + shift)
+                end_influence = min(number_of_frames + shift, int(peak + scaled_interval / 2.0) + shift)
                 # Add the influence range as a tuple (start, end) to the list
                 influence_ranges.append((start_influence, end_influence))
 
@@ -220,21 +219,27 @@ class BatchCreativeInterpolationNode:
                         
         influence_ranges = calculate_keyframe_peaks_and_influence(frames_per_keyframe, len(images), length_of_key_frame_influence)
 
-        for i, image in enumerate(images):
+        for i, (start, end) in enumerate(influence_ranges):
             
             batch_index_from, batch_index_to_excl = influence_ranges[i]
                                                                                                 
-            if i == 0:  # First image
+            if i == 0:  # buffer image
+                image = images[0]
                 strength_from = 1.0
-                strength_to = 0.0                
-                return_at_midpoint = False                
-                
+                strength_to = 1.0            
+                return_at_midpoint = False 
+            elif i == 1: # First image
+                image = images[0]
+                strength_from = 1.0
+                strength_to = 0.0
+                return_at_midpoint = False                            
             elif i == len(images) - 1:  # Last image
+                image = images[i-1]
                 strength_from = 0.0
                 strength_to = 1.0
-                return_at_midpoint = False                
-                
+                return_at_midpoint = False                                
             else:  # Middle images
+                image = images[i-1]
                 strength_from = 0.0
                 strength_to = 1.0
                 return_at_midpoint = True
@@ -242,11 +247,6 @@ class BatchCreativeInterpolationNode:
             latent_keyframe_interpolation_node = LatentKeyframeInterpolationNode()
             latent_keyframe, = latent_keyframe_interpolation_node.load_keyframe(batch_index_from, strength_from, batch_index_to_excl, strength_to, interpolation,return_at_midpoint)                        
                 
-            if i == len(images) - 1:
-                for i in range(10):
-                    keyframe = TimestepKeyframe(batch_index_to_excl + i, 1.0)
-                    timestep_keyframe.add(keyframe)
-
             scaled_soft_control_net_weights = ScaledSoftControlNetWeights()
             control_net_weights, _ = scaled_soft_control_net_weights.load_weights(0.85, False)
 
@@ -261,8 +261,7 @@ class BatchCreativeInterpolationNode:
 
             control_net_loader = ControlNetLoaderAdvanced()
             control_net, = control_net_loader.load_controlnet(control_net_name, timestep_keyframe)
-            
-            
+                        
             apply_advanced_control_net = AdvancedControlNetApply()            
                         
             positive, negative = apply_advanced_control_net.apply_controlnet(positive, negative, control_net, image.unsqueeze(0), cn_strength, 0.0, 1.0)