From 3a430c9e5f1c4fa9f42817eb41f3354e469e5a73 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Wed, 25 Oct 2023 08:15:11 -0500 Subject: [PATCH] Started work on new features --- control/control.py | 17 ++++++++++++++++- control/latent_keyframe_nodes.py | 11 ++++++----- control/nodes.py | 12 ++++++++++-- control/reference_nodes.py | 12 ++++++++++++ 4 files changed, 44 insertions(+), 8 deletions(-) create mode 100644 control/reference_nodes.py diff --git a/control/control.py b/control/control.py index 0508ae4..10960a9 100644 --- a/control/control.py +++ b/control/control.py @@ -11,6 +11,14 @@ ControlNetWeightsType = list[float] T2IAdapterWeightsType = list[float] +class StrengthInterpolation: + LINEAR = "linear" + EASE_IN = "ease-in" + EASE_OUT = "ease-out" + EASE_IN_OUT = "ease-in-out" + NONE = "none" + + class LatentKeyframe: def __init__(self, batch_index: int, strength: float) -> None: self.batch_index = batch_index @@ -50,11 +58,15 @@ class LatentKeyframeGroup: class TimestepKeyframe: def __init__(self, start_percent: float = 0.0, + strength: float = 1.0, + interpolation: str = StrengthInterpolation.LINEAR, control_net_weights: ControlNetWeightsType = None, t2i_adapter_weights: T2IAdapterWeightsType = None, latent_keyframes: LatentKeyframeGroup = None, default_latent_strength: float = 0.0) -> None: self.start_percent = start_percent + self.strength = strength + self.interpolation = interpolation self.control_net_weights = control_net_weights self.t2i_adapter_weights = t2i_adapter_weights self.latent_keyframes = latent_keyframes @@ -229,7 +241,10 @@ class ControlNetAdvanced(ControlNet): self.mask_cond_hint = self.mask_cond_hint.to(self.control_model.dtype).to(self.device) context = cond['c_crossattn'] - y = cond.get('c_adm', None) + # uses 'y' in new ComfyUI update + y = cond.get('y', None) + if y is None: # TODO: remove this in the future since no longer used by newest ComfyUI + y = cond.get('c_adm', None) if y is not None: y = y.to(self.control_model.dtype) control = self.control_model(x=x_noisy.to(self.control_model.dtype), hint=self.cond_hint, timesteps=t, context=context.to(self.control_model.dtype), y=y) diff --git a/control/latent_keyframe_nodes.py b/control/latent_keyframe_nodes.py index d12935c..57cd280 100644 --- a/control/latent_keyframe_nodes.py +++ b/control/latent_keyframe_nodes.py @@ -3,6 +3,7 @@ import numpy as np from collections.abc import Iterable from .control import LatentKeyframe, LatentKeyframeGroup +from .control import StrengthInterpolation as SI from .logger import logger @@ -143,7 +144,7 @@ class LatentKeyframeInterpolationNode: "batch_index_to_excl": ("INT", {"default": 0, "min": -10000, "max": 10000, "step": 1}), "strength_from": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.0001}, ), "strength_to": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.0001}, ), - "interpolation": (["linear", "ease-in", "ease-out", "ease-in-out"], ), + "interpolation": ([SI.LINEAR, SI.EASE_IN, SI.EASE_OUT, SI.EASE_IN_OUT], ), }, "optional": { "prev_latent_keyframe": ("LATENT_KEYFRAME", ), @@ -174,15 +175,15 @@ class LatentKeyframeInterpolationNode: steps = batch_index_to_excl - batch_index_from diff = strength_to - strength_from - if interpolation == "linear": + if interpolation == SI.LINEAR: weights = np.linspace(strength_from, strength_to, steps) - elif interpolation == "ease-in": + elif interpolation == SI.EASE_IN: index = np.linspace(0, 1, steps) weights = diff * np.power(index, 2) + strength_from - elif interpolation == "ease-out": + elif interpolation == SI.EASE_OUT: index = np.linspace(0, 1, steps) weights = diff * (1 - np.power(1 - index, 2)) + strength_from - elif interpolation == "ease-in-out": + elif interpolation == SI.EASE_IN_OUT: index = np.linspace(0, 1, steps) weights = diff * ((1 - np.cos(index * np.pi)) / 2) + strength_from diff --git a/control/nodes.py b/control/nodes.py index b43f78f..74aeddb 100644 --- a/control/nodes.py +++ b/control/nodes.py @@ -2,8 +2,9 @@ import numpy as np import folder_paths -from .control import ControlNetAdvanced, T2IAdapterAdvanced, load_controlnet, ControlNetWeightsType, T2IAdapterWeightsType,\ +from .control import load_controlnet, ControlNetWeightsType, T2IAdapterWeightsType,\ LatentKeyframeGroup, TimestepKeyframe, TimestepKeyframeGroup, is_advanced_controlnet +from .control import StrengthInterpolation as SI from .weight_nodes import ScaledSoftControlNetWeights, SoftControlNetWeights, CustomControlNetWeights, \ SoftT2IAdapterWeights, CustomT2IAdapterWeights from .latent_keyframe_nodes import LatentKeyframeGroupNode, LatentKeyframeInterpolationNode, LatentKeyframeBatchedGroupNode, LatentKeyframeNode @@ -17,6 +18,9 @@ class TimestepKeyframeNode: return { "required": { "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}, ), + "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.001}, ), + "interpolation": ([SI.LINEAR, SI.EASE_IN, SI.EASE_OUT, SI.EASE_IN_OUT, SI.NONE], ), + "default_latent_strength": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 10.0, "step": 0.001}, ), }, "optional": { "control_net_weights": ("CONTROL_NET_WEIGHTS", ), @@ -33,13 +37,17 @@ class TimestepKeyframeNode: def load_keyframe(self, start_percent: float, + strength: float, + interpolation: str, + default_latent_strength: float, control_net_weights: ControlNetWeightsType=None, t2i_adapter_weights: T2IAdapterWeightsType=None, latent_keyframe: LatentKeyframeGroup=None, prev_timestep_keyframe: TimestepKeyframeGroup=None): if not prev_timestep_keyframe: prev_timestep_keyframe = TimestepKeyframeGroup() - keyframe = TimestepKeyframe(start_percent, control_net_weights, t2i_adapter_weights, latent_keyframe) + keyframe = TimestepKeyframe(start_percent=start_percent, strength=strength, interpolation=interpolation, default_latent_strength=default_latent_strength, + control_net_weights=control_net_weights, t2i_adapter_weights=t2i_adapter_weights, latent_keyframes=latent_keyframe) prev_timestep_keyframe.add(keyframe) return (prev_timestep_keyframe,) diff --git a/control/reference_nodes.py b/control/reference_nodes.py new file mode 100644 index 0000000..6879f97 --- /dev/null +++ b/control/reference_nodes.py @@ -0,0 +1,12 @@ +class AnimateDiffLoaderWithContext: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ("MODEL",), + "image": ("IMAGE",), + }, + } + + RETURN_TYPES = ("MODEL",) + CATEGORY = "" \ No newline at end of file