From e15ac1fe30ddd373b8bba516ecccaa0da0cf3192 Mon Sep 17 00:00:00 2001 From: ryanontheinstide Date: Wed, 23 Oct 2024 14:42:06 -0400 Subject: [PATCH 01/10] flex base --- nodes/flex/flex_base.py | 61 +++++++++++++++++++++++++++++ nodes/images/image_base.py | 79 +++++++++++++++++++++----------------- nodes/masks/mask_base.py | 2 +- 3 files changed, 106 insertions(+), 36 deletions(-) create mode 100644 nodes/flex/flex_base.py diff --git a/nodes/flex/flex_base.py b/nodes/flex/flex_base.py new file mode 100644 index 0000000..1e9a6c5 --- /dev/null +++ b/nodes/flex/flex_base.py @@ -0,0 +1,61 @@ +from abc import ABC, abstractmethod +from comfy.utils import ProgressBar +import numpy as np +import torch + +class FlexBase(ABC): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), + "feature_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), + "feature_param": (cls.get_modifiable_params(), {"default": cls.get_modifiable_params()[0] if cls.get_modifiable_params() else "None"}), + "feature_mode": (["relative", "absolute"], {"default": "relative"}), + }, + "optional": { + "feature": ("FEATURE",), + "feature_pipe": ("FEATURE_PIPE",), + } + } + + CATEGORY = "RyanOnTheInside/FlexBase" + RETURN_TYPES = () # To be defined by subclasses + FUNCTION = "apply_effect" + + def __init__(self): + self.progress_bar = None + + def start_progress(self, total_steps, desc="Processing"): + self.progress_bar = ProgressBar(total_steps) + + def update_progress(self): + if self.progress_bar: + self.progress_bar.update(1) + + def end_progress(self): + self.progress_bar = None + + @classmethod + @abstractmethod + def get_modifiable_params(cls): + """Return a list of parameter names that can be modulated.""" + return [] + + def modulate_param(self, param_name, param_value, feature_value, strength, mode): + if mode == "relative": + # Adjust parameter relative to its value and the feature + return param_value * (1 + (feature_value - 0.5) * 2 * strength) + else: # absolute + # Adjust parameter directly based on the feature + return param_value * feature_value * strength + + @abstractmethod + def apply_effect(self, *args, **kwargs): + """Main method to apply the effect.""" + pass + + @abstractmethod + def apply_effect_internal(self, *args, **kwargs): + """Internal method to be implemented by subclasses.""" + pass diff --git a/nodes/images/image_base.py b/nodes/images/image_base.py index 7f405a7..cdf211a 100644 --- a/nodes/images/image_base.py +++ b/nodes/images/image_base.py @@ -4,17 +4,16 @@ from abc import ABC, abstractmethod from tqdm import tqdm from comfy.utils import ProgressBar from ... import RyanOnTheInside +from ..flex.flex_base import FlexBase -class FlexImageBase(RyanOnTheInside, ABC): +class FlexImageBase(RyanOnTheInside, FlexBase): @classmethod def INPUT_TYPES(cls): return { + **super().INPUT_TYPES(), "required": { + **super().INPUT_TYPES()["required"], "images": ("IMAGE",), - "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), - "feature_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), - "feature_param": (cls.get_modifiable_params(), {"default": cls.get_modifiable_params()[0]}), - "feature_mode": (["relative", "absolute"], {"default": "relative"}), }, "optional": { "opt_feature": ("FEATURE",), @@ -51,28 +50,32 @@ class FlexImageBase(RyanOnTheInside, ABC): else: # absolute return param_value * feature_value * strength - def apply_effect(self, images, strength, feature_threshold, feature_param, feature_mode, opt_feature=None, opt_feature_pipe=None, **kwargs): - if (opt_feature is None) != (opt_feature_pipe is None): - raise ValueError("Both opt_feature and opt_feature_pipe must be provided together, or neither should be provided.") + def apply_effect(self, images, strength, feature_threshold, feature_param, feature_mode, feature=None, feature_pipe=None, **kwargs): + if (feature is None) != (feature_pipe is None): + raise ValueError("Both feature and feature_pipe must be provided together, or neither should be provided.") - if opt_feature is None and opt_feature_pipe is None: - # If neither feature nor feature_pipe is provided, process all frames without modulation + if feature is None and feature_pipe is None: + # Process all frames without modulation num_frames = images.shape[0] images_np = images.cpu().numpy() - + self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}") - + result = [] for i in range(num_frames): - processed_image = self.process_image(images_np[i], 0.5, strength, - feature_param=feature_param, - feature_mode=feature_mode, - **kwargs) + processed_image = self.process_image( + images_np[i], + 0.5, + strength, + feature_param=feature_param, + feature_mode=feature_mode, + **kwargs + ) result.append(processed_image) self.update_progress() else: - # Original behavior when both feature and feature_pipe are provided - num_frames = opt_feature_pipe.frame_count + # Process frames with modulation based on feature values + num_frames = feature_pipe.frame_count images_np = images.cpu().numpy() self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}") @@ -80,13 +83,17 @@ class FlexImageBase(RyanOnTheInside, ABC): result = [] for i in range(num_frames): image = images_np[i] - feature_value = opt_feature.get_value_at_frame(i) + feature_value = feature.get_value_at_frame(i) kwargs['frame_index'] = i if feature_value >= feature_threshold: - processed_image = self.process_image(image, feature_value, strength, - feature_param=feature_param, - feature_mode=feature_mode, - **kwargs) + processed_image = self.process_image( + image, + feature_value, + strength, + feature_param=feature_param, + feature_mode=feature_mode, + **kwargs + ) else: processed_image = image @@ -97,29 +104,31 @@ class FlexImageBase(RyanOnTheInside, ABC): # Convert the list of numpy arrays to a single numpy array result_np = np.stack(result) - - # Convert the numpy array to a PyTorch tensor and ensure it's in BHWC format + + # Convert the numpy array to a PyTorch tensor in BHWC format result_tensor = torch.from_numpy(result_np).float() - - # If the tensor is not in BHWC format, transpose it if result_tensor.shape[1] == 3: # If it's in BCHW format result_tensor = result_tensor.permute(0, 2, 3, 1) return (result_tensor,) - def process_image(self, image: np.ndarray, feature_value: float, strength: float, + def process_image(self, image: np.ndarray, feature_value: float, strength: float, feature_param: str, feature_mode: str, **kwargs) -> np.ndarray: # Modulate the selected parameter for param_name in self.get_modifiable_params(): - if param_name in kwargs: - if param_name == feature_param: - kwargs[param_name] = self.modulate_param(param_name, kwargs[param_name], - feature_value, strength, feature_mode) - - # Call the child class's implementation + if param_name in kwargs and param_name == feature_param: + kwargs[param_name] = self.modulate_param( + param_name, + kwargs[param_name], + feature_value, + strength, + feature_mode + ) + + # Call the subclass's implementation return self.apply_effect_internal(image, **kwargs) @abstractmethod def apply_effect_internal(self, image: np.ndarray, **kwargs) -> np.ndarray: - """Apply the effect to the image. To be implemented by child classes.""" + """To be implemented by subclasses.""" pass diff --git a/nodes/masks/mask_base.py b/nodes/masks/mask_base.py index ed36032..7363361 100644 --- a/nodes/masks/mask_base.py +++ b/nodes/masks/mask_base.py @@ -275,7 +275,7 @@ class OpticalFlowMaskBase(MaskBase, ABC): processed_masks = np.stack(result) return self.apply_mask_operation(processed_masks, masks, strength, **kwargs) -#TODO check if input mask is blank and just skip, then check children +#TODO SOON TO UPDATE TO SUB FLEXBASE. UNTIL THEN REFER TO IMAGES AUDIO OR VIDEO FOR EXAMPLES ON HOW TO EXTEND class FlexMaskBase(MaskBase): feature_threshold_default=0.0 @classmethod From 3568df3e612ccfd3283f07fef993488d86185942 Mon Sep 17 00:00:00 2001 From: ryanontheinstide Date: Thu, 24 Oct 2024 09:32:11 -0400 Subject: [PATCH 02/10] d --- nodes/audio/flex_audio.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/nodes/audio/flex_audio.py b/nodes/audio/flex_audio.py index f2f7bbe..4d9e91c 100644 --- a/nodes/audio/flex_audio.py +++ b/nodes/audio/flex_audio.py @@ -151,9 +151,6 @@ class FlexAudioBase(RyanOnTheInside, ABC): pass - - - class FlexAudioPitchShift(FlexAudioBase): @classmethod def INPUT_TYPES(cls): From be5236acc6680a669e9d06c140b29684d42cb17a Mon Sep 17 00:00:00 2001 From: ryanontheinstide Date: Sat, 26 Oct 2024 07:24:47 -0400 Subject: [PATCH 03/10] Squashed commit of the following: commit caeca22fe49b4c5dde0236f2d0e6b820aca763a6 Author: ryanontheinstide Date: Fri Oct 25 14:23:30 2024 -0400 smiley example commit e446017b1b1e4f2dde8d376b7c58682b1037eb6e Author: ryanontheinstide Date: Fri Oct 25 14:08:07 2024 -0400 visualizer beta commit 585ee8d574cd5075807bf0c956fbff1916eb8935 Author: ryanontheinstide Date: Fri Oct 25 09:17:14 2024 -0400 interpolation_mode removed commit a3d5e5b2941e3c0e1d42508837a57985e1145369 Author: ryanontheinstide Date: Thu Oct 24 13:01:17 2024 -0400 fixed inherit inputs commit 7b0009877622c782866c502baf39852f62438e32 Author: ryanontheinstide Date: Thu Oct 24 11:23:21 2024 -0400 unusable feature param --- __init__.py | 3 + examples/smiley_interp_tutoprial.json | 7351 +++++++++++++++++++++++++ nodes/audio/flex_audio_visualizer.py | 4 +- nodes/images/flex_images.py | 276 +- nodes/latents/latent_base.py | 4 +- pyproject.toml | 2 +- 6 files changed, 7483 insertions(+), 157 deletions(-) create mode 100644 examples/smiley_interp_tutoprial.json diff --git a/__init__.py b/__init__.py index bed38f8..f562581 100644 --- a/__init__.py +++ b/__init__.py @@ -459,6 +459,9 @@ WEB_DIRECTORY = "./web/js" NODE_DISPLAY_NAME_MAPPINGS = { + "FlexAudioVisualizerCircular": "**BETA** Flex Audio Visualizer Circular", + "FlexAudioVisualizerLine": "**BETA** Flex Audio Visualizer Line", + "FlexVideoSpeed": "**BETA** Flex Video Speed", "FlexVideoDirection": "Flex Video Direction", "FlexVideoFrameBlend": "**BETA**Flex Video Frame Blend", diff --git a/examples/smiley_interp_tutoprial.json b/examples/smiley_interp_tutoprial.json new file mode 100644 index 0000000..abc7c72 --- /dev/null +++ b/examples/smiley_interp_tutoprial.json @@ -0,0 +1,7351 @@ +{ + "last_node_id": 201, + "last_link_id": 315, + "nodes": [ + { + "id": 18, + "type": "ADE_AnimateDiffLoaderGen1", + "pos": [ + 5267.175142726612, + 1352.8338497623226 + ], + "size": { + "0": 271.7644958496094, + "1": 242 + }, + "flags": {}, + "order": 69, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 20 + }, + { + "name": "context_options", + "type": "CONTEXT_OPTIONS", + "link": 21, + "slot_index": 1 + }, + { + "name": "motion_lora", + "type": "MOTION_LORA", + "link": 22, + "slot_index": 2 + }, + { + "name": "ad_settings", + "type": "AD_SETTINGS", + "link": null, + "slot_index": 3 + }, + { + "name": "ad_keyframes", + "type": "AD_KEYFRAMES", + "link": null + }, + { + "name": "sample_settings", + "type": "SAMPLE_SETTINGS", + "link": 23, + "slot_index": 5 + }, + { + "name": "scale_multival", + "type": "MULTIVAL", + "link": 24, + "slot_index": 6 + }, + { + "name": "effect_multival", + "type": "MULTIVAL", + "link": 25, + "slot_index": 7 + }, + { + "name": "per_block", + "type": "PER_BLOCK", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 27 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ADE_AnimateDiffLoaderGen1", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "ALCM_sd15_t2v_beta.ckpt", + "lcm avg(sqrt_linear,linear)" + ] + }, + { + "id": 19, + "type": "ADE_AnimateDiffSamplingSettings", + "pos": [ + 5267.175142726612, + 1962.8338497623224 + ], + "size": { + "0": 273.3500061035156, + "1": 254 + }, + "flags": {}, + "order": 46, + "mode": 0, + "inputs": [ + { + "name": "noise_layers", + "type": "NOISE_LAYERS", + "link": null, + "slot_index": 0 + }, + { + "name": "iteration_opts", + "type": "ITERATION_OPTS", + "link": null + }, + { + "name": "custom_cfg", + "type": "CUSTOM_CFG", + "link": 26, + "slot_index": 2 + }, + { + "name": "sigma_schedule", + "type": "SIGMA_SCHEDULE", + "link": null, + "slot_index": 3 + }, + { + "name": "seed_override", + "type": "INT", + "link": null, + "widget": { + "name": "seed_override" + } + }, + { + "name": "seed_override", + "type": "INT", + "link": null, + "widget": { + "name": "seed_override" + } + } + ], + "outputs": [ + { + "name": "settings", + "type": "SAMPLE_SETTINGS", + "links": [ + 23 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ADE_AnimateDiffSamplingSettings", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 0, + "FreeNoise", + "comfy", + 0, + 0, + false, + "" + ] + }, + { + "id": 20, + "type": "ADE_MultivalDynamic", + "pos": [ + 4917.175142726612, + 1762.8338497623226 + ], + "size": { + "0": 259.9388122558594, + "1": 63.332008361816406 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "name": "mask_optional", + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "MULTIVAL", + "type": "MULTIVAL", + "links": [ + 24 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "Scale 🎭🅐🅓", + "properties": { + "Node name for S&R": "ADE_MultivalDynamic", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 1.1400000000000001, + "" + ] + }, + { + "id": 21, + "type": "ADE_AnimateDiffUniformContextOptions", + "pos": [ + 5267.175142726612, + 1652.8338497623226 + ], + "size": { + "0": 273.269775390625, + "1": 270 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "prev_context", + "type": "CONTEXT_OPTIONS", + "link": null + }, + { + "name": "view_opts", + "type": "VIEW_OPTS", + "link": null + } + ], + "outputs": [ + { + "name": "CONTEXT_OPTS", + "type": "CONTEXT_OPTIONS", + "links": [ + 21 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "Context Options 🎭🅐🅓", + "properties": { + "Node name for S&R": "ADE_AnimateDiffUniformContextOptions", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 16, + 1, + 4, + "uniform", + false, + "pyramid", + false, + 0, + 1, + "" + ] + }, + { + "id": 22, + "type": "ADE_MultivalDynamic", + "pos": [ + 4917.175142726612, + 1652.8338497623226 + ], + "size": { + "0": 265.1632385253906, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "mask_optional", + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "MULTIVAL", + "type": "MULTIVAL", + "links": [ + 25 + ], + "shape": 3 + } + ], + "title": "Effect 🎭🅐🅓", + "properties": { + "Node name for S&R": "ADE_MultivalDynamic", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 1.1, + "" + ] + }, + { + "id": 23, + "type": "ADE_CustomCFGSimple", + "pos": [ + 4917.175142726612, + 1862.8338497623226 + ], + "size": { + "0": 257.2469787597656, + "1": 60.893348693847656 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "cfg_extras", + "type": "CFG_EXTRAS", + "link": null + } + ], + "outputs": [ + { + "name": "CUSTOM_CFG", + "type": "CUSTOM_CFG", + "links": [ + 26 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ADE_CustomCFGSimple", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 2, + "" + ] + }, + { + "id": 24, + "type": "ADE_AdjustPESweetspotStretch", + "pos": [ + 4917.175142726612, + 1962.8338497623224 + ], + "size": { + "0": 253.07310485839844, + "1": 106 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "prev_pe_adjust", + "type": "PE_ADJUST", + "link": null + } + ], + "outputs": [ + { + "name": "PE_ADJUST", + "type": "PE_ADJUST", + "links": [], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ADE_AdjustPESweetspotStretch", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 16, + 18, + false, + "" + ] + }, + { + "id": 25, + "type": "ADE_AdjustWeightAllMult", + "pos": [ + 4907.175142726612, + 1492.8338497623226 + ], + "size": { + "0": 270.3999938964844, + "1": 82 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "prev_weight_adjust", + "type": "WEIGHT_ADJUST", + "link": null + } + ], + "outputs": [ + { + "name": "WEIGHT_ADJUST", + "type": "WEIGHT_ADJUST", + "links": [], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ADE_AdjustWeightAllMult", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 1.01, + false, + "" + ] + }, + { + "id": 26, + "type": "ADE_AnimateDiffLoRALoader", + "pos": [ + 4907.175142726612, + 1352.8338497623226 + ], + "size": { + "0": 261.19134521484375, + "1": 82 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "prev_motion_lora", + "type": "MOTION_LORA", + "link": null + } + ], + "outputs": [ + { + "name": "MOTION_LORA", + "type": "MOTION_LORA", + "links": [ + 22 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "AnimateDiff LoRA", + "properties": { + "Node name for S&R": "ADE_AnimateDiffLoRALoader", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "LiquidAF-0-1.safetensors", + 0.8, + "" + ] + }, + { + "id": 36, + "type": "Note", + "pos": [ + 2650, + 1790 + ], + "size": { + "0": 380.91632080078125, + "1": 230.8916015625 + }, + "flags": {}, + "order": 7, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "5. finally we do all this " + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 64, + "type": "IPAdapterClipVisionEnhancerBatch", + "pos": [ + 6169.688715034702, + 1321.1038500857544 + ], + "size": { + "0": 316.3516845703125, + "1": 326 + }, + "flags": {}, + "order": 96, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 86 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 87 + }, + { + "name": "image", + "type": "IMAGE", + "link": 290 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "attn_mask", + "type": "MASK", + "link": 289 + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 90, + 91 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "IPAdapterClipVisionEnhancerBatch" + }, + "widgets_values": [ + 1, + "linear", + 0, + 1, + "V only", + 2, + 0.5, + 0 + ] + }, + { + "id": 16, + "type": "VHS_VideoCombine", + "pos": [ + 7371.0057215536435, + 1315.6671376136148 + ], + "size": [ + 210, + 439 + ], + "flags": {}, + "order": 114, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 16 + }, + { + "name": "audio", + "type": "*", + "link": 49 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 30, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_07445-audio.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 30 + } + } + } + }, + { + "id": 101, + "type": "SetNode", + "pos": { + "0": -3236.29248046875, + "1": 1459.73095703125, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 62, + "mode": 0, + "inputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "link": 162 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_emptyimg", + "properties": { + "previousName": "emptyimg" + }, + "widgets_values": [ + "emptyimg" + ] + }, + { + "id": 13, + "type": "CLIPTextEncode", + "pos": [ + 3110, + 1720 + ], + "size": { + "0": 425.27801513671875, + "1": 180.6060791015625 + }, + "flags": {}, + "order": 71, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 13 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 182 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "text, watermark" + ] + }, + { + "id": 12, + "type": "CLIPTextEncode", + "pos": [ + 3100, + 1470 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 70, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 12 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 181 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "ral-acidzlime, spooky dripping slime" + ] + }, + { + "id": 111, + "type": "CR LoRA Stack", + "pos": [ + 1880, + 1682 + ], + "size": { + "0": 315, + "1": 342 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "lora_stack", + "type": "LORA_STACK", + "link": null + } + ], + "outputs": [ + { + "name": "LORA_STACK", + "type": "LORA_STACK", + "links": [ + 184 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "show_help", + "type": "STRING", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CR LoRA Stack" + }, + "widgets_values": [ + "Off", + "halloweenTech-202.safetensors", + 0.58, + 0.46, + "Off", + "add_detail.safetensors", + 1, + 1, + "Off", + "ral-acidzlime-sd15.safetensors", + 1, + 1 + ] + }, + { + "id": 114, + "type": "CLIPTextEncode", + "pos": [ + 2480, + 3750 + ], + "size": { + "0": 425.27801513671875, + "1": 180.6060791015625 + }, + "flags": {}, + "order": 68, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 192 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 217 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "text, watermark" + ] + }, + { + "id": 119, + "type": "ADE_AnimateDiffSamplingSettings", + "pos": [ + 4524.482187121224, + 3989.7472017283962 + ], + "size": { + "0": 273.3500061035156, + "1": 254 + }, + "flags": {}, + "order": 48, + "mode": 0, + "inputs": [ + { + "name": "noise_layers", + "type": "NOISE_LAYERS", + "link": null, + "slot_index": 0 + }, + { + "name": "iteration_opts", + "type": "ITERATION_OPTS", + "link": null + }, + { + "name": "custom_cfg", + "type": "CUSTOM_CFG", + "link": 205, + "slot_index": 2 + }, + { + "name": "sigma_schedule", + "type": "SIGMA_SCHEDULE", + "link": null, + "slot_index": 3 + }, + { + "name": "seed_override", + "type": "INT", + "link": null, + "widget": { + "name": "seed_override" + } + }, + { + "name": "seed_override", + "type": "INT", + "link": null, + "widget": { + "name": "seed_override" + } + } + ], + "outputs": [ + { + "name": "settings", + "type": "SAMPLE_SETTINGS", + "links": [ + 202 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ADE_AnimateDiffSamplingSettings", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 0, + "FreeNoise", + "comfy", + 0, + 0, + false, + "" + ] + }, + { + "id": 120, + "type": "ADE_MultivalDynamic", + "pos": [ + 4174.482187121224, + 3789.7472017283962 + ], + "size": { + "0": 259.9388122558594, + "1": 63.332008361816406 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "mask_optional", + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "MULTIVAL", + "type": "MULTIVAL", + "links": [ + 203 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "Scale 🎭🅐🅓", + "properties": { + "Node name for S&R": "ADE_MultivalDynamic", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 1.1400000000000001, + "" + ] + }, + { + "id": 121, + "type": "ADE_AnimateDiffUniformContextOptions", + "pos": [ + 4524.482187121224, + 3679.7472017283962 + ], + "size": { + "0": 273.269775390625, + "1": 270 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "prev_context", + "type": "CONTEXT_OPTIONS", + "link": null + }, + { + "name": "view_opts", + "type": "VIEW_OPTS", + "link": null + } + ], + "outputs": [ + { + "name": "CONTEXT_OPTS", + "type": "CONTEXT_OPTIONS", + "links": [ + 200 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "Context Options 🎭🅐🅓", + "properties": { + "Node name for S&R": "ADE_AnimateDiffUniformContextOptions", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 16, + 1, + 4, + "uniform", + false, + "pyramid", + false, + 0, + 1, + "" + ] + }, + { + "id": 122, + "type": "ADE_MultivalDynamic", + "pos": [ + 4174.482187121224, + 3679.7472017283962 + ], + "size": { + "0": 265.1632385253906, + "1": 58 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "mask_optional", + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "MULTIVAL", + "type": "MULTIVAL", + "links": [ + 204 + ], + "shape": 3 + } + ], + "title": "Effect 🎭🅐🅓", + "properties": { + "Node name for S&R": "ADE_MultivalDynamic", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 1.1, + "" + ] + }, + { + "id": 123, + "type": "ADE_CustomCFGSimple", + "pos": [ + 4174.482187121224, + 3889.7472017283962 + ], + "size": { + "0": 257.2469787597656, + "1": 60.893348693847656 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "cfg_extras", + "type": "CFG_EXTRAS", + "link": null + } + ], + "outputs": [ + { + "name": "CUSTOM_CFG", + "type": "CUSTOM_CFG", + "links": [ + 205 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ADE_CustomCFGSimple", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 2, + "" + ] + }, + { + "id": 124, + "type": "ADE_AdjustPESweetspotStretch", + "pos": [ + 4174.482187121224, + 3989.7472017283962 + ], + "size": { + "0": 253.07310485839844, + "1": 106 + }, + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "prev_pe_adjust", + "type": "PE_ADJUST", + "link": null + } + ], + "outputs": [ + { + "name": "PE_ADJUST", + "type": "PE_ADJUST", + "links": [], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ADE_AdjustPESweetspotStretch", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 16, + 18, + false, + "" + ] + }, + { + "id": 125, + "type": "ADE_AdjustWeightAllMult", + "pos": [ + 4164.482187121224, + 3519.7472017283962 + ], + "size": { + "0": 270.3999938964844, + "1": 82 + }, + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "prev_weight_adjust", + "type": "WEIGHT_ADJUST", + "link": null + } + ], + "outputs": [ + { + "name": "WEIGHT_ADJUST", + "type": "WEIGHT_ADJUST", + "links": [], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ADE_AdjustWeightAllMult", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 1.01, + false, + "" + ] + }, + { + "id": 126, + "type": "ADE_AnimateDiffLoRALoader", + "pos": [ + 4164.482187121224, + 3379.7472017283962 + ], + "size": { + "0": 261.19134521484375, + "1": 82 + }, + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "prev_motion_lora", + "type": "MOTION_LORA", + "link": null + } + ], + "outputs": [ + { + "name": "MOTION_LORA", + "type": "MOTION_LORA", + "links": [ + 201 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "AnimateDiff LoRA", + "properties": { + "Node name for S&R": "ADE_AnimateDiffLoRALoader", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "LiquidAF-0-1.safetensors", + 0.8, + "" + ] + }, + { + "id": 128, + "type": "CR Apply LoRA Stack", + "pos": [ + 2060, + 3620 + ], + "size": { + "0": 254.40000915527344, + "1": 66 + }, + "flags": {}, + "order": 58, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 295 + }, + { + "name": "clip", + "type": "CLIP", + "link": 296 + }, + { + "name": "lora_stack", + "type": "LORA_STACK", + "link": 315 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 199 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 191, + 192 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "show_help", + "type": "STRING", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CR Apply LoRA Stack" + } + }, + { + "id": 113, + "type": "CLIPTextEncode", + "pos": [ + 2470, + 3500 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 67, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 191 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 216 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "ral-acidzlime, spooky dripping slime" + ] + }, + { + "id": 142, + "type": "AudioFeatureExtractor", + "pos": [ + -3373.123323701926, + 1782.541571086188 + ], + "size": { + "0": 361.20001220703125, + "1": 78 + }, + "flags": {}, + "order": 74, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 238 + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "link": 239 + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": [ + 240 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "AudioFeatureExtractor" + }, + "widgets_values": [ + "amplitude_envelope" + ] + }, + { + "id": 143, + "type": "SetNode", + "pos": { + "0": -2988.001220703125, + "1": 1805.5142822265625, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 86, + "mode": 0, + "inputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "link": 240 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_vocal_feature", + "properties": { + "previousName": "vocal_feature" + }, + "widgets_values": [ + "vocal_feature" + ] + }, + { + "id": 63, + "type": "VHS_VideoCombine", + "pos": [ + 11303, + 2172 + ], + "size": [ + 210, + 439 + ], + "flags": {}, + "order": 121, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 251 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 314 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 30, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_07447-audio.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 30 + } + } + } + }, + { + "id": 152, + "type": "AudioFeatureExtractor", + "pos": [ + -3358.363458806817, + 2010.0002441406245 + ], + "size": { + "0": 361.20001220703125, + "1": 78 + }, + "flags": {}, + "order": 75, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 257 + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "link": 258 + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": [ + 259 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "AudioFeatureExtractor" + }, + "widgets_values": [ + "rms_energy" + ] + }, + { + "id": 153, + "type": "SetNode", + "pos": { + "0": -2978.363525390625, + "1": 2040.00048828125, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 87, + "mode": 0, + "inputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "link": 259 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_vocal_feature_rms", + "properties": { + "previousName": "vocal_feature_rms" + }, + "widgets_values": [ + "vocal_feature_rms" + ] + }, + { + "id": 154, + "type": "AudioFeatureExtractor", + "pos": [ + -3318.363458806817, + 2210.0002441406245 + ], + "size": { + "0": 361.20001220703125, + "1": 78 + }, + "flags": {}, + "order": 76, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 260 + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "link": 261 + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": [ + 262 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "AudioFeatureExtractor" + }, + "widgets_values": [ + "spectral_centroid" + ] + }, + { + "id": 155, + "type": "SetNode", + "pos": { + "0": -2938.363525390625, + "1": 2240.0009765625, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 252, + "1": 58 + }, + "flags": {}, + "order": 88, + "mode": 0, + "inputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "link": 262 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_vocal_feature_rms_centroid", + "properties": { + "previousName": "vocal_feature_rms_centroid" + }, + "widgets_values": [ + "vocal_feature_rms_centroid" + ] + }, + { + "id": 156, + "type": "AudioFeatureExtractor", + "pos": [ + -3278.363458806817, + 2410.0002441406245 + ], + "size": { + "0": 361.20001220703125, + "1": 78 + }, + "flags": {}, + "order": 77, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 263 + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "link": 264 + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": [ + 265 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "AudioFeatureExtractor" + }, + "widgets_values": [ + "onset_strength" + ] + }, + { + "id": 157, + "type": "SetNode", + "pos": { + "0": -2898.363525390625, + "1": 2440.000732421875, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 252, + "1": 58 + }, + "flags": {}, + "order": 89, + "mode": 0, + "inputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "link": 265 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_vocal_feature_onsetr", + "properties": { + "previousName": "vocal_feature_onsetr" + }, + "widgets_values": [ + "vocal_feature_onsetr" + ] + }, + { + "id": 158, + "type": "AudioFeatureExtractor", + "pos": [ + -3308.3634588068176, + 2610.000244140625 + ], + "size": { + "0": 361.20001220703125, + "1": 78 + }, + "flags": {}, + "order": 78, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 266 + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "link": 267 + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": [ + 268 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "AudioFeatureExtractor" + }, + "widgets_values": [ + "chroma_features" + ] + }, + { + "id": 159, + "type": "SetNode", + "pos": { + "0": -2928.363525390625, + "1": 2640.00048828125, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 252, + "1": 58 + }, + "flags": {}, + "order": 90, + "mode": 0, + "inputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "link": 268 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_vocal_feature_rms_chroma", + "properties": { + "previousName": "vocal_feature_rms_chroma" + }, + "widgets_values": [ + "vocal_feature_rms_chroma" + ] + }, + { + "id": 41, + "type": "DownloadOpenUnmixModel", + "pos": [ + -4547.363458806818, + 2147.0002441406245 + ], + "size": { + "0": 361.20001220703125, + "1": 58 + }, + "flags": {}, + "order": 16, + "mode": 0, + "outputs": [ + { + "name": "OPEN_UNMIX_MODEL", + "type": "OPEN_UNMIX_MODEL", + "links": [ + 36 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DownloadOpenUnmixModel" + }, + "widgets_values": [ + "umxl" + ] + }, + { + "id": 49, + "type": "PreviewAudio", + "pos": [ + -4244.363458806818, + 1616.0002441406245 + ], + "size": { + "0": 315, + "1": 76 + }, + "flags": {}, + "order": 55, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 56 + } + ], + "properties": { + "Node name for S&R": "PreviewAudio" + }, + "widgets_values": [ + null + ] + }, + { + "id": 45, + "type": "SetNode", + "pos": { + "0": -4216.36328125, + "1": 1472.00048828125, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 54, + "mode": 0, + "inputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "link": 48 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_audio", + "properties": { + "previousName": "audio" + }, + "widgets_values": [ + "audio" + ] + }, + { + "id": 50, + "type": "PreviewAudio", + "pos": [ + -3355.363458806817, + 1601.0002441406248 + ], + "size": { + "0": 315, + "1": 76 + }, + "flags": {}, + "order": 72, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 57 + } + ], + "properties": { + "Node name for S&R": "PreviewAudio" + }, + "widgets_values": [ + null + ] + }, + { + "id": 167, + "type": "SetNode", + "pos": { + "0": 1137, + "1": 1802, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 92, + "mode": 0, + "inputs": [ + { + "name": "MASK", + "type": "MASK", + "link": 287 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_mask_ball", + "properties": { + "previousName": "mask_ball" + }, + "widgets_values": [ + "mask_ball" + ] + }, + { + "id": 170, + "type": "SetNode", + "pos": { + "0": 1115.43310546875, + "1": 2059.093505859375, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 93, + "mode": 0, + "inputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "link": 288 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_img_ball", + "properties": { + "previousName": "img_ball" + }, + "widgets_values": [ + "img_ball" + ] + }, + { + "id": 173, + "type": "SetNode", + "pos": { + "0": 2083, + "1": 1200, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 52, + "mode": 0, + "inputs": [ + { + "name": "VAE", + "type": "VAE", + "link": 291 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_vae", + "properties": { + "previousName": "vae" + }, + "widgets_values": [ + "vae" + ] + }, + { + "id": 174, + "type": "SetNode", + "pos": { + "0": 2053, + "1": 1068, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 50, + "mode": 0, + "inputs": [ + { + "name": "MODEL", + "type": "MODEL", + "link": 292 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_model", + "properties": { + "previousName": "model" + }, + "widgets_values": [ + "model" + ] + }, + { + "id": 175, + "type": "SetNode", + "pos": { + "0": 2037, + "1": 967, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 51, + "mode": 0, + "inputs": [ + { + "name": "CLIP", + "type": "CLIP", + "link": 293 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_clip", + "properties": { + "previousName": "clip" + }, + "widgets_values": [ + "clip" + ] + }, + { + "id": 30, + "type": "CR Apply LoRA Stack", + "pos": [ + 2528, + 1486 + ], + "size": { + "0": 254.40000915527344, + "1": 66 + }, + "flags": {}, + "order": 59, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 28 + }, + { + "name": "clip", + "type": "CLIP", + "link": 29 + }, + { + "name": "lora_stack", + "type": "LORA_STACK", + "link": 30 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 20 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 12, + 13 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "show_help", + "type": "STRING", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CR Apply LoRA Stack" + } + }, + { + "id": 177, + "type": "GetNode", + "pos": { + "0": 1790, + "1": 3620, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 295 + ], + "slot_index": 0 + } + ], + "title": "Get_model", + "properties": {}, + "widgets_values": [ + "model" + ] + }, + { + "id": 29, + "type": "CR LoRA Stack", + "pos": [ + 2270, + 1670 + ], + "size": { + "0": 315, + "1": 342 + }, + "flags": {}, + "order": 47, + "mode": 0, + "inputs": [ + { + "name": "lora_stack", + "type": "LORA_STACK", + "link": 184 + } + ], + "outputs": [ + { + "name": "LORA_STACK", + "type": "LORA_STACK", + "links": [ + 30, + 297 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "show_help", + "type": "STRING", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CR LoRA Stack" + }, + "widgets_values": [ + "On", + "AnimateLCM_sd15_t2v_lora.safetensors", + 1, + 1, + "On", + "add_detail.safetensors", + 1, + 1, + "On", + "ral-acidzlime-sd15.safetensors", + 1, + 1 + ] + }, + { + "id": 178, + "type": "GetNode", + "pos": { + "0": 1750, + "1": 3750, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 296 + ], + "slot_index": 0 + } + ], + "title": "Get_clip", + "properties": {}, + "widgets_values": [ + "clip" + ] + }, + { + "id": 180, + "type": "GetNode", + "pos": { + "0": 1830, + "1": 3890, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "LORA_STACK", + "type": "LORA_STACK", + "links": null + } + ], + "title": "Get_lora_stack", + "properties": {}, + "widgets_values": [ + "lora_stack" + ] + }, + { + "id": 176, + "type": "GetNode", + "pos": { + "0": 1800, + "1": 3550, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 294 + ], + "slot_index": 0 + } + ], + "title": "Get_vae", + "properties": {}, + "widgets_values": [ + "vae" + ] + }, + { + "id": 184, + "type": "SetNode", + "pos": { + "0": 1219.13427734375, + "1": 1531.6217041015625, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 103, + "mode": 0, + "inputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "link": 299 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_freq_ball_img", + "properties": { + "previousName": "freq_ball_img" + }, + "widgets_values": [ + "freq_ball_img" + ] + }, + { + "id": 146, + "type": "VAEDecode", + "pos": [ + 8563.564381014008, + 2168.304049340784 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 115, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 249 + }, + { + "name": "vae", + "type": "VAE", + "link": 303 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 247 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 189, + "type": "GetNode", + "pos": { + "0": 8553.5625, + "1": 2048.30419921875, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 303 + ], + "slot_index": 0 + } + ], + "title": "Get_vae", + "properties": {}, + "widgets_values": [ + "vae" + ] + }, + { + "id": 191, + "type": "SetNode", + "pos": { + "0": -3664.36328125, + "1": 2473.00048828125, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 79, + "mode": 0, + "inputs": [ + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "link": 305 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_feature_pipe", + "properties": { + "previousName": "feature_pipe" + }, + "widgets_values": [ + "feature_pipe" + ] + }, + { + "id": 27, + "type": "IPAdapterUnifiedLoader", + "pos": [ + 5784.337867433583, + 1308.1993598917295 + ], + "size": { + "0": 315, + "1": 78 + }, + "flags": {}, + "order": 83, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 27 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": null + } + ], + "outputs": [ + { + "name": "model", + "type": "MODEL", + "links": [ + 86 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "links": [ + 87 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "IPAdapterUnifiedLoader" + }, + "widgets_values": [ + "PLUS (high strength)" + ] + }, + { + "id": 193, + "type": "GetNode", + "pos": { + "0": 9919, + "1": 2470, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 308 + ], + "slot_index": 0 + } + ], + "title": "Get_vae", + "properties": {}, + "widgets_values": [ + "vae" + ] + }, + { + "id": 14, + "type": "VAEDecode", + "pos": [ + 7055, + 1328 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 112, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 14 + }, + { + "name": "vae", + "type": "VAE", + "link": 309 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 16 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 118, + "type": "ADE_AnimateDiffLoaderGen1", + "pos": [ + 4524.482187121224, + 3379.7472017283962 + ], + "size": { + "0": 271.7644958496094, + "1": 242 + }, + "flags": {}, + "order": 66, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 199 + }, + { + "name": "context_options", + "type": "CONTEXT_OPTIONS", + "link": 200, + "slot_index": 1 + }, + { + "name": "motion_lora", + "type": "MOTION_LORA", + "link": 201, + "slot_index": 2 + }, + { + "name": "ad_settings", + "type": "AD_SETTINGS", + "link": null, + "slot_index": 3 + }, + { + "name": "ad_keyframes", + "type": "AD_KEYFRAMES", + "link": null + }, + { + "name": "sample_settings", + "type": "SAMPLE_SETTINGS", + "link": 202, + "slot_index": 5 + }, + { + "name": "scale_multival", + "type": "MULTIVAL", + "link": 203, + "slot_index": 6 + }, + { + "name": "effect_multival", + "type": "MULTIVAL", + "link": 204, + "slot_index": 7 + }, + { + "name": "per_block", + "type": "PER_BLOCK", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 206 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ADE_AnimateDiffLoaderGen1", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "ALCM_sd15_t2v_beta.ckpt", + "lcm avg(sqrt_linear,linear)" + ] + }, + { + "id": 196, + "type": "GetNode", + "pos": { + "0": 5029.1181640625, + "1": 3590.9638671875, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 310 + ], + "slot_index": 0 + } + ], + "title": "Get_mask_ball", + "properties": {}, + "widgets_values": [ + "mask_ball" + ] + }, + { + "id": 195, + "type": "GetNode", + "pos": { + "0": 5033.1220703125, + "1": 3449.7607421875, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 311 + ], + "slot_index": 0 + } + ], + "title": "Get_img_ball", + "properties": {}, + "widgets_values": [ + "img_ball" + ] + }, + { + "id": 185, + "type": "GetNode", + "pos": { + "0": 3071, + "1": 3300, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 300 + ], + "slot_index": 0 + } + ], + "title": "Get_freq_ball_img", + "properties": {}, + "widgets_values": [ + "freq_ball_img" + ] + }, + { + "id": 134, + "type": "LineArtPreprocessor", + "pos": [ + 3301, + 3338 + ], + "size": { + "0": 315, + "1": 82 + }, + "flags": {}, + "order": 49, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 300 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 219 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LineArtPreprocessor" + }, + "widgets_values": [ + "disable", + 512 + ] + }, + { + "id": 187, + "type": "GetNode", + "pos": { + "0": 3387, + "1": 3678, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 301 + ], + "slot_index": 0 + } + ], + "title": "Get_mask_ball", + "properties": {}, + "widgets_values": [ + "mask_ball" + ] + }, + { + "id": 197, + "type": "GetNode", + "pos": { + "0": 6711.1376953125, + "1": 3278.64892578125, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 312 + ], + "slot_index": 0 + } + ], + "title": "Get_audio", + "properties": {}, + "widgets_values": [ + "audio" + ] + }, + { + "id": 188, + "type": "GetNode", + "pos": { + "0": 6070.1376953125, + "1": 3306.64892578125, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 302 + ], + "slot_index": 0 + } + ], + "title": "Get_vae", + "properties": {}, + "widgets_values": [ + "vae" + ] + }, + { + "id": 46, + "type": "GetNode", + "pos": { + "0": 6608, + "1": 2041, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 49 + ], + "slot_index": 0 + } + ], + "title": "Get_audio", + "properties": {}, + "widgets_values": [ + "audio" + ] + }, + { + "id": 198, + "type": "GetNode", + "pos": { + "0": 8591.5625, + "1": 2391.30419921875, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 313 + ], + "slot_index": 0 + } + ], + "title": "Get_audio", + "properties": {}, + "widgets_values": [ + "audio" + ] + }, + { + "id": 199, + "type": "GetNode", + "pos": { + "0": 10869, + "1": 2147, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 314 + ], + "slot_index": 0 + } + ], + "title": "Get_audio", + "properties": {}, + "widgets_values": [ + "audio" + ] + }, + { + "id": 62, + "type": "VAEDecode", + "pos": [ + 10590, + 2555 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 119, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 80 + }, + { + "name": "vae", + "type": "VAE", + "link": 308 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 250 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 78, + "type": "GetNode", + "pos": { + "0": -1442, + "1": 2188, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 124 + ], + "slot_index": 0 + } + ], + "title": "Get_audio", + "properties": {}, + "widgets_values": [ + "audio" + ] + }, + { + "id": 192, + "type": "GetNode", + "pos": { + "0": 571, + "1": 1275, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 307 + ], + "slot_index": 0 + } + ], + "title": "Get_audio", + "properties": {}, + "widgets_values": [ + "audio" + ] + }, + { + "id": 11, + "type": "CheckpointLoaderSimple", + "pos": [ + 1825, + 1488 + ], + "size": { + "0": 315, + "1": 98 + }, + "flags": {}, + "order": 34, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 28, + 292 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 29, + 293 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 15, + 291 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "photon_v1.safetensors" + ] + }, + { + "id": 15, + "type": "VAEEncode", + "pos": [ + 2552, + 1399 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 102, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 186 + }, + { + "name": "vae", + "type": "VAE", + "link": 15 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 63 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEEncode" + } + }, + { + "id": 172, + "type": "GetNode", + "pos": { + "0": 5806.3388671875, + "1": 1436.19921875, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 35, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 290 + ], + "slot_index": 0 + } + ], + "title": "Get_img_ball", + "properties": {}, + "widgets_values": [ + "img_ball" + ] + }, + { + "id": 194, + "type": "GetNode", + "pos": { + "0": 6811, + "1": 1218, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 36, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 309 + ], + "slot_index": 0 + } + ], + "title": "Get_vae", + "properties": {}, + "widgets_values": [ + "vae" + ] + }, + { + "id": 133, + "type": "ControlNetLoader", + "pos": [ + 3074, + 3529 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 37, + "mode": 0, + "outputs": [ + { + "name": "CONTROL_NET", + "type": "CONTROL_NET", + "links": [ + 218 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ControlNetLoader" + }, + "widgets_values": [ + "control_v11f1p_sd15_depth.pth" + ] + }, + { + "id": 127, + "type": "IPAdapterUnifiedLoader", + "pos": [ + 5024, + 3322 + ], + "size": { + "0": 315, + "1": 78 + }, + "flags": {}, + "order": 81, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 206 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": null + } + ], + "outputs": [ + { + "name": "model", + "type": "MODEL", + "links": [ + 212 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "links": [ + 213 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "IPAdapterUnifiedLoader" + }, + "widgets_values": [ + "PLUS (high strength)" + ] + }, + { + "id": 115, + "type": "VAEDecode", + "pos": [ + 6449, + 3520 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 108, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 193 + }, + { + "name": "vae", + "type": "VAE", + "link": 302 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 197 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 129, + "type": "Apply ControlNet Stack", + "pos": [ + 3741, + 3805 + ], + "size": { + "0": 304.79998779296875, + "1": 66 + }, + "flags": {}, + "order": 95, + "mode": 4, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 210 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 211 + }, + { + "name": "cnet_stack", + "type": "CONTROL_NET_STACK", + "link": null, + "slot_index": 2 + } + ], + "outputs": [ + { + "name": "CONDITIONING+", + "type": "CONDITIONING", + "links": [ + 188 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "CONDITIONING-", + "type": "CONDITIONING", + "links": [ + 189 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "Apply ControlNet Stack", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "shape": 1 + }, + { + "id": 132, + "type": "ACN_AdvancedControlNetApply", + "pos": [ + 3733, + 3444 + ], + "size": { + "0": 285.6000061035156, + "1": 286 + }, + "flags": {}, + "order": 82, + "mode": 0, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 216 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 217 + }, + { + "name": "control_net", + "type": "CONTROL_NET", + "link": 218, + "slot_index": 2 + }, + { + "name": "image", + "type": "IMAGE", + "link": 219 + }, + { + "name": "mask_optional", + "type": "MASK", + "link": 301 + }, + { + "name": "timestep_kf", + "type": "TIMESTEP_KEYFRAME", + "link": null + }, + { + "name": "latent_kf_override", + "type": "LATENT_KEYFRAME", + "link": null + }, + { + "name": "weights_override", + "type": "CONTROL_NET_WEIGHTS", + "link": null + }, + { + "name": "model_optional", + "type": "MODEL", + "link": null + }, + { + "name": "vae_optional", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 210 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "negative", + "type": "CONDITIONING", + "links": [ + 211 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "model_opt", + "type": "MODEL", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ACN_AdvancedControlNetApply" + }, + "widgets_values": [ + 0.5, + 0, + 0.5, + "" + ] + }, + { + "id": 116, + "type": "VAEEncode", + "pos": [ + 2112, + 3372 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 56, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 298 + }, + { + "name": "vae", + "type": "VAE", + "link": 294 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 190 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEEncode" + } + }, + { + "id": 39, + "type": "VHS_LoadAudioUpload", + "pos": [ + -4568, + 1515 + ], + "size": { + "0": 226.8000030517578, + "1": 130 + }, + "flags": {}, + "order": 38, + "mode": 0, + "outputs": [ + { + "name": "audio", + "type": "AUDIO", + "links": [ + 37, + 38, + 48, + 56 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadAudioUpload" + }, + "widgets_values": { + "audio": "Delta Deez - Songfinch - Out the Mud.mp3", + "start_time": 75, + "duration": 15, + "choose audio to upload": "image" + } + }, + { + "id": 40, + "type": "AudioSeparator", + "pos": [ + -3949, + 2099 + ], + "size": { + "0": 317.4000244140625, + "1": 158 + }, + "flags": {}, + "order": 61, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "OPEN_UNMIX_MODEL", + "link": 36, + "slot_index": 0 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 37 + }, + { + "name": "video_frames", + "type": "IMAGE", + "link": 39 + } + ], + "outputs": [ + { + "name": "audio", + "type": "AUDIO", + "links": [ + 57 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "drums_audio", + "type": "AUDIO", + "links": [ + 41, + 283 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "vocals_audio", + "type": "AUDIO", + "links": [ + 238, + 257, + 260, + 263, + 266 + ], + "shape": 3, + "slot_index": 2 + }, + { + "name": "bass_audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "other_audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "links": [ + 42, + 239, + 258, + 261, + 264, + 267, + 305 + ], + "shape": 3, + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "AudioSeparator" + }, + "widgets_values": [ + 30 + ] + }, + { + "id": 43, + "type": "AudioFeatureExtractor", + "pos": [ + -3306, + 2831 + ], + "size": { + "0": 361.20001220703125, + "1": 78 + }, + "flags": {}, + "order": 73, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 41 + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "link": 42 + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": [ + 58, + 110 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "AudioFeatureExtractor" + }, + "widgets_values": [ + "amplitude_envelope" + ] + }, + { + "id": 70, + "type": "FeatureMixer", + "pos": [ + -2861, + 2784 + ], + "size": { + "0": 367.79998779296875, + "1": 342 + }, + "flags": {}, + "order": 85, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 110 + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": [ + 284 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": [ + 111 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "FeatureMixer" + }, + "widgets_values": [ + 1, + 0, + 1, + 1, + 1, + 1, + 1, + 0, + 0.3, + 1, + 0.5, + false + ] + }, + { + "id": 51, + "type": "PreviewFeature", + "pos": [ + -2844, + 3248 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 84, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 58 + } + ], + "outputs": [ + { + "name": "FEATURE_PREVIEW", + "type": "IMAGE", + "links": [ + 59 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PreviewFeature" + }, + "widgets_values": [ + false + ] + }, + { + "id": 52, + "type": "PreviewImage", + "pos": [ + -2448, + 2947 + ], + "size": { + "0": 210, + "1": 246 + }, + "flags": {}, + "order": 97, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 59 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 71, + "type": "PreviewImage", + "pos": [ + -2189, + 2834 + ], + "size": { + "0": 210, + "1": 246 + }, + "flags": {}, + "order": 99, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 111 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 166, + "type": "FlexAudioVisualizerCircular", + "pos": [ + -2046, + 2063 + ], + "size": { + "0": 403.20001220703125, + "1": 558 + }, + "flags": {}, + "order": 98, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 283 + }, + { + "name": "opt_feature", + "type": "FEATURE", + "link": 284 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 285, + 286 + ], + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FlexAudioVisualizerCircular" + }, + "widgets_values": [ + 30, + 768, + 464, + 1, + "max_frequency", + "relative", + "None", + 0.5, + 0.5, + "bar", + "frequency", + 0.5, + 90, + 360, + 2048, + 20, + 8000, + 150, + 2, + 100, + 200 + ] + }, + { + "id": 42, + "type": "EmptyImageAndMaskFromAudio", + "pos": [ + -4410, + 1817 + ], + "size": { + "0": 411.6000061035156, + "1": 146 + }, + "flags": {}, + "order": 53, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 38 + } + ], + "outputs": [ + { + "name": "empty_image", + "type": "IMAGE", + "links": [ + 39, + 162 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "empty_mask", + "type": "MASK", + "links": null, + "shape": 3 + }, + { + "name": "frame_count", + "type": "INT", + "links": [ + 66, + 67 + ], + "shape": 3, + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "EmptyImageAndMaskFromAudio" + }, + "widgets_values": [ + 30, + 768, + 464 + ] + }, + { + "id": 57, + "type": "VHS_LoadVideo", + "pos": [ + 245, + 1805 + ], + "size": [ + 235.1999969482422, + 381.45624816417694 + ], + "flags": {}, + "order": 64, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + }, + { + "name": "frame_load_cap", + "type": "INT", + "link": 67, + "widget": { + "name": "frame_load_cap" + } + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 68 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "floating_in_pool_768_ball_MASK.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "floating_in_pool_768_ball_MASK.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 58, + "type": "_mfc", + "pos": [ + 533, + 1768 + ], + "size": { + "0": 315, + "1": 150 + }, + "flags": {}, + "order": 80, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 68 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 69, + 100, + 287 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 288 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "_mfc" + }, + "widgets_values": [ + 255, + 255, + 255, + 11 + ] + }, + { + "id": 55, + "type": "ImageCompositeMasked", + "pos": [ + 876, + 1365 + ], + "size": { + "0": 315, + "1": 146 + }, + "flags": {}, + "order": 91, + "mode": 0, + "inputs": [ + { + "name": "destination", + "type": "IMAGE", + "link": 183 + }, + { + "name": "source", + "type": "IMAGE", + "link": 65 + }, + { + "name": "mask", + "type": "MASK", + "link": 69 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 73, + 99, + 186, + 299 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ImageCompositeMasked" + }, + "widgets_values": [ + 15, + 0, + false + ] + }, + { + "id": 44, + "type": "SetNode", + "pos": { + "0": -1559, + "1": 1886, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 106, + "mode": 0, + "inputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "link": 286 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": [], + "slot_index": 0 + } + ], + "title": "Set_freq", + "properties": { + "previousName": "freq" + }, + "widgets_values": [ + "freq" + ] + }, + { + "id": 110, + "type": "GetNode", + "pos": { + "0": 642, + "1": 1385, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 39, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 183 + ], + "slot_index": 0 + } + ], + "title": "Get_freq", + "properties": {}, + "widgets_values": [ + "freq" + ] + }, + { + "id": 54, + "type": "VHS_LoadVideo", + "pos": [ + 263, + 1313 + ], + "size": [ + 235.1999969482422, + 381.45624816417694 + ], + "flags": {}, + "order": 63, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + }, + { + "name": "frame_load_cap", + "type": "INT", + "link": 66, + "widget": { + "name": "frame_load_cap" + } + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 65 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "floating_in_pool_768.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "floating_in_pool_768.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 179, + "type": "SetNode", + "pos": { + "0": 2697, + "1": 1636, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 60, + "mode": 0, + "inputs": [ + { + "name": "LORA_STACK", + "type": "LORA_STACK", + "link": 297 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_lora_stack", + "properties": { + "previousName": "lora_stack" + }, + "widgets_values": [ + "lora_stack" + ] + }, + { + "id": 65, + "type": "ACN_AdvancedControlNetApply", + "pos": [ + 4358, + 1400 + ], + "size": { + "0": 285.6000061035156, + "1": 286 + }, + "flags": {}, + "order": 107, + "mode": 0, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 181 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 182 + }, + { + "name": "control_net", + "type": "CONTROL_NET", + "link": 98, + "slot_index": 2 + }, + { + "name": "image", + "type": "IMAGE", + "link": 101 + }, + { + "name": "mask_optional", + "type": "MASK", + "link": 100 + }, + { + "name": "timestep_kf", + "type": "TIMESTEP_KEYFRAME", + "link": null + }, + { + "name": "latent_kf_override", + "type": "LATENT_KEYFRAME", + "link": null + }, + { + "name": "weights_override", + "type": "CONTROL_NET_WEIGHTS", + "link": null + }, + { + "name": "model_optional", + "type": "MODEL", + "link": null + }, + { + "name": "vae_optional", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 97 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "negative", + "type": "CONDITIONING", + "links": [ + 96 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "model_opt", + "type": "MODEL", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ACN_AdvancedControlNetApply" + }, + "widgets_values": [ + 0.5, + 0, + 0.5, + "" + ] + }, + { + "id": 32, + "type": "Apply ControlNet Stack", + "pos": [ + 4367, + 1753 + ], + "size": { + "0": 304.79998779296875, + "1": 66 + }, + "flags": {}, + "order": 109, + "mode": 0, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 97 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 96 + }, + { + "name": "cnet_stack", + "type": "CONTROL_NET_STACK", + "link": null, + "slot_index": 2 + } + ], + "outputs": [ + { + "name": "CONDITIONING+", + "type": "CONDITIONING", + "links": [ + 46, + 77 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "CONDITIONING-", + "type": "CONDITIONING", + "links": [ + 47, + 78 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "Apply ControlNet Stack", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "shape": 1 + }, + { + "id": 66, + "type": "ControlNetLoader", + "pos": [ + 3884, + 1648 + ], + "size": [ + 355.70568603425454, + 58 + ], + "flags": {}, + "order": 40, + "mode": 0, + "outputs": [ + { + "name": "CONTROL_NET", + "type": "CONTROL_NET", + "links": [ + 98 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ControlNetLoader" + }, + "widgets_values": [ + "control_v11f1p_sd15_depth.pth" + ] + }, + { + "id": 67, + "type": "LineArtPreprocessor", + "pos": [ + 3908, + 1446 + ], + "size": { + "0": 315, + "1": 82 + }, + "flags": {}, + "order": 101, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 99 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 101 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LineArtPreprocessor" + }, + "widgets_values": [ + "disable", + 512 + ] + }, + { + "id": 171, + "type": "GetNode", + "pos": { + "0": 5804, + "1": 1547, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 289 + ], + "slot_index": 0 + } + ], + "title": "Get_mask_ball", + "properties": {}, + "widgets_values": [ + "mask_ball" + ] + }, + { + "id": 183, + "type": "GetNode", + "pos": { + "0": 1729, + "1": 3309, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 42, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 298 + ], + "slot_index": 0 + } + ], + "title": "Get_freq", + "properties": {}, + "widgets_values": [ + "freq" + ] + }, + { + "id": 60, + "type": "VHS_VideoCombine", + "pos": [ + 1533, + 2104 + ], + "size": [ + 210, + 438.296875 + ], + "flags": {}, + "order": 100, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 73 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 307 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 30, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_07438-audio.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 30 + } + } + } + }, + { + "id": 77, + "type": "VHS_VideoCombine", + "pos": [ + -1088, + 2105 + ], + "size": [ + 210, + 439 + ], + "flags": {}, + "order": 105, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 285 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 124 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 30, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_07437-audio.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 30 + } + } + } + }, + { + "id": 131, + "type": "IPAdapterClipVisionEnhancerBatch", + "pos": [ + 5453, + 3366 + ], + "size": { + "0": 316.3516845703125, + "1": 326 + }, + "flags": {}, + "order": 94, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 212 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 213 + }, + { + "name": "image", + "type": "IMAGE", + "link": 311 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "attn_mask", + "type": "MASK", + "link": 310 + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 187 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "IPAdapterClipVisionEnhancerBatch" + }, + "widgets_values": [ + 1, + "linear", + 0, + 1, + "V only", + 2, + 0.5, + 0 + ] + }, + { + "id": 117, + "type": "VHS_VideoCombine", + "pos": [ + 6930, + 3369 + ], + "size": [ + 210, + 439 + ], + "flags": {}, + "order": 110, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 197 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 312 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 30, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_07439-audio.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 30 + } + } + } + }, + { + "id": 112, + "type": "KSampler", + "pos": [ + 6026, + 3421 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 104, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 187 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 188 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 189 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 190 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 193, + 244 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 156680208700286, + "fixed", + 4, + 1, + "lcm", + "sgm_uniform", + 0.6 + ] + }, + { + "id": 10, + "type": "KSampler", + "pos": [ + 6925, + 1507 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 111, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 90 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 46 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 47 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 63 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 14, + 241 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 156680208700286, + "fixed", + 4, + 1, + "lcm", + "sgm_uniform", + 0.6 + ] + }, + { + "id": 190, + "type": "GetNode", + "pos": { + "0": 7841, + "1": 2135, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 43, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": [ + 306 + ], + "slot_index": 0 + } + ], + "title": "Get_feature_pipe", + "properties": {}, + "widgets_values": [ + "feature_pipe" + ] + }, + { + "id": 144, + "type": "GetNode", + "pos": { + "0": 7351, + "1": 2146, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": [ + 328.3037149734573, + 58 + ], + "flags": {}, + "order": 44, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": [ + 252 + ], + "slot_index": 0 + } + ], + "title": "Get_vocal_feature_rms_centroid", + "properties": {}, + "widgets_values": [ + "vocal_feature_rms_centroid" + ] + }, + { + "id": 149, + "type": "FeatureMixer", + "pos": [ + 7509, + 2269 + ], + "size": { + "0": 367.79998779296875, + "1": 342 + }, + "flags": {}, + "order": 57, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 252 + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": [ + 280 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": [ + 269 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "FeatureMixer" + }, + "widgets_values": [ + 1, + 0, + 1, + 1, + 1, + 1, + 1, + 0, + 0.3, + 0, + 5, + false + ] + }, + { + "id": 150, + "type": "PreviewImage", + "pos": [ + 7920, + 2410 + ], + "size": { + "0": 210, + "1": 246 + }, + "flags": {}, + "order": 65, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 269 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 147, + "type": "VHS_VideoCombine", + "pos": [ + 8907, + 2142 + ], + "size": [ + 210, + 439 + ], + "flags": {}, + "order": 117, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 247 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 313 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 30, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_07446-audio.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 30 + } + } + } + }, + { + "id": 145, + "type": "FlexLatentInterpolate", + "pos": [ + 8195, + 2117 + ], + "size": { + "0": 344.3999938964844, + "1": 214 + }, + "flags": {}, + "order": 113, + "mode": 0, + "inputs": [ + { + "name": "latents", + "type": "LATENT", + "link": 241 + }, + { + "name": "feature", + "type": "FEATURE", + "link": 280 + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "link": 306 + }, + { + "name": "latent_2", + "type": "LATENT", + "link": 244 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 249, + 282 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "FlexLatentInterpolate" + }, + "widgets_values": [ + 1, + 0, + "None", + "relative", + "Spherical" + ] + }, + { + "id": 35, + "type": "NNLatentUpscale", + "pos": [ + 9851, + 2170 + ], + "size": { + "0": 315, + "1": 82 + }, + "flags": {}, + "order": 116, + "mode": 0, + "inputs": [ + { + "name": "latent", + "type": "LATENT", + "link": 282 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 84 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "NNLatentUpscale" + }, + "widgets_values": [ + "SD 1.x", + 1.67 + ] + }, + { + "id": 61, + "type": "KSampler", + "pos": [ + 10259, + 2135 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 118, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 91 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 77 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 78 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 84 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 80 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 156680208700286, + "fixed", + 4, + 1, + "lcm", + "sgm_uniform", + 0.55 + ] + }, + { + "id": 148, + "type": "ImageCASBatch", + "pos": [ + 10650, + 2301 + ], + "size": { + "0": 462, + "1": 82 + }, + "flags": {}, + "order": 120, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 250 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 251 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ImageCASBatch" + }, + "widgets_values": [ + 0.8, + 4 + ] + }, + { + "id": 201, + "type": "GetNode", + "pos": { + "0": 2078, + "1": 3833, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 45, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "LORA_STACK", + "type": "LORA_STACK", + "links": [ + 315 + ], + "slot_index": 0 + } + ], + "title": "Get_lora_stack", + "properties": {}, + "widgets_values": [ + "lora_stack" + ] + } + ], + "links": [ + [ + 12, + 30, + 1, + 12, + 0, + "CLIP" + ], + [ + 13, + 30, + 1, + 13, + 0, + "CLIP" + ], + [ + 14, + 10, + 0, + 14, + 0, + "LATENT" + ], + [ + 15, + 11, + 2, + 15, + 1, + "VAE" + ], + [ + 16, + 14, + 0, + 16, + 0, + "IMAGE" + ], + [ + 20, + 30, + 0, + 18, + 0, + "MODEL" + ], + [ + 21, + 21, + 0, + 18, + 1, + "CONTEXT_OPTIONS" + ], + [ + 22, + 26, + 0, + 18, + 2, + "MOTION_LORA" + ], + [ + 23, + 19, + 0, + 18, + 5, + "SAMPLE_SETTINGS" + ], + [ + 24, + 20, + 0, + 18, + 6, + "MULTIVAL" + ], + [ + 25, + 22, + 0, + 18, + 7, + "MULTIVAL" + ], + [ + 26, + 23, + 0, + 19, + 2, + "CUSTOM_CFG" + ], + [ + 27, + 18, + 0, + 27, + 0, + "MODEL" + ], + [ + 28, + 11, + 0, + 30, + 0, + "MODEL" + ], + [ + 29, + 11, + 1, + 30, + 1, + "CLIP" + ], + [ + 30, + 29, + 0, + 30, + 2, + "LORA_STACK" + ], + [ + 36, + 41, + 0, + 40, + 0, + "OPEN_UNMIX_MODEL" + ], + [ + 37, + 39, + 0, + 40, + 1, + "AUDIO" + ], + [ + 38, + 39, + 0, + 42, + 0, + "AUDIO" + ], + [ + 39, + 42, + 0, + 40, + 2, + "IMAGE" + ], + [ + 41, + 40, + 1, + 43, + 0, + "AUDIO" + ], + [ + 42, + 40, + 5, + 43, + 1, + "FEATURE_PIPE" + ], + [ + 46, + 32, + 0, + 10, + 1, + "CONDITIONING" + ], + [ + 47, + 32, + 1, + 10, + 2, + "CONDITIONING" + ], + [ + 48, + 39, + 0, + 45, + 0, + "*" + ], + [ + 49, + 46, + 0, + 16, + 1, + "AUDIO" + ], + [ + 56, + 39, + 0, + 49, + 0, + "AUDIO" + ], + [ + 57, + 40, + 0, + 50, + 0, + "AUDIO" + ], + [ + 58, + 43, + 0, + 51, + 0, + "FEATURE" + ], + [ + 59, + 51, + 0, + 52, + 0, + "IMAGE" + ], + [ + 63, + 15, + 0, + 10, + 3, + "LATENT" + ], + [ + 65, + 54, + 0, + 55, + 1, + "IMAGE" + ], + [ + 66, + 42, + 2, + 54, + 2, + "INT" + ], + [ + 67, + 42, + 2, + 57, + 2, + "INT" + ], + [ + 68, + 57, + 0, + 58, + 0, + "IMAGE" + ], + [ + 69, + 58, + 0, + 55, + 2, + "MASK" + ], + [ + 73, + 55, + 0, + 60, + 0, + "IMAGE" + ], + [ + 77, + 32, + 0, + 61, + 1, + "CONDITIONING" + ], + [ + 78, + 32, + 1, + 61, + 2, + "CONDITIONING" + ], + [ + 80, + 61, + 0, + 62, + 0, + "LATENT" + ], + [ + 84, + 35, + 0, + 61, + 3, + "LATENT" + ], + [ + 86, + 27, + 0, + 64, + 0, + "MODEL" + ], + [ + 87, + 27, + 1, + 64, + 1, + "IPADAPTER" + ], + [ + 90, + 64, + 0, + 10, + 0, + "MODEL" + ], + [ + 91, + 64, + 0, + 61, + 0, + "MODEL" + ], + [ + 96, + 65, + 1, + 32, + 1, + "CONDITIONING" + ], + [ + 97, + 65, + 0, + 32, + 0, + "CONDITIONING" + ], + [ + 98, + 66, + 0, + 65, + 2, + "CONTROL_NET" + ], + [ + 99, + 55, + 0, + 67, + 0, + "IMAGE" + ], + [ + 100, + 58, + 0, + 65, + 4, + "MASK" + ], + [ + 101, + 67, + 0, + 65, + 3, + "IMAGE" + ], + [ + 110, + 43, + 0, + 70, + 0, + "FEATURE" + ], + [ + 111, + 70, + 1, + 71, + 0, + "IMAGE" + ], + [ + 124, + 78, + 0, + 77, + 1, + "AUDIO" + ], + [ + 162, + 42, + 0, + 101, + 0, + "*" + ], + [ + 181, + 12, + 0, + 65, + 0, + "CONDITIONING" + ], + [ + 182, + 13, + 0, + 65, + 1, + "CONDITIONING" + ], + [ + 183, + 110, + 0, + 55, + 0, + "IMAGE" + ], + [ + 184, + 111, + 0, + 29, + 0, + "LORA_STACK" + ], + [ + 186, + 55, + 0, + 15, + 0, + "IMAGE" + ], + [ + 187, + 131, + 0, + 112, + 0, + "MODEL" + ], + [ + 188, + 129, + 0, + 112, + 1, + "CONDITIONING" + ], + [ + 189, + 129, + 1, + 112, + 2, + "CONDITIONING" + ], + [ + 190, + 116, + 0, + 112, + 3, + "LATENT" + ], + [ + 191, + 128, + 1, + 113, + 0, + "CLIP" + ], + [ + 192, + 128, + 1, + 114, + 0, + "CLIP" + ], + [ + 193, + 112, + 0, + 115, + 0, + "LATENT" + ], + [ + 197, + 115, + 0, + 117, + 0, + "IMAGE" + ], + [ + 199, + 128, + 0, + 118, + 0, + "MODEL" + ], + [ + 200, + 121, + 0, + 118, + 1, + "CONTEXT_OPTIONS" + ], + [ + 201, + 126, + 0, + 118, + 2, + "MOTION_LORA" + ], + [ + 202, + 119, + 0, + 118, + 5, + "SAMPLE_SETTINGS" + ], + [ + 203, + 120, + 0, + 118, + 6, + "MULTIVAL" + ], + [ + 204, + 122, + 0, + 118, + 7, + "MULTIVAL" + ], + [ + 205, + 123, + 0, + 119, + 2, + "CUSTOM_CFG" + ], + [ + 206, + 118, + 0, + 127, + 0, + "MODEL" + ], + [ + 210, + 132, + 0, + 129, + 0, + "CONDITIONING" + ], + [ + 211, + 132, + 1, + 129, + 1, + "CONDITIONING" + ], + [ + 212, + 127, + 0, + 131, + 0, + "MODEL" + ], + [ + 213, + 127, + 1, + 131, + 1, + "IPADAPTER" + ], + [ + 216, + 113, + 0, + 132, + 0, + "CONDITIONING" + ], + [ + 217, + 114, + 0, + 132, + 1, + "CONDITIONING" + ], + [ + 218, + 133, + 0, + 132, + 2, + "CONTROL_NET" + ], + [ + 219, + 134, + 0, + 132, + 3, + "IMAGE" + ], + [ + 238, + 40, + 2, + 142, + 0, + "AUDIO" + ], + [ + 239, + 40, + 5, + 142, + 1, + "FEATURE_PIPE" + ], + [ + 240, + 142, + 0, + 143, + 0, + "*" + ], + [ + 241, + 10, + 0, + 145, + 0, + "LATENT" + ], + [ + 244, + 112, + 0, + 145, + 3, + "LATENT" + ], + [ + 247, + 146, + 0, + 147, + 0, + "IMAGE" + ], + [ + 249, + 145, + 0, + 146, + 0, + "LATENT" + ], + [ + 250, + 62, + 0, + 148, + 0, + "IMAGE" + ], + [ + 251, + 148, + 0, + 63, + 0, + "IMAGE" + ], + [ + 252, + 144, + 0, + 149, + 0, + "FEATURE" + ], + [ + 257, + 40, + 2, + 152, + 0, + "AUDIO" + ], + [ + 258, + 40, + 5, + 152, + 1, + "FEATURE_PIPE" + ], + [ + 259, + 152, + 0, + 153, + 0, + "*" + ], + [ + 260, + 40, + 2, + 154, + 0, + "AUDIO" + ], + [ + 261, + 40, + 5, + 154, + 1, + "FEATURE_PIPE" + ], + [ + 262, + 154, + 0, + 155, + 0, + "*" + ], + [ + 263, + 40, + 2, + 156, + 0, + "AUDIO" + ], + [ + 264, + 40, + 5, + 156, + 1, + "FEATURE_PIPE" + ], + [ + 265, + 156, + 0, + 157, + 0, + "*" + ], + [ + 266, + 40, + 2, + 158, + 0, + "AUDIO" + ], + [ + 267, + 40, + 5, + 158, + 1, + "FEATURE_PIPE" + ], + [ + 268, + 158, + 0, + 159, + 0, + "*" + ], + [ + 269, + 149, + 1, + 150, + 0, + "IMAGE" + ], + [ + 280, + 149, + 0, + 145, + 1, + "FEATURE" + ], + [ + 282, + 145, + 0, + 35, + 0, + "LATENT" + ], + [ + 283, + 40, + 1, + 166, + 0, + "AUDIO" + ], + [ + 284, + 70, + 0, + 166, + 1, + "FEATURE" + ], + [ + 285, + 166, + 0, + 77, + 0, + "IMAGE" + ], + [ + 286, + 166, + 0, + 44, + 0, + "IMAGE" + ], + [ + 287, + 58, + 0, + 167, + 0, + "*" + ], + [ + 288, + 58, + 1, + 170, + 0, + "*" + ], + [ + 289, + 171, + 0, + 64, + 4, + "MASK" + ], + [ + 290, + 172, + 0, + 64, + 2, + "IMAGE" + ], + [ + 291, + 11, + 2, + 173, + 0, + "*" + ], + [ + 292, + 11, + 0, + 174, + 0, + "*" + ], + [ + 293, + 11, + 1, + 175, + 0, + "*" + ], + [ + 294, + 176, + 0, + 116, + 1, + "VAE" + ], + [ + 295, + 177, + 0, + 128, + 0, + "MODEL" + ], + [ + 296, + 178, + 0, + 128, + 1, + "CLIP" + ], + [ + 297, + 29, + 0, + 179, + 0, + "*" + ], + [ + 298, + 183, + 0, + 116, + 0, + "IMAGE" + ], + [ + 299, + 55, + 0, + 184, + 0, + "*" + ], + [ + 300, + 185, + 0, + 134, + 0, + "IMAGE" + ], + [ + 301, + 187, + 0, + 132, + 4, + "MASK" + ], + [ + 302, + 188, + 0, + 115, + 1, + "VAE" + ], + [ + 303, + 189, + 0, + 146, + 1, + "VAE" + ], + [ + 305, + 40, + 5, + 191, + 0, + "*" + ], + [ + 306, + 190, + 0, + 145, + 2, + "FEATURE_PIPE" + ], + [ + 307, + 192, + 0, + 60, + 1, + "AUDIO" + ], + [ + 308, + 193, + 0, + 62, + 1, + "VAE" + ], + [ + 309, + 194, + 0, + 14, + 1, + "VAE" + ], + [ + 310, + 196, + 0, + 131, + 4, + "MASK" + ], + [ + 311, + 195, + 0, + 131, + 2, + "IMAGE" + ], + [ + 312, + 197, + 0, + 117, + 1, + "AUDIO" + ], + [ + 313, + 198, + 0, + 147, + 1, + "AUDIO" + ], + [ + 314, + 199, + 0, + 63, + 1, + "AUDIO" + ], + [ + 315, + 201, + 0, + 128, + 2, + "LORA_STACK" + ] + ], + "groups": [ + { + "title": "ipadap", + "bounding": [ + 5762, + 1206, + 795, + 482 + ], + "color": "#3f789e", + "font_size": 24, + "locked": false + }, + { + "title": "samp", + "bounding": [ + 6868, + 1192, + 820, + 594 + ], + "color": "#3f789e", + "font_size": 24, + "locked": false + }, + { + "title": "samp22", + "bounding": [ + 9860, + 2028, + 1775, + 718 + ], + "color": "#3f789e", + "font_size": 24, + "locked": false + }, + { + "title": "adiff", + "bounding": [ + 4850, + 1136, + 812, + 1134 + ], + "color": "#3f789e", + "font_size": 24, + "locked": false + }, + { + "title": "model", + "bounding": [ + 1783, + 1297, + 1802, + 791 + ], + "color": "#3f789e", + "font_size": 24, + "locked": false + }, + { + "title": "controlnet", + "bounding": [ + 3830, + 1310, + 928, + 553 + ], + "color": "#3f789e", + "font_size": 24, + "locked": false + }, + { + "title": "Load Audio\\", + "bounding": [ + -4607, + 1227, + 2400, + 2155 + ], + "color": "#3f789e", + "font_size": 24, + "locked": false + }, + { + "title": "Load Images", + "bounding": [ + 196, + 1189, + 1570, + 1136 + ], + "color": "#3f789e", + "font_size": 24, + "locked": false + }, + { + "title": "Visualizer", + "bounding": [ + -2107, + 1980, + 1316, + 674 + ], + "color": "#3f789e", + "font_size": 24, + "locked": false + }, + { + "title": "Setup 2", + "bounding": [ + 1652, + 3179, + 1323, + 1047 + ], + "color": "#3f789e", + "font_size": 24, + "locked": false + }, + { + "title": "Controlnet", + "bounding": [ + 3017, + 3188, + 1082, + 784 + ], + "color": "#3f789e", + "font_size": 24, + "locked": false + }, + { + "title": "Adiff2", + "bounding": [ + 4141, + 3192, + 785, + 1120 + ], + "color": "#3f789e", + "font_size": 24, + "locked": false + }, + { + "title": "Ipadapter", + "bounding": [ + 4964, + 3209, + 891, + 611 + ], + "color": "#3f789e", + "font_size": 24, + "locked": false + }, + { + "title": "Sampler", + "bounding": [ + 5884, + 3192, + 1247, + 670 + ], + "color": "#3f789e", + "font_size": 24, + "locked": false + }, + { + "title": "Combine them", + "bounding": [ + 7317, + 1931, + 1842, + 888 + ], + "color": "#3f789e", + "font_size": 24, + "locked": false + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.10000000000000014, + "offset": { + "0": 6065.250457941647, + "1": 2064.0735111058475 + } + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/nodes/audio/flex_audio_visualizer.py b/nodes/audio/flex_audio_visualizer.py index b66a6b6..5ad3e75 100644 --- a/nodes/audio/flex_audio_visualizer.py +++ b/nodes/audio/flex_audio_visualizer.py @@ -286,7 +286,7 @@ class FlexAudioVisualizerLine(FlexAudioVisualizerBase): @classmethod def get_modifiable_params(cls): - return ["visualization_method", "visualization_feature", "smoothing", "rotation", "position_y", + return ["smoothing", "rotation", "position_y", "num_bars", "max_height", "min_height", "separation", "curvature", "reflect", "curve_smoothing", "fft_size", "min_frequency", "max_frequency", "None"] @@ -507,7 +507,7 @@ class FlexAudioVisualizerCircular(FlexAudioVisualizerBase): @classmethod def get_modifiable_params(cls): - return ["visualization_method", "visualization_feature", "smoothing", "rotation", "num_points", + return ["smoothing", "rotation", "num_points", "fft_size", "min_frequency", "max_frequency", "radius", "line_width", "amplitude_scale", "base_radius", "None"] diff --git a/nodes/images/flex_images.py b/nodes/images/flex_images.py index 7503601..012ab7a 100644 --- a/nodes/images/flex_images.py +++ b/nodes/images/flex_images.py @@ -9,14 +9,12 @@ from .image_utils import transform_image class FlexImageEdgeDetect(FlexImageBase): @classmethod def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), - "required": { - **super().INPUT_TYPES()["required"], - "low_threshold": ("FLOAT", {"default": 100, "min": 0, "max": 255, "step": 1}), - "high_threshold": ("FLOAT", {"default": 200, "min": 0, "max": 255, "step": 1}), - } - } + base_inputs = super().INPUT_TYPES() + base_inputs["required"].update({ + "low_threshold": ("FLOAT", {"default": 100, "min": 0, "max": 255, "step": 1}), + "high_threshold": ("FLOAT", {"default": 200, "min": 0, "max": 255, "step": 1}), + }) + return base_inputs @classmethod def get_modifiable_params(cls): @@ -35,16 +33,14 @@ class FlexImageEdgeDetect(FlexImageBase): class FlexImagePosterize(FlexImageBase): @classmethod def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), - "required": { - **super().INPUT_TYPES()["required"], - "max_levels": ("INT", {"default": 8, "min": 2, "max": 256, "step": 1}), - "dither_strength": ("FLOAT", {"default": 0.3, "min": 0.0, "max": 1.0, "step": 0.01}), - "channel_separation": ("FLOAT", {"default": 0.2, "min": 0.0, "max": 1.0, "step": 0.01}), - "gamma": ("FLOAT", {"default": 1.2, "min": 0.1, "max": 2.2, "step": 0.1}), - } - } + base_inputs = super().INPUT_TYPES() + base_inputs["required"].update({ + "max_levels": ("INT", {"default": 8, "min": 2, "max": 256, "step": 1}), + "dither_strength": ("FLOAT", {"default": 0.3, "min": 0.0, "max": 1.0, "step": 0.01}), + "channel_separation": ("FLOAT", {"default": 0.2, "min": 0.0, "max": 1.0, "step": 0.01}), + "gamma": ("FLOAT", {"default": 1.2, "min": 0.1, "max": 2.2, "step": 0.1}), + }) + return base_inputs @classmethod def get_modifiable_params(cls): @@ -90,19 +86,17 @@ class FlexImagePosterize(FlexImageBase): class FlexImageKaleidoscope(FlexImageBase): @classmethod def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), - "required": { - **super().INPUT_TYPES()["required"], - "segments": ("INT", {"default": 8, "min": 2, "max": 32, "step": 1}), - "center_x": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), - "center_y": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), - "zoom": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 2.0, "step": 0.1}), - "rotation": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 360.0, "step": 1.0}), - "precession": ("FLOAT", {"default": 0.0, "min": -1.0, "max": 1.0, "step": 0.01}), - "speed": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 5.0, "step": 0.1}), - } - } + base_inputs = super().INPUT_TYPES() + base_inputs["required"].update({ + "segments": ("INT", {"default": 8, "min": 2, "max": 32, "step": 1}), + "center_x": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + "center_y": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + "zoom": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 2.0, "step": 0.1}), + "rotation": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 360.0, "step": 1.0}), + "precession": ("FLOAT", {"default": 0.0, "min": -1.0, "max": 1.0, "step": 0.01}), + "speed": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 5.0, "step": 0.1}), + }) + return base_inputs @classmethod def get_modifiable_params(cls): @@ -144,15 +138,13 @@ class FlexImageKaleidoscope(FlexImageBase): class FlexImageColorGrade(FlexImageBase): @classmethod def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), - "required": { - **super().INPUT_TYPES()["required"], - "intensity": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), - "mix": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), - "lut_file": ("STRING", {"default": ""}), - }, - } + base_inputs = super().INPUT_TYPES() + base_inputs["required"].update({ + "intensity": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), + "mix": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), + "lut_file": ("STRING", {"default": ""}), + }) + return base_inputs @classmethod def get_modifiable_params(cls): @@ -231,14 +223,12 @@ class FlexImageGlitch(FlexImageBase): class FlexImageChromaticAberration(FlexImageBase): @classmethod def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), - "required": { - **super().INPUT_TYPES()["required"], - "shift_amount": ("FLOAT", {"default": 0.01, "min": 0.0, "max": 0.1, "step": 0.001}), - "angle": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 360.0, "step": 1.0}), - } - } + base_inputs = super().INPUT_TYPES() + base_inputs["required"].update({ + "shift_amount": ("FLOAT", {"default": 0.01, "min": 0.0, "max": 0.1, "step": 0.001}), + "angle": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 360.0, "step": 1.0}), + }) + return base_inputs @classmethod def get_modifiable_params(cls): @@ -262,13 +252,11 @@ class FlexImageChromaticAberration(FlexImageBase): class FlexImagePixelate(FlexImageBase): @classmethod def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), - "required": { - **super().INPUT_TYPES()["required"], - "pixel_size": ("INT", {"default": 10, "min": 1, "max": 100, "step": 1}), - } - } + base_inputs = super().INPUT_TYPES() + base_inputs["required"].update({ + "pixel_size": ("INT", {"default": 10, "min": 1, "max": 100, "step": 1}), + }) + return base_inputs @classmethod def get_modifiable_params(cls): @@ -297,15 +285,13 @@ class FlexImagePixelate(FlexImageBase): class FlexImageBloom(FlexImageBase): @classmethod def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), - "required": { - **super().INPUT_TYPES()["required"], - "threshold": ("FLOAT", {"default": 0.7, "min": 0.0, "max": 1.0, "step": 0.01}), - "blur_amount": ("FLOAT", {"default": 10.0, "min": 0.0, "max": 50.0, "step": 0.1}), - "intensity": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), - } - } + base_inputs = super().INPUT_TYPES() + base_inputs["required"].update({ + "threshold": ("FLOAT", {"default": 0.7, "min": 0.0, "max": 1.0, "step": 0.01}), + "blur_amount": ("FLOAT", {"default": 10.0, "min": 0.0, "max": 50.0, "step": 0.1}), + "intensity": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + }) + return base_inputs @classmethod def get_modifiable_params(cls): @@ -327,18 +313,16 @@ class FlexImageBloom(FlexImageBase): class FlexImageTiltShift(FlexImageBase): @classmethod def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), - "required": { - **super().INPUT_TYPES()["required"], - "blur_amount": ("FLOAT", {"default": 10.0, "min": 0.0, "max": 50.0, "step": 0.1}), - "focus_position_x": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), - "focus_position_y": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), - "focus_width": ("FLOAT", {"default": 0.2, "min": 0.0, "max": 1.0, "step": 0.01}), - "focus_height": ("FLOAT", {"default": 0.2, "min": 0.0, "max": 1.0, "step": 0.01}), - "focus_shape": (["rectangle", "ellipse"], {"default": "rectangle"}), - } - } + base_inputs = super().INPUT_TYPES() + base_inputs["required"].update({ + "blur_amount": ("FLOAT", {"default": 10.0, "min": 0.0, "max": 50.0, "step": 0.1}), + "focus_position_x": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + "focus_position_y": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + "focus_width": ("FLOAT", {"default": 0.2, "min": 0.0, "max": 1.0, "step": 0.01}), + "focus_height": ("FLOAT", {"default": 0.2, "min": 0.0, "max": 1.0, "step": 0.01}), + "focus_shape": (["rectangle", "ellipse"], {"default": "rectangle"}), + }) + return base_inputs @classmethod def get_modifiable_params(cls): @@ -373,17 +357,16 @@ class FlexImageTiltShift(FlexImageBase): class FlexImageParallax(FlexImageBase): @classmethod def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), - "required": { - **super().INPUT_TYPES()["required"], - "shift_x": ("FLOAT", {"default": 0.1, "min": -1.0, "max": 1.0, "step": 0.01}), - "shift_y": ("FLOAT", {"default": 0.1, "min": -1.0, "max": 1.0, "step": 0.01}), - "shift_z": ("FLOAT", {"default": 0.1, "min": -1.0, "max": 1.0, "step": 0.01}), - }, - "optional": {"depth_map": ("IMAGE",), - }, - } + base_inputs = super().INPUT_TYPES() + base_inputs["required"].update({ + "shift_x": ("FLOAT", {"default": 0.1, "min": -1.0, "max": 1.0, "step": 0.01}), + "shift_y": ("FLOAT", {"default": 0.1, "min": -1.0, "max": 1.0, "step": 0.01}), + "shift_z": ("FLOAT", {"default": 0.1, "min": -1.0, "max": 1.0, "step": 0.01}), + }) + base_inputs["optional"].update({ + "depth_map": ("IMAGE",), + }) + return base_inputs @classmethod def get_modifiable_params(cls): @@ -443,15 +426,13 @@ class FlexImageParallax(FlexImageBase): class FlexImageContrast(FlexImageBase): @classmethod def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), - "required": { - **super().INPUT_TYPES()["required"], - "contrast": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 3.0, "step": 0.01}), - "brightness": ("FLOAT", {"default": 0.0, "min": -1.0, "max": 1.0, "step": 0.01}), - "preserve_luminosity": ("BOOLEAN", {"default": True}), - } - } + base_inputs = super().INPUT_TYPES() + base_inputs["required"].update({ + "contrast": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 3.0, "step": 0.01}), + "brightness": ("FLOAT", {"default": 0.0, "min": -1.0, "max": 1.0, "step": 0.01}), + "preserve_luminosity": ("BOOLEAN", {"default": True}), + }) + return base_inputs @classmethod def get_modifiable_params(cls): @@ -484,22 +465,20 @@ import cv2 class FlexImageWarp(FlexImageBase): @classmethod def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), - "required": { - **super().INPUT_TYPES()["required"], - "warp_type": (["noise", "twist", "bulge"], {"default": "noise"}), - "warp_strength": ("FLOAT", {"default": 0.1, "min": -1.0, "max": 1.0, "step": 0.01}), - "center_x": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), - "center_y": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), - "radius": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.01}), - }, - "optional": { - "warp_frequency": ("FLOAT", {"default": 5.0, "min": 0.1, "max": 20.0, "step": 0.1}), - "warp_octaves": ("INT", {"default": 3, "min": 1, "max": 5, "step": 1}), - "warp_seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), - } - } + base_inputs = super().INPUT_TYPES() + base_inputs["required"].update({ + "warp_type": (["noise", "twist", "bulge"], {"default": "noise"}), + "warp_strength": ("FLOAT", {"default": 0.1, "min": -1.0, "max": 1.0, "step": 0.01}), + "center_x": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + "center_y": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + "radius": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.01}), + }) + base_inputs["optional"].update({ + "warp_frequency": ("FLOAT", {"default": 5.0, "min": 0.1, "max": 20.0, "step": 0.1}), + "warp_octaves": ("INT", {"default": 3, "min": 1, "max": 5, "step": 1}), + "warp_seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), + }) + return base_inputs @classmethod def get_modifiable_params(cls): @@ -563,17 +542,15 @@ class FlexImageWarp(FlexImageBase): class FlexImageVignette(FlexImageBase): @classmethod def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), - "required": { - **super().INPUT_TYPES()["required"], - "intensity": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), - "radius": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 2.0, "step": 0.01}), - "feather": ("FLOAT", {"default": 0.3, "min": 0.0, "max": 1.0, "step": 0.01}), - "center_x": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), - "center_y": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), - } - } + base_inputs = super().INPUT_TYPES() + base_inputs["required"].update({ + "intensity": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + "radius": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 2.0, "step": 0.01}), + "feather": ("FLOAT", {"default": 0.3, "min": 0.0, "max": 1.0, "step": 0.01}), + "center_x": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + "center_y": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + }) + return base_inputs @classmethod def get_modifiable_params(cls): @@ -618,15 +595,13 @@ class FlexImageVignette(FlexImageBase): class FlexImageTransform(FlexImageBase): @classmethod def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), - "required": { - **super().INPUT_TYPES()["required"], - "transform_type": (["translate", "rotate", "scale"],), - "x_value": ("FLOAT", {"default": 0.0, "min": -1000.0, "max": 1000.0, "step": 0.1}), - "y_value": ("FLOAT", {"default": 0.0, "min": -1000.0, "max": 1000.0, "step": 0.1}), - } - } + base_inputs = super().INPUT_TYPES() + base_inputs["required"].update({ + "transform_type": (["translate", "rotate", "scale"],), + "x_value": ("FLOAT", {"default": 0.0, "min": -1000.0, "max": 1000.0, "step": 0.1}), + "y_value": ("FLOAT", {"default": 0.0, "min": -1000.0, "max": 1000.0, "step": 0.1}), + }) + return base_inputs @classmethod def get_modifiable_params(cls): @@ -638,16 +613,14 @@ class FlexImageTransform(FlexImageBase): class FlexImageHueShift(FlexImageBase): @classmethod def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), - "required": { - **super().INPUT_TYPES()["required"], - "hue_shift": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 360.0, "step": 1.0}), - }, - "optional": { - "opt_mask": ("MASK",), - } - } + base_inputs = super().INPUT_TYPES() + base_inputs["required"].update({ + "hue_shift": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 360.0, "step": 1.0}), + }) + base_inputs["optional"].update({ + "opt_mask": ("MASK",), + }) + return base_inputs @classmethod def get_modifiable_params(cls): @@ -688,18 +661,15 @@ import numpy as np import cv2 class FlexImageDepthWarp(FlexImageBase): + @classmethod def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), - "required": { - **super().INPUT_TYPES()["required"], - "warp_strength": ("FLOAT", {"default": 0.1, "min": -10.0, "max": 10.0, "step": 0.01}), - }, - "optional": { - "depth_map": ("IMAGE",), - }, - } + base_inputs = super().INPUT_TYPES() + base_inputs["required"].update({ + "warp_strength": ("FLOAT", {"default": 0.1, "min": -10.0, "max": 10.0, "step": 0.01}), + "depth_map": ("IMAGE",), + }) + return base_inputs @classmethod def get_modifiable_params(cls): @@ -750,4 +720,4 @@ class FlexImageDepthWarp(FlexImageBase): return np.clip(warped_image, 0, 1) else: print("Warning: No depth map provided.") - return image \ No newline at end of file + return image diff --git a/nodes/latents/latent_base.py b/nodes/latents/latent_base.py index 8572c99..a7da4d9 100644 --- a/nodes/latents/latent_base.py +++ b/nodes/latents/latent_base.py @@ -6,6 +6,8 @@ from abc import ABC, abstractmethod from comfy.utils import ProgressBar from ... import RyanOnTheInside +#NOTE: this is a work in progress, it sucks + class FlexLatentBase(RyanOnTheInside, ABC): @classmethod def INPUT_TYPES(cls): @@ -101,7 +103,7 @@ class FlexLatentInterpolate(FlexLatentBase): @classmethod def get_modifiable_params(cls): - return ["interpolation_mode", "None"] + return ["None"] def process_latent(self, latent: np.ndarray, **kwargs) -> np.ndarray: feature_value = kwargs['feature_value'] diff --git a/pyproject.toml b/pyproject.toml index 740fca6..3e98481 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui_ryanonyheinside" description = "Custom nodes introducing particle simulations, optical flow, audio manipulation & reactivity, and temporal masks" -version = "1.9.4" +version = "1.9.5" license = {file = "LICENSE"} dependencies = ["pygame","opencv-python==4.10.0","scipy","torchaudio", "pillow","librosa==0.10.2","pymunk==6.8.1","matplotlib","openunmix","mido"] From 0ba97255b549238b04fb29bcbac7158b1c2fe9ef Mon Sep 17 00:00:00 2001 From: ryanontheinstide Date: Sat, 26 Oct 2024 07:47:41 -0400 Subject: [PATCH 04/10] moved flexmaskbase --- nodes/masks/flex_mask_base.py | 59 ++++++++++++++++++++++++++++++++ nodes/masks/flex_masks.py | 2 +- nodes/masks/flex_masks_normal.py | 3 +- nodes/masks/mask_base.py | 56 +----------------------------- 4 files changed, 63 insertions(+), 57 deletions(-) create mode 100644 nodes/masks/flex_mask_base.py diff --git a/nodes/masks/flex_mask_base.py b/nodes/masks/flex_mask_base.py new file mode 100644 index 0000000..8779670 --- /dev/null +++ b/nodes/masks/flex_mask_base.py @@ -0,0 +1,59 @@ +from .mask_base import MaskBase +from abc import ABC, abstractmethod +import numpy as np +import torch + +#TODO SOON TO UPDATE TO SUB FLEXBASE. UNTIL THEN REFER TO IMAGES AUDIO OR VIDEO FOR EXAMPLES ON HOW TO EXTEND +class FlexMaskBase(MaskBase): + feature_threshold_default=0.0 + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + **super().INPUT_TYPES()["required"], + "feature": ("FEATURE",), + "feature_pipe": ("FEATURE_PIPE",), + "feature_threshold": ("FLOAT", {"default": cls.feature_threshold_default, "min": 0.0, "max": 1.0, "step": 0.01}), + } + } + + CATEGORY = "RyanOnTheInside/FlexMasks" + RETURN_TYPES = ("MASK",) + FUNCTION = "main_function" + + @abstractmethod + def process_mask(self, mask: np.ndarray, feature_value: float, strength: float, **kwargs) -> np.ndarray: + pass + + def apply_mask_operation(self, masks, feature, feature_pipe, strength, feature_threshold, invert, subtract_original, grow_with_blur, **kwargs): + num_frames = feature_pipe.frame_count + original_masks = masks.clone() + + self.start_progress(num_frames, desc="Applying flex mask operation") + + result = [] + for i in range(num_frames): + kwargs['frame_index'] = i + mask = masks[i].numpy() + feature_value = feature.get_value_at_frame(i) + + if feature_value >= feature_threshold: + processed_mask = self.process_mask(mask, feature_value, strength, **kwargs) + else: + if hasattr(self, 'process_mask_below_threshold'): + processed_mask = self.process_mask_below_threshold(mask, feature_value, strength, **kwargs) + else: + processed_mask = mask + + + result.append(processed_mask) + self.update_progress() + + self.end_progress() + + processed_masks = torch.from_numpy(np.stack(result)).float() + return super().apply_mask_operation(processed_masks, original_masks, strength, invert, subtract_original, grow_with_blur, **kwargs) + + @abstractmethod + def main_function(self, masks, feature, feature_pipe, strength, feature_threshold, invert, subtract_original, grow_with_blur, **kwargs): + pass \ No newline at end of file diff --git a/nodes/masks/flex_masks.py b/nodes/masks/flex_masks.py index 676c7e0..482e4e3 100644 --- a/nodes/masks/flex_masks.py +++ b/nodes/masks/flex_masks.py @@ -1,4 +1,4 @@ -from .mask_base import FlexMaskBase +from .flex_mask_base import FlexMaskBase from .mask_utils import morph_mask, warp_mask, transform_mask, combine_masks,apply_easing import math import numpy as np diff --git a/nodes/masks/flex_masks_normal.py b/nodes/masks/flex_masks_normal.py index 9fb2ada..3ffb591 100644 --- a/nodes/masks/flex_masks_normal.py +++ b/nodes/masks/flex_masks_normal.py @@ -1,8 +1,9 @@ import numpy as np import torch import cv2 -from .mask_base import FlexMaskBase +from .flex_mask_base import FlexMaskBase +#NOTE: this is a work in progress, it sucks class FlexMaskNormalBase(FlexMaskBase): @classmethod def INPUT_TYPES(cls): diff --git a/nodes/masks/mask_base.py b/nodes/masks/mask_base.py index 7363361..85a88b8 100644 --- a/nodes/masks/mask_base.py +++ b/nodes/masks/mask_base.py @@ -211,8 +211,7 @@ class TemporalMaskBase(MaskBase, ABC): processed_masks = self.apply_temporal_mask_operation(masks, strength, start_frame, end_frame, effect_duration, temporal_easing, palindrome, **kwargs) ret = (self.apply_mask_operation(processed_masks[0], original_masks, strength, invert, subtract_original, grow_with_blur, **kwargs),) return ret - - + class OpticalFlowMaskBase(MaskBase, ABC): @classmethod def INPUT_TYPES(cls): @@ -275,58 +274,5 @@ class OpticalFlowMaskBase(MaskBase, ABC): processed_masks = np.stack(result) return self.apply_mask_operation(processed_masks, masks, strength, **kwargs) -#TODO SOON TO UPDATE TO SUB FLEXBASE. UNTIL THEN REFER TO IMAGES AUDIO OR VIDEO FOR EXAMPLES ON HOW TO EXTEND -class FlexMaskBase(MaskBase): - feature_threshold_default=0.0 - @classmethod - def INPUT_TYPES(cls): - return { - "required": { - **super().INPUT_TYPES()["required"], - "feature": ("FEATURE",), - "feature_pipe": ("FEATURE_PIPE",), - "feature_threshold": ("FLOAT", {"default": cls.feature_threshold_default, "min": 0.0, "max": 1.0, "step": 0.01}), - } - } - CATEGORY = "RyanOnTheInside/FlexMasks" - RETURN_TYPES = ("MASK",) - FUNCTION = "main_function" - - @abstractmethod - def process_mask(self, mask: np.ndarray, feature_value: float, strength: float, **kwargs) -> np.ndarray: - pass - - def apply_mask_operation(self, masks, feature, feature_pipe, strength, feature_threshold, invert, subtract_original, grow_with_blur, **kwargs): - num_frames = feature_pipe.frame_count - original_masks = masks.clone() - - self.start_progress(num_frames, desc="Applying flex mask operation") - - result = [] - for i in range(num_frames): - kwargs['frame_index'] = i - mask = masks[i].numpy() - feature_value = feature.get_value_at_frame(i) - - if feature_value >= feature_threshold: - processed_mask = self.process_mask(mask, feature_value, strength, **kwargs) - else: - if hasattr(self, 'process_mask_below_threshold'): - processed_mask = self.process_mask_below_threshold(mask, feature_value, strength, **kwargs) - else: - processed_mask = mask - - - result.append(processed_mask) - self.update_progress() - - self.end_progress() - - processed_masks = torch.from_numpy(np.stack(result)).float() - return super().apply_mask_operation(processed_masks, original_masks, strength, invert, subtract_original, grow_with_blur, **kwargs) - - @abstractmethod - def main_function(self, masks, feature, feature_pipe, strength, feature_threshold, invert, subtract_original, grow_with_blur, **kwargs): - pass From 672b933c3aa03dd449463d107bea2cc973473628 Mon Sep 17 00:00:00 2001 From: ryanontheinstide Date: Sat, 26 Oct 2024 09:01:14 -0400 Subject: [PATCH 05/10] import error --- nodes/masks/flex_mask_base.py | 2 +- nodes/masks/flex_masks.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/nodes/masks/flex_mask_base.py b/nodes/masks/flex_mask_base.py index 8779670..bcbab02 100644 --- a/nodes/masks/flex_mask_base.py +++ b/nodes/masks/flex_mask_base.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod import numpy as np import torch -#TODO SOON TO UPDATE TO SUB FLEXBASE. UNTIL THEN REFER TO IMAGES AUDIO OR VIDEO FOR EXAMPLES ON HOW TO EXTEND +#TODO SOON TO UPDATE TO SUB FLEXBASE. UNTIL THEN REFER TO ANYOTHER TARGET (images, audio, etc) FOR EXAMPLES ON HOW TO EXTEND class FlexMaskBase(MaskBase): feature_threshold_default=0.0 @classmethod diff --git a/nodes/masks/flex_masks.py b/nodes/masks/flex_masks.py index 482e4e3..a0265be 100644 --- a/nodes/masks/flex_masks.py +++ b/nodes/masks/flex_masks.py @@ -5,9 +5,7 @@ import numpy as np from .voronoi_noise import VoronoiNoise #NOTE credit for Voronoi goes to Alan Huang https://github.com/alanhuang67/ from comfy.model_management import get_torch_device import cv2 -from .mask_base import FlexMaskBase from scipy.ndimage import distance_transform_edt -from .mask_base import FlexMaskBase from .shape_utils import create_shape_mask, get_available_shapes import torch from typing import List From fd506e8764251f97c1fe35e48ef87bbd77e105ab Mon Sep 17 00:00:00 2001 From: ryanontheinstide Date: Sat, 26 Oct 2024 10:53:14 -0400 Subject: [PATCH 06/10] progress --- nodes/audio/audio_nodes_effects.py | 2 - nodes/audio/audio_nodes_utility.py | 1 - nodes/audio/flex_audio.py | 1 - nodes/audio/flex_audio_visualizer.py | 251 ++++++++++++++------------- nodes/flex/flex_base.py | 9 +- nodes/images/image_base.py | 40 +++-- 6 files changed, 166 insertions(+), 138 deletions(-) diff --git a/nodes/audio/audio_nodes_effects.py b/nodes/audio/audio_nodes_effects.py index 92b936b..3db0465 100644 --- a/nodes/audio/audio_nodes_effects.py +++ b/nodes/audio/audio_nodes_effects.py @@ -69,8 +69,6 @@ class AudioGain(AudioEffect): amplified_waveform = apply_gain(waveform, gain_db) return ({"waveform": amplified_waveform, "sample_rate": sample_rate},) - - class AudioTimeStretch(AudioEffect): @classmethod def INPUT_TYPES(cls): diff --git a/nodes/audio/audio_nodes_utility.py b/nodes/audio/audio_nodes_utility.py index b93a71a..bfdc6b8 100644 --- a/nodes/audio/audio_nodes_utility.py +++ b/nodes/audio/audio_nodes_utility.py @@ -189,7 +189,6 @@ class AudioSubtract(AudioUtility): subtracted_waveform = waveform1 - waveform2 return ({"waveform": subtracted_waveform, "sample_rate": sample_rate},) - class AudioDither(AudioUtility): @classmethod def INPUT_TYPES(cls): diff --git a/nodes/audio/flex_audio.py b/nodes/audio/flex_audio.py index 4d9e91c..67bd174 100644 --- a/nodes/audio/flex_audio.py +++ b/nodes/audio/flex_audio.py @@ -150,7 +150,6 @@ class FlexAudioBase(RyanOnTheInside, ABC): """Apply the effect to the audio frame. To be implemented by child classes.""" pass - class FlexAudioPitchShift(FlexAudioBase): @classmethod def INPUT_TYPES(cls): diff --git a/nodes/audio/flex_audio_visualizer.py b/nodes/audio/flex_audio_visualizer.py index 5ad3e75..a28e0ea 100644 --- a/nodes/audio/flex_audio_visualizer.py +++ b/nodes/audio/flex_audio_visualizer.py @@ -1,9 +1,11 @@ import torch import numpy as np import cv2 -from abc import ABC, abstractmethod +from abc import abstractmethod from comfy.utils import ProgressBar +from ..flex.flex_base import FlexBase from ... import RyanOnTheInside + import matplotlib.pyplot as plt class BaseAudioProcessor: @@ -88,116 +90,47 @@ class BaseAudioProcessor: # Apply smoothing self.spectrum = smoothing * self.spectrum + (1 - smoothing) * new_spectrum -class FlexAudioVisualizerBase(RyanOnTheInside, ABC): +class FlexAudioVisualizerBase(FlexBase, RyanOnTheInside): @classmethod def INPUT_TYPES(cls): - return { + base_inputs = super().INPUT_TYPES() + base_required = base_inputs.get("required", {}) + base_optional = base_inputs.get("optional", {}) + + new_inputs = { "required": { "audio": ("AUDIO",), "frame_rate": ("FLOAT", {"default": 30.0, "min": 1.0, "max": 240.0, "step": 1.0}), - "screen_width": ("INT", {"default": 800, "min": 100, "max": 1920, "step": 1}), - "screen_height": ("INT", {"default": 600, "min": 100, "max": 1080, "step": 1}), - "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), - "feature_param": (cls.get_modifiable_params(), {"default": cls.get_modifiable_params()[0]}), - "feature_mode": (["relative", "absolute"], {"default": "relative"}), + "screen_width": ("INT", {"default": 768, "min": 100, "max": 1920, "step": 1}), + "screen_height": ("INT", {"default": 464, "min": 100, "max": 1080, "step": 1}), "audio_feature_param": (cls.get_modifiable_params(), {"default": "None"}), "position_x": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), "position_y": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), }, "optional": { - "opt_feature": ("FEATURE",) + "opt_feature": ("FEATURE",), + "opt_feature_pipe": ("FEATURE_PIPE",), } } + required = {**base_required, **new_inputs["required"]} + optional = {**base_optional, **new_inputs["optional"]} + + return { + "required": required, + "optional": optional + } + CATEGORY = "RyanOnTheInside/FlexAudioVisualizer" - RETURN_TYPES = ("IMAGE","MASK") + RETURN_TYPES = ("IMAGE", "MASK") FUNCTION = "apply_effect" - def __init__(self): - self.progress_bar = None - - def start_progress(self, total_steps, desc="Processing"): - self.progress_bar = ProgressBar(total_steps) - - def update_progress(self): - if self.progress_bar: - self.progress_bar.update(1) - - def end_progress(self): - self.progress_bar = None - @classmethod @abstractmethod def get_modifiable_params(cls): """Return a list of parameter names that can be modulated.""" pass - def modulate_param(self, param_name, param_value, feature_value, strength, mode): - if mode == "relative": - modulated_value = param_value * (1 + (feature_value - 0.5) * strength * 2) - else: # absolute - modulated_value = param_value * feature_value * strength - # Ensure type consistency - return type(param_value)(modulated_value) - - def apply_effect(self, audio, frame_rate, screen_width, screen_height, strength, feature_param, feature_mode, - audio_feature_param, opt_feature=None, **kwargs): - # Calculate num_frames based on audio duration and frame_rate - audio_duration = len(audio['waveform'].squeeze(0).mean(axis=0)) / audio['sample_rate'] - num_frames = int(audio_duration * frame_rate) - - # Initialize the audio processor - processor = BaseAudioProcessor(audio, num_frames, screen_height, screen_width, frame_rate) - - # Initialize results list - result = [] - - self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}") - - for i in range(num_frames): - # Make a copy of kwargs for this frame to avoid altering original parameters - frame_kwargs = kwargs.copy() - - # Get audio data for visualization (returns audio_feature_value) - audio_feature_value = self.get_audio_data(processor, i, **frame_kwargs) - - # Modulate parameters based on opt_feature if provided - if opt_feature is not None and feature_param != "None": - # Get feature value from opt_feature - feature_value = opt_feature.get_value_at_frame(i) - - if feature_value is not None: - # Modulate the specified parameter using feature_value - original_value = frame_kwargs[feature_param] - modulated_value = self.modulate_param(feature_param, original_value, feature_value, strength, feature_mode) - # Ensure the modulated value is within valid ranges - modulated_value = self.validate_param(feature_param, modulated_value) - frame_kwargs[feature_param] = modulated_value - - # Modulate parameters based on audio feature value if audio_feature_param is set - if audio_feature_param != "None" and audio_feature_value is not None: - original_value = frame_kwargs[audio_feature_param] - modulated_value = self.modulate_param(audio_feature_param, original_value, audio_feature_value, strength, feature_mode) - # Ensure the modulated value is within valid ranges - modulated_value = self.validate_param(audio_feature_param, modulated_value) - frame_kwargs[audio_feature_param] = modulated_value - - # Generate the image for the current frame using frame_kwargs - image = self.draw(processor, **frame_kwargs) - result.append(image) - - self.update_progress() - - self.end_progress() - - # Convert the list of numpy arrays to a single numpy array - result_np = np.stack(result) # Shape: (N, H, W, 3) - - #lazy - result_tensor = torch.from_numpy(result_np).float() - mask = result_tensor[:, :, :, 0] - return (result_tensor,mask,) - def validate_param(self, param_name, param_value): """ Ensure that modulated parameter values stay within valid ranges. @@ -255,33 +188,103 @@ class FlexAudioVisualizerBase(RyanOnTheInside, ABC): """ pass + def apply_effect(self, *args, **kwargs): + return self.apply_effect_internal(*args, **kwargs) + + def apply_effect_internal(self, audio, frame_rate, screen_width, screen_height, strength, feature_param, feature_mode, + audio_feature_param, opt_feature=None, **kwargs): + # Calculate num_frames based on audio duration and frame_rate + audio_duration = len(audio['waveform'].squeeze(0).mean(axis=0)) / audio['sample_rate'] + num_frames = int(audio_duration * frame_rate) + + # Initialize the audio processor + processor = BaseAudioProcessor(audio, num_frames, screen_height, screen_width, frame_rate) + + # Initialize results list + result = [] + + self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}") + + for i in range(num_frames): + # Make a copy of kwargs for this frame to avoid altering original parameters + frame_kwargs = kwargs.copy() + + # Get audio data for visualization (returns audio_feature_value) + audio_feature_value = self.get_audio_data(processor, i, **frame_kwargs) + + # Modulate parameters based on opt_feature if provided + if opt_feature is not None and feature_param != "None": + # Get feature value from opt_feature + feature_value = opt_feature.get_value_at_frame(i) + + if feature_value is not None: + # Modulate the specified parameter using feature_value + original_value = frame_kwargs[feature_param] + modulated_value = self.modulate_param(feature_param, original_value, feature_value, strength, feature_mode) + # Ensure the modulated value is within valid ranges + modulated_value = self.validate_param(feature_param, modulated_value) + frame_kwargs[feature_param] = modulated_value + + # Modulate parameters based on audio feature value if audio_feature_param is set + if audio_feature_param != "None" and audio_feature_value is not None: + original_value = frame_kwargs[audio_feature_param] + modulated_value = self.modulate_param(audio_feature_param, original_value, audio_feature_value, strength, feature_mode) + # Ensure the modulated value is within valid ranges + modulated_value = self.validate_param(audio_feature_param, modulated_value) + frame_kwargs[audio_feature_param] = modulated_value + + # Generate the image for the current frame using frame_kwargs + image = self.draw(processor, **frame_kwargs) + result.append(image) + + self.update_progress() + + self.end_progress() + + # Convert the list of numpy arrays to a single numpy array + result_np = np.stack(result) # Shape: (N, H, W, 3) + + # Convert to tensor + result_tensor = torch.from_numpy(result_np).float() + mask = result_tensor[:, :, :, 0] + return (result_tensor, mask) + class FlexAudioVisualizerLine(FlexAudioVisualizerBase): @classmethod def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), + base_inputs = super().INPUT_TYPES() + base_required = base_inputs.get("required", {}) + base_optional = base_inputs.get("optional", {}) + + new_inputs = { "required": { - **super().INPUT_TYPES()["required"], "visualization_method": (["bar", "line"], {"default": "bar"}), "visualization_feature": (["frequency", "waveform"], {"default": "frequency"}), # Parameters common to both methods/features "smoothing": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), - "rotation": ("FLOAT", {"default":0.0, "min": 0.0, "max": 360.0, "step": 1.0}), - # "position_y": ("FLOAT", {"default":0.5, "min":0.0, "max":1.0,"step":0.01}), + "rotation": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 360.0, "step": 1.0}), # Additional parameters - "num_bars": ("INT", {"default":64, "min":1, "max":1024, "step":1}), - "max_height": ("FLOAT", {"default":200.0, "min":10.0, "max":2000.0, "step":10.0}), - "min_height": ("FLOAT", {"default":10.0, "min":0.0, "max":500.0, "step":5.0}), - "separation": ("FLOAT", {"default":5.0, "min":0.0, "max":100.0, "step":1.0}), - "curvature": ("FLOAT", {"default":0.0, "min":0.0, "max":50.0, "step":1.0}), + "num_bars": ("INT", {"default": 64, "min": 1, "max": 1024, "step": 1}), + "max_height": ("FLOAT", {"default": 200.0, "min": 10.0, "max": 2000.0, "step": 10.0}), + "min_height": ("FLOAT", {"default": 10.0, "min": 0.0, "max": 500.0, "step": 5.0}), + "separation": ("FLOAT", {"default": 5.0, "min": 0.0, "max": 100.0, "step": 1.0}), + "curvature": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 50.0, "step": 1.0}), "reflect": ("BOOLEAN", {"default": False}), - "curve_smoothing": ("FLOAT", {"default":0.0, "min": 0.0, "max":1.0, "step": 0.01}), - "fft_size": ("INT", {"default":2048, "min":256, "max":8192, "step":256}), - "min_frequency": ("FLOAT", {"default":20.0, "min":20.0, "max":20000.0, "step":10.0}), - "max_frequency": ("FLOAT", {"default":8000.0, "min":20.0, "max":20000.0, "step":10.0}), + "curve_smoothing": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), + "fft_size": ("INT", {"default": 2048, "min": 256, "max": 8192, "step": 256}), + "min_frequency": ("FLOAT", {"default": 20.0, "min": 20.0, "max": 20000.0, "step": 10.0}), + "max_frequency": ("FLOAT", {"default": 8000.0, "min": 20.0, "max": 20000.0, "step": 10.0}), } } + required = {**base_required, **new_inputs["required"]} + optional = base_optional + + return { + "required": required, + "optional": optional + } + FUNCTION = "apply_effect" @classmethod @@ -409,11 +412,11 @@ class FlexAudioVisualizerLine(FlexAudioVisualizerBase): mask = np.full((rect_height, rect_width), 0, dtype=np.uint8) cv2.rectangle(mask, (0, 0), (rect_width - 1, rect_height - 1), 255, -1) if radius > 1: - mask = cv2.GaussianBlur(mask, (radius*2+1, radius*2+1), 0) + mask = cv2.GaussianBlur(mask, (radius * 2 + 1, radius * 2 + 1), 0) # Apply mask rect[mask > 0] = color # Place rect onto image - image[y_start:y_end, x:x+rect_width] = rect + image[y_start:y_end, x:x + rect_width] = rect else: cv2.rectangle(image, (x, y_start), (x_end, y_end), color, thickness=-1) elif visualization_method == 'line': @@ -482,27 +485,37 @@ class FlexAudioVisualizerLine(FlexAudioVisualizerBase): class FlexAudioVisualizerCircular(FlexAudioVisualizerBase): @classmethod def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), + base_inputs = super().INPUT_TYPES() + base_required = base_inputs.get("required", {}) + base_optional = base_inputs.get("optional", {}) + + new_inputs = { "required": { - **super().INPUT_TYPES()["required"], "visualization_method": (["bar", "line"], {"default": "bar"}), "visualization_feature": (["frequency", "waveform"], {"default": "frequency"}), # Parameters common to both methods/features - "smoothing": ("FLOAT", {"default":0.5, "min":0.0, "max":1.0, "step":0.01}), - "rotation": ("FLOAT", {"default": 0.0, "min":0.0, "max":360.0, "step":1.0}), - "num_points": ("INT", {"default":360, "min":3, "max":1000, "step":1}), + "smoothing": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + "rotation": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 360.0, "step": 1.0}), + "num_points": ("INT", {"default": 360, "min": 3, "max": 1000, "step": 1}), # Additional parameters - "fft_size": ("INT", {"default":2048, "min":256, "max":8192, "step":256}), - "min_frequency": ("FLOAT", {"default":20.0, "min":20.0, "max":20000.0, "step":10.0}), - "max_frequency": ("FLOAT", {"default":8000.0, "min":20.0, "max":20000.0, "step":10.0}), - "radius": ("FLOAT", {"default":200.0, "min":10.0, "max":1000.0, "step":10.0}), - "line_width": ("INT", {"default":2, "min":1, "max":10, "step":1}), - "amplitude_scale": ("FLOAT", {"default":100.0, "min":1.0, "max":1000.0, "step":10.0}), - "base_radius": ("FLOAT", {"default":200.0, "min":10.0, "max":1000.0, "step":10.0}), + "fft_size": ("INT", {"default": 2048, "min": 256, "max": 8192, "step": 256}), + "min_frequency": ("FLOAT", {"default": 20.0, "min": 20.0, "max": 20000.0, "step": 10.0}), + "max_frequency": ("FLOAT", {"default": 8000.0, "min": 20.0, "max": 20000.0, "step": 10.0}), + "radius": ("FLOAT", {"default": 200.0, "min": 10.0, "max": 1000.0, "step": 10.0}), + "line_width": ("INT", {"default": 2, "min": 1, "max": 10, "step": 1}), + "amplitude_scale": ("FLOAT", {"default": 100.0, "min": 1.0, "max": 1000.0, "step": 10.0}), + "base_radius": ("FLOAT", {"default": 200.0, "min": 10.0, "max": 1000.0, "step": 10.0}), } } + required = {**base_required, **new_inputs["required"]} + optional = base_optional + + return { + "required": required, + "optional": optional + } + FUNCTION = "apply_effect" @classmethod @@ -622,4 +635,4 @@ class FlexAudioVisualizerCircular(FlexAudioVisualizerBase): if len(points) > 2: cv2.polylines(image, [points], isClosed=True, color=(1.0, 1.0, 1.0), thickness=line_width) - return image \ No newline at end of file + return image diff --git a/nodes/flex/flex_base.py b/nodes/flex/flex_base.py index 1e9a6c5..ac302bb 100644 --- a/nodes/flex/flex_base.py +++ b/nodes/flex/flex_base.py @@ -10,12 +10,15 @@ class FlexBase(ABC): "required": { "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), "feature_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), - "feature_param": (cls.get_modifiable_params(), {"default": cls.get_modifiable_params()[0] if cls.get_modifiable_params() else "None"}), + "feature_param": ( + cls.get_modifiable_params(), + {"default": cls.get_modifiable_params()[0] if cls.get_modifiable_params() else "None"} + ), "feature_mode": (["relative", "absolute"], {"default": "relative"}), }, "optional": { - "feature": ("FEATURE",), - "feature_pipe": ("FEATURE_PIPE",), + "opt_feature": ("FEATURE",), + "opt_feature_pipe": ("FEATURE_PIPE",), } } diff --git a/nodes/images/image_base.py b/nodes/images/image_base.py index cdf211a..33469b6 100644 --- a/nodes/images/image_base.py +++ b/nodes/images/image_base.py @@ -15,10 +15,7 @@ class FlexImageBase(RyanOnTheInside, FlexBase): **super().INPUT_TYPES()["required"], "images": ("IMAGE",), }, - "optional": { - "opt_feature": ("FEATURE",), - "opt_feature_pipe": ("FEATURE_PIPE",), - } + # Optional inputs are inherited from FlexBase } CATEGORY = "RyanOnTheInside/FlexImage" @@ -50,11 +47,23 @@ class FlexImageBase(RyanOnTheInside, FlexBase): else: # absolute return param_value * feature_value * strength - def apply_effect(self, images, strength, feature_threshold, feature_param, feature_mode, feature=None, feature_pipe=None, **kwargs): - if (feature is None) != (feature_pipe is None): - raise ValueError("Both feature and feature_pipe must be provided together, or neither should be provided.") + def apply_effect( + self, + images, + strength, + feature_threshold, + feature_param, + feature_mode, + opt_feature=None, + opt_feature_pipe=None, + **kwargs + ): + if (opt_feature is None) != (opt_feature_pipe is None): + raise ValueError( + "Both opt_feature and opt_feature_pipe must be provided together, or neither should be provided." + ) - if feature is None and feature_pipe is None: + if opt_feature is None and opt_feature_pipe is None: # Process all frames without modulation num_frames = images.shape[0] images_np = images.cpu().numpy() @@ -75,7 +84,7 @@ class FlexImageBase(RyanOnTheInside, FlexBase): self.update_progress() else: # Process frames with modulation based on feature values - num_frames = feature_pipe.frame_count + num_frames = opt_feature_pipe.frame_count images_np = images.cpu().numpy() self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}") @@ -83,7 +92,7 @@ class FlexImageBase(RyanOnTheInside, FlexBase): result = [] for i in range(num_frames): image = images_np[i] - feature_value = feature.get_value_at_frame(i) + feature_value = opt_feature.get_value_at_frame(i) kwargs['frame_index'] = i if feature_value >= feature_threshold: processed_image = self.process_image( @@ -112,8 +121,15 @@ class FlexImageBase(RyanOnTheInside, FlexBase): return (result_tensor,) - def process_image(self, image: np.ndarray, feature_value: float, strength: float, - feature_param: str, feature_mode: str, **kwargs) -> np.ndarray: + def process_image( + self, + image: np.ndarray, + feature_value: float, + strength: float, + feature_param: str, + feature_mode: str, + **kwargs + ) -> np.ndarray: # Modulate the selected parameter for param_name in self.get_modifiable_params(): if param_name in kwargs and param_name == feature_param: From 57fee654e823c6f2f7e447c4cfb3340c92d84f0d Mon Sep 17 00:00:00 2001 From: ryanontheinstide Date: Sat, 26 Oct 2024 11:12:52 -0400 Subject: [PATCH 07/10] Squashed commit of the following: commit 8b2c379a9c01af4ade0b70cf23d74801a68e1e38 Author: ryanontheinstide Date: Sat Oct 26 11:04:08 2024 -0400 proximity feature bugfix commit e6906cf9c0c94aecfb7aa7709e52ded02f4dcb78 Author: ryanontheinstide Date: Sat Oct 26 11:03:40 2024 -0400 proximity feature bug fix commit caeca22fe49b4c5dde0236f2d0e6b820aca763a6 Author: ryanontheinstide Date: Fri Oct 25 14:23:30 2024 -0400 smiley example commit e446017b1b1e4f2dde8d376b7c58682b1037eb6e Author: ryanontheinstide Date: Fri Oct 25 14:08:07 2024 -0400 visualizer beta commit 585ee8d574cd5075807bf0c956fbff1916eb8935 Author: ryanontheinstide Date: Fri Oct 25 09:17:14 2024 -0400 interpolation_mode removed commit a3d5e5b2941e3c0e1d42508837a57985e1145369 Author: ryanontheinstide Date: Thu Oct 24 13:01:17 2024 -0400 fixed inherit inputs commit 7b0009877622c782866c502baf39852f62438e32 Author: ryanontheinstide Date: Thu Oct 24 11:23:21 2024 -0400 unusable feature param --- nodes/flex/features_proximity.py | 3 +++ pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/nodes/flex/features_proximity.py b/nodes/flex/features_proximity.py index 65cbd3d..2b941d4 100644 --- a/nodes/flex/features_proximity.py +++ b/nodes/flex/features_proximity.py @@ -14,6 +14,9 @@ class ProximityFeature(BaseFeature): self.proximity_values = None self.normalization_method = normalization_method + def get_extraction_methods(self): + return ["normalization_method"] + def extract(self): from scipy.spatial.distance import cdist diff --git a/pyproject.toml b/pyproject.toml index 3e98481..84f3792 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui_ryanonyheinside" description = "Custom nodes introducing particle simulations, optical flow, audio manipulation & reactivity, and temporal masks" -version = "1.9.5" +version = "1.9.6" license = {file = "LICENSE"} dependencies = ["pygame","opencv-python==4.10.0","scipy","torchaudio", "pillow","librosa==0.10.2","pymunk==6.8.1","matplotlib","openunmix","mido"] From c9d5f054dcf7639ad564349fa070a64c97a3cbfd Mon Sep 17 00:00:00 2001 From: ryanontheinstide Date: Sat, 26 Oct 2024 12:11:29 -0400 Subject: [PATCH 08/10] subclass FlexBase --- nodes/audio/flex_audio.py | 125 ++++++++++++++++++++++---------------- 1 file changed, 74 insertions(+), 51 deletions(-) diff --git a/nodes/audio/flex_audio.py b/nodes/audio/flex_audio.py index 67bd174..cf40917 100644 --- a/nodes/audio/flex_audio.py +++ b/nodes/audio/flex_audio.py @@ -4,25 +4,35 @@ from abc import ABC, abstractmethod from tqdm import tqdm from comfy.utils import ProgressBar from ... import RyanOnTheInside +from ..flex.flex_base import FlexBase from .audio_utils import pitch_shift, time_stretch -import torch -import numpy as np -from abc import ABC, abstractmethod -class FlexAudioBase(RyanOnTheInside, ABC): +class FlexAudioBase(FlexBase): @classmethod def INPUT_TYPES(cls): + base_input_types = super().INPUT_TYPES() + base_required = base_input_types.get("required", {}) + base_optional = base_input_types.get("optional", {}) + + # Update required inputs + base_required.update({ + "audio": ("AUDIO",), + "target_fps": ("FLOAT", {"default": 3.0, "min": 1.0, "max": 60.0, "step": 1.0}), + }) + + # Update optional inputs + base_optional.update({ + "opt_feature": ("FEATURE",), + "opt_feature_pipe": ("FEATURE_PIPE",), + "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), + "feature_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), + "feature_param": (cls.get_modifiable_params(), {"default": cls.get_modifiable_params()[0]}), + "feature_mode": (["relative", "absolute"], {"default": "relative"}), + }) + return { - "required": { - "audio": ("AUDIO",), - "feature": ("FEATURE",), - "feature_pipe": ("FEATURE_PIPE",), - "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), - "feature_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), - "feature_param": (cls.get_modifiable_params(), {"default": cls.get_modifiable_params()[0]}), - "feature_mode": (["relative", "absolute"], {"default": "relative"}), - "target_fps": ("FLOAT", {"default": 3.0, "min": 1.0, "max": 60.0, "step": 1.0}), - } + "required": base_required, + "optional": base_optional, } CATEGORY = "RyanOnTheInside/FlexAudio" @@ -30,7 +40,7 @@ class FlexAudioBase(RyanOnTheInside, ABC): FUNCTION = "apply_effect" def __init__(self): - self.progress_bar = None + super().__init__() def start_progress(self, total_steps, desc="Processing"): self.progress_bar = ProgressBar(total_steps) @@ -48,24 +58,18 @@ class FlexAudioBase(RyanOnTheInside, ABC): """Return a list of parameter names that can be modulated.""" return [] - def modulate_param(self, param_value, feature_value, strength, mode): - if mode == "relative": - # Adjust to vary between -param_value and +param_value - return param_value * (2 * feature_value - 1) * strength - else: # absolute - # Directly use feature_value to determine the shift - return param_value * (2 * feature_value - 1) - - def apply_effect(self, audio, feature, feature_pipe, strength, feature_threshold, feature_param, feature_mode, target_fps, **kwargs): + def apply_effect(self, audio, opt_feature=None, opt_feature_pipe=None, strength=1.0, feature_threshold=0.0, feature_param=None, feature_mode="relative", target_fps=3.0, **kwargs): waveform = audio['waveform'] # Shape: [Batch, Channels, Samples] sample_rate = audio['sample_rate'] - original_num_frames = feature_pipe.frame_count device = torch.device("cuda" if torch.cuda.is_available() else "cpu") waveform = waveform.to(device) audio_duration = waveform.shape[-1] / sample_rate - target_num_frames = int(audio_duration * target_fps) - num_frames = min(target_num_frames, original_num_frames) + num_frames = int(audio_duration * target_fps) + + if opt_feature_pipe is not None: + original_num_frames = opt_feature_pipe.frame_count + num_frames = min(num_frames, original_num_frames) self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}") @@ -78,11 +82,14 @@ class FlexAudioBase(RyanOnTheInside, ABC): # Initialize list to store processed frames processed_frames = [] - # Pre-compute averaged feature values - feature_values = np.array([feature.get_value_at_frame(i) for i in range(original_num_frames)]) - frames_per_segment = max(1, original_num_frames // num_frames) - averaged_features = [np.mean(feature_values[i:i+frames_per_segment]) - for i in range(0, original_num_frames, frames_per_segment)] + # Pre-compute averaged feature values if feature is provided + if opt_feature is not None and opt_feature_pipe is not None: + feature_values = np.array([opt_feature.get_value_at_frame(i) for i in range(opt_feature_pipe.frame_count)]) + frames_per_segment = max(1, opt_feature_pipe.frame_count // num_frames) + averaged_features = [np.mean(feature_values[i:i+frames_per_segment]) + for i in range(0, opt_feature_pipe.frame_count, frames_per_segment)] + else: + averaged_features = [None] * num_frames # No feature modulation # Define cross-fade length (e.g., 10% of frame length) crossfade_length = int(samples_per_frame * 0.1) @@ -97,7 +104,7 @@ class FlexAudioBase(RyanOnTheInside, ABC): kwargs['frame_index'] = i kwargs['sample_rate'] = sample_rate - if feature_value >= feature_threshold: + if feature_value is not None and feature_value >= feature_threshold: try: processed_frame = self.process_audio_frame( audio_frame, feature_value, strength, @@ -111,7 +118,8 @@ class FlexAudioBase(RyanOnTheInside, ABC): traceback.print_exc() processed_frame = audio_frame else: - processed_frame = audio_frame + # Process without feature modulation + processed_frame = self.apply_effect_internal(audio_frame, **kwargs) if i > 0: # Apply cross-fade with previous frame @@ -134,13 +142,14 @@ class FlexAudioBase(RyanOnTheInside, ABC): def process_audio_frame(self, audio_frame: torch.Tensor, feature_value: float, strength: float, feature_param: str, feature_mode: str, **kwargs) -> torch.Tensor: - # Modulate the selected parameter - if feature_param in self.get_modifiable_params() and feature_param in kwargs: + # Modulate the selected parameter if feature_value is provided + if feature_value is not None and feature_param in self.get_modifiable_params() and feature_param in kwargs: + param_value = kwargs[feature_param] modulated_value = self.modulate_param( - kwargs[feature_param], feature_value, strength, feature_mode + feature_param, param_value, feature_value, strength, feature_mode ) - # Ensure n_steps is within the desired range - kwargs[feature_param] = max(-kwargs[feature_param], min(kwargs[feature_param], modulated_value)) + # Ensure the modulated value is within the desired range + kwargs[feature_param] = modulated_value # Call the child class's implementation return self.apply_effect_internal(audio_frame, **kwargs) @@ -153,11 +162,18 @@ class FlexAudioBase(RyanOnTheInside, ABC): class FlexAudioPitchShift(FlexAudioBase): @classmethod def INPUT_TYPES(cls): + base_input_types = super().INPUT_TYPES() + base_required = base_input_types.get("required", {}) + base_optional = base_input_types.get("optional", {}) + + # Update required inputs + base_required.update({ + "n_steps": ("FLOAT", {"default": 6.0, "min": 0.0, "max": 12.0, "step": 0.1}), + }) + return { - "required": { - **super().INPUT_TYPES()["required"], - "n_steps": ("FLOAT", {"default": 6.0, "min": 0.0, "max": 12.0, "step": 0.1}), - } + "required": base_required, + "optional": base_optional, } @classmethod @@ -166,18 +182,25 @@ class FlexAudioPitchShift(FlexAudioBase): def apply_effect_internal(self, audio_frame: torch.Tensor, n_steps: float, **kwargs) -> torch.Tensor: sample_rate = kwargs.get('sample_rate', 44100) # Default to 44100 if not provided - - # Use the absolute value of n_steps for pitch shifting + + # Use the value of n_steps for pitch shifting return pitch_shift(audio_frame, sample_rate, n_steps) - + class FlexAudioTimeStretch(FlexAudioBase): @classmethod def INPUT_TYPES(cls): + base_input_types = super().INPUT_TYPES() + base_required = base_input_types.get("required", {}) + base_optional = base_input_types.get("optional", {}) + + # Update required inputs + base_required.update({ + "rate": ("FLOAT", {"default": 1.0, "min": 0.5, "max": 2.0, "step": 0.01}), + }) + return { - "required": { - **super().INPUT_TYPES()["required"], - "rate": ("FLOAT", {"default": 1.0, "min": 0.5, "max": 2.0, "step": 0.01}), - } + "required": base_required, + "optional": base_optional, } @classmethod @@ -185,4 +208,4 @@ class FlexAudioTimeStretch(FlexAudioBase): return ["rate"] def apply_effect_internal(self, audio_frame: torch.Tensor, rate: float, **kwargs) -> torch.Tensor: - return time_stretch(audio_frame, rate) \ No newline at end of file + return time_stretch(audio_frame, rate) From eefa76397bedcc038ba84041f1a6ffd76b0ae98d Mon Sep 17 00:00:00 2001 From: ryanontheinstide Date: Sat, 26 Oct 2024 14:00:01 -0400 Subject: [PATCH 09/10] refactor --- __init__.py | 2 +- nodes/audio/audio_nodes.py | 3 +- nodes/audio/flex_audio.py | 4 +- nodes/depth/depth_base.py | 125 +++++--- nodes/flex/feature_extractors_midi.py | 2 +- .../{midi_feature.py => features_midi.py} | 0 nodes/flex/midi_feature_extractor.py | 163 ---------- nodes/flex/proximity_feature.py | 92 ------ nodes/flex/proximity_feature_extractor.py | 279 ------------------ .../{image_base.py => flex_image_base.py} | 0 nodes/images/flex_images.py | 2 +- nodes/latents/flex_latent_base.py | 105 +++++++ .../{latent_base.py => flex_latents.py} | 162 +--------- 13 files changed, 203 insertions(+), 736 deletions(-) rename nodes/flex/{midi_feature.py => features_midi.py} (100%) delete mode 100644 nodes/flex/midi_feature_extractor.py delete mode 100644 nodes/flex/proximity_feature.py delete mode 100644 nodes/flex/proximity_feature_extractor.py rename nodes/images/{image_base.py => flex_image_base.py} (100%) create mode 100644 nodes/latents/flex_latent_base.py rename nodes/latents/{latent_base.py => flex_latents.py} (59%) diff --git a/__init__.py b/__init__.py index f562581..ae7d0ba 100644 --- a/__init__.py +++ b/__init__.py @@ -253,7 +253,7 @@ from .nodes.audio.flex_audio import ( FlexAudioTimeStretch, ) -from .nodes.latents.latent_base import ( +from .nodes.latents.flex_latents import ( FlexLatentInterpolate, EmbeddingGuidedLatentInterpolate, FlexLatentBlend, diff --git a/nodes/audio/audio_nodes.py b/nodes/audio/audio_nodes.py index b51674b..b49e048 100644 --- a/nodes/audio/audio_nodes.py +++ b/nodes/audio/audio_nodes.py @@ -496,8 +496,7 @@ class EmptyMaskFromAudio(AudioNodeBase): # if not folder_paths.exists_annotated_filepath(audio): # return "Invalid audio file: {}".format(audio) # return True - -#TODO + class EmptyImageAndMaskFromAudio(AudioNodeBase): @classmethod def INPUT_TYPES(cls): diff --git a/nodes/audio/flex_audio.py b/nodes/audio/flex_audio.py index cf40917..af225a0 100644 --- a/nodes/audio/flex_audio.py +++ b/nodes/audio/flex_audio.py @@ -1,13 +1,13 @@ import torch import numpy as np -from abc import ABC, abstractmethod +from abc import abstractmethod from tqdm import tqdm from comfy.utils import ProgressBar from ... import RyanOnTheInside from ..flex.flex_base import FlexBase from .audio_utils import pitch_shift, time_stretch -class FlexAudioBase(FlexBase): +class FlexAudioBase(FlexBase, RyanOnTheInside): @classmethod def INPUT_TYPES(cls): base_input_types = super().INPUT_TYPES() diff --git a/nodes/depth/depth_base.py b/nodes/depth/depth_base.py index a5aaddb..fa8b4ee 100644 --- a/nodes/depth/depth_base.py +++ b/nodes/depth/depth_base.py @@ -1,24 +1,22 @@ import torch import numpy as np from abc import ABC, abstractmethod -from tqdm import tqdm from comfy.utils import ProgressBar from ... import RyanOnTheInside +from ..flex.flex_base import FlexBase import cv2 -class FlexDepthBase(RyanOnTheInside, ABC): +#NOTE: in hindsight, much of this would have been better suited as mask-based operations +class FlexDepthBase(RyanOnTheInside, FlexBase): @classmethod def INPUT_TYPES(cls): return { + **super().INPUT_TYPES(), "required": { + **super().INPUT_TYPES()["required"], "depth_maps": ("IMAGE",), - "feature": ("FEATURE",), - "feature_pipe": ("FEATURE_PIPE",), - "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), - "feature_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), - "feature_param": (cls.get_modifiable_params(), {"default": cls.get_modifiable_params()[0]}), - "feature_mode": (["relative", "absolute"], {"default": "relative"}), - } + }, + # Optional inputs are inherited from FlexBase } CATEGORY = "RyanOnTheInside/DepthEffects" @@ -28,16 +26,6 @@ class FlexDepthBase(RyanOnTheInside, ABC): def __init__(self): self.progress_bar = None - def start_progress(self, total_steps, desc="Processing"): - self.progress_bar = ProgressBar(total_steps) - - def update_progress(self): - if self.progress_bar: - self.progress_bar.update(1) - - def end_progress(self): - self.progress_bar = None - @classmethod @abstractmethod def get_modifiable_params(cls): @@ -50,27 +38,64 @@ class FlexDepthBase(RyanOnTheInside, ABC): else: # absolute return param_value * feature_value * strength - def apply_effect(self, depth_maps, feature, feature_pipe, strength, feature_threshold, feature_param, feature_mode, **kwargs): - num_frames = feature_pipe.frame_count + def apply_effect( + self, + depth_maps, + strength, + feature_threshold, + feature_param, + feature_mode, + opt_feature=None, + opt_feature_pipe=None, + **kwargs + ): + if (opt_feature is None) != (opt_feature_pipe is None): + raise ValueError( + "Both opt_feature and opt_feature_pipe must be provided together, or neither should be provided." + ) + + num_frames = depth_maps.shape[0] depth_maps_np = depth_maps.cpu().numpy() - self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}") + if opt_feature is None and opt_feature_pipe is None: + self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}") - result = [] - for i in range(num_frames): - depth_map = depth_maps_np[i] - feature_value = feature.get_value_at_frame(i) - kwargs['frame_index'] = i - if feature_value >= feature_threshold: - processed_depth_map = self.process_depth_map(depth_map, feature_value, strength, - feature_param=feature_param, - feature_mode=feature_mode, - **kwargs) - else: - processed_depth_map = depth_map + result = [] + for i in range(num_frames): + processed_depth_map = self.process_depth_map( + depth_maps_np[i], + 0.5, # Default feature value when no feature is provided + strength, + feature_param=feature_param, + feature_mode=feature_mode, + frame_index=i, + **kwargs + ) + result.append(processed_depth_map) + self.update_progress() + else: + num_frames = opt_feature_pipe.frame_count + self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}") - result.append(processed_depth_map) - self.update_progress() + result = [] + for i in range(num_frames): + depth_map = depth_maps_np[i] + feature_value = opt_feature.get_value_at_frame(i) + kwargs['frame_index'] = i + if feature_value >= feature_threshold: + processed_depth_map = self.process_depth_map( + depth_map, + feature_value, + strength, + feature_param=feature_param, + feature_mode=feature_mode, + **kwargs + ) + else: + processed_depth_map = depth_map + + result.append(processed_depth_map) + self.update_progress() self.end_progress() @@ -81,20 +106,30 @@ class FlexDepthBase(RyanOnTheInside, ABC): result_tensor = torch.from_numpy(result_np).float() # Ensure the tensor is in BHWC format - if result_tensor.shape[1] != depth_maps.shape[1]: # Adjust as needed + if result_tensor.shape[1] != depth_maps.shape[1]: result_tensor = result_tensor.permute(0, 2, 3, 1) return (result_tensor,) - def process_depth_map(self, depth_map: np.ndarray, feature_value: float, strength: float, - feature_param: str, feature_mode: str, **kwargs) -> np.ndarray: + def process_depth_map( + self, + depth_map: np.ndarray, + feature_value: float, + strength: float, + feature_param: str, + feature_mode: str, + **kwargs + ) -> np.ndarray: # Modulate the selected parameter for param_name in self.get_modifiable_params(): - if param_name in kwargs: - if param_name == feature_param: - kwargs[param_name] = self.modulate_param(param_name, kwargs[param_name], - feature_value, strength, feature_mode) - + if param_name in kwargs and param_name == feature_param: + kwargs[param_name] = self.modulate_param( + param_name, + kwargs[param_name], + feature_value, + strength, + feature_mode + ) # Call the child class's implementation return self.apply_effect_internal(depth_map, **kwargs) @@ -284,3 +319,5 @@ class DepthRippleEffect(FlexDepthBase): modified_depth = np.clip(modified_depth, 0.0, 1.0) return modified_depth + + diff --git a/nodes/flex/feature_extractors_midi.py b/nodes/flex/feature_extractors_midi.py index 6fd40ac..9a390d0 100644 --- a/nodes/flex/feature_extractors_midi.py +++ b/nodes/flex/feature_extractors_midi.py @@ -5,7 +5,7 @@ import folder_paths from server import PromptServer from aiohttp import web import shutil -from .midi_feature import MIDIFeature +from .features_midi import MIDIFeature from .feature_extractors import FeatureExtractorBase class MIDILoadAndExtract(FeatureExtractorBase): diff --git a/nodes/flex/midi_feature.py b/nodes/flex/features_midi.py similarity index 100% rename from nodes/flex/midi_feature.py rename to nodes/flex/features_midi.py diff --git a/nodes/flex/midi_feature_extractor.py b/nodes/flex/midi_feature_extractor.py deleted file mode 100644 index b065939..0000000 --- a/nodes/flex/midi_feature_extractor.py +++ /dev/null @@ -1,163 +0,0 @@ -import mido -import os -from .feature_pipe import FeaturePipe -import folder_paths -from server import PromptServer -from aiohttp import web -import shutil -from .midi_feature import MIDIFeature -from .feature_extractors import FeatureExtractorBase - -class MIDILoadAndExtract(FeatureExtractorBase): - @classmethod - def INPUT_TYPES(cls): - return { - "required": { - "midi_file": (folder_paths.get_filename_list("midi_files"),), - "track_selection": (["all"],), - "attribute": (MIDIFeature.get_attribute_names(), {"default": "Note On/Off"}), - "frame_rate": ("FLOAT", {"default": 30, "min": 0.1, "max": 120, "step": 0.1}), - "video_frames": ("IMAGE",), - "chord_only": ("BOOLEAN", {"default": False}), - "notes": ("STRING", {"default": ""}), - }, - } - - RETURN_TYPES = ("MIDI", "FEATURE", "FEATURE_PIPE") - FUNCTION = "process_midi" - CATEGORY = "RyanOnTheInside/Audio" - - def process_midi(self, midi_file, track_selection, notes, attribute, frame_rate, video_frames, chord_only=False): - try: - midi_path = folder_paths.get_full_path("midi_files", midi_file) - if not midi_path or not os.path.exists(midi_path): - raise FileNotFoundError(f"MIDI file not found: {midi_file}") - - midi_data = mido.MidiFile(midi_path) - - selected_notes = [int(n.strip()) for n in notes.split(',') if n.strip().isdigit()] - feature_pipe = FeaturePipe(frame_rate, video_frames) - - # Convert friendly attribute name to internal attribute name - internal_attribute = MIDIFeature.get_attribute_value(attribute) - - feature = MIDIFeature(f"midi_{internal_attribute}", midi_data, internal_attribute, - feature_pipe.frame_rate, feature_pipe.frame_count, - notes=selected_notes, chord_only=chord_only) - - feature.extract() - - return (midi_data, feature, feature_pipe) - - except Exception as e: - # error_msg = f"Error in MIDILoadAndExtract.process_midi: {type(e).__name__}: {str(e)}\n" - # error_msg += traceback.format_exc() - # print(error_msg) - raise RuntimeError(f"Error processing MIDI file: {type(e).__name__}: {str(e)}") - - @classmethod - def analyze_midi(cls, midi_path): - midi_data = mido.MidiFile(midi_path) - tracks = ["all"] - all_notes = set() - track_notes = {} - for i, track in enumerate(midi_data.tracks): - track_notes[str(i)] = set() - for msg in track: - if msg.type == 'note_on': - track_notes[str(i)].add(msg.note) - all_notes.add(msg.note) - if len(track_notes[str(i)]) == 0: - tracks.append(f"{i}: (Empty)") - else: - tracks.append(f"{i}: {track.name or f'Track {i}'}") - - return { - "tracks": tracks, - "all_notes": ",".join(map(str, sorted(set(all_notes)))), - "track_notes": {k: ",".join(map(str, sorted(v))) for k, v in track_notes.items()} - } - - @classmethod - def VALIDATE_INPUTS(cls, midi_file, track_selection, notes, attribute, frame_rate, video_frames): - if not folder_paths.exists_and_is_file(midi_file): - return "MIDI file not found: {}".format(midi_file) - - if notes != "all": - try: - note_list = [int(n.strip()) for n in notes.split(',') if n.strip()] - if not all(0 <= n <= 127 for n in note_list): - return "Invalid note value. All notes must be between 0 and 127." - except ValueError: - return "Invalid notes format. Please provide comma-separated integers or 'all'." - - return True - - -routes = PromptServer.instance.routes -@PromptServer.instance.routes.post('/get_track_notes') -async def get_track_notes(request): - data = await request.json() - midi_file = data.get('midi_file') - - if not midi_file: - return web.json_response({"error": "Missing required parameters"}, status=400) - - midi_path = folder_paths.get_full_path("midi_files", midi_file) - if not midi_path or not os.path.exists(midi_path): - return web.json_response({"error": "MIDI file not found"}, status=404) - - analysis = MIDILoadAndExtract.analyze_midi(midi_path) - return web.json_response(analysis) - -@routes.post('/upload_midi') -async def upload_midi(request): - data = await request.post() - midi_file = data['file'] - - if midi_file and midi_file.filename: - safe_filename = os.path.basename(midi_file.filename) - - midi_dir = folder_paths.get_folder_paths("midi_files")[0] - - if not os.path.exists(midi_dir): - os.makedirs(midi_dir, exist_ok=True) - - midi_path = os.path.join(midi_dir, safe_filename) - - with open(midi_path, 'wb') as f: - shutil.copyfileobj(midi_file.file, f) - - midi_files = folder_paths.get_filename_list("midi_files") - analysis = MIDILoadAndExtract.analyze_midi(midi_path) - - return web.json_response({ - "status": "success", - "uploaded_file": safe_filename, - "midi_files": midi_files, - "analysis": analysis - }) - else: - return web.json_response({"status": "error", "message": "No file uploaded"}, status=400) - -@PromptServer.instance.routes.post('/refresh_midi_data') -async def refresh_midi_data(request): - data = await request.json() - midi_file = data.get('midi_file') - track_selection = data.get('track_selection') - - if not midi_file: - return web.json_response({"error": "Missing required parameters"}, status=400) - - midi_path = folder_paths.get_full_path("midi_files", midi_file) - if not midi_path or not os.path.exists(midi_path): - return web.json_response({"error": "MIDI file not found"}, status=404) - - analysis = MIDILoadAndExtract.analyze_midi(midi_path) - - # Filter notes based on track selection - if track_selection != "all": - track_index = track_selection.split(':')[0] - analysis['all_notes'] = analysis['track_notes'].get(track_index, "") - - return web.json_response(analysis) \ No newline at end of file diff --git a/nodes/flex/proximity_feature.py b/nodes/flex/proximity_feature.py deleted file mode 100644 index 281646c..0000000 --- a/nodes/flex/proximity_feature.py +++ /dev/null @@ -1,92 +0,0 @@ -from .features import BaseFeature -import torch -import numpy as np - -import numpy as np -from .features import BaseFeature - -class ProximityFeature(BaseFeature): - def __init__(self, name, anchor_locations, query_locations, frame_rate, frame_count, frame_dimensions, normalization_method='frame'): - super().__init__(name, "proximity", frame_rate, frame_count) - self.anchor_locations = anchor_locations - self.query_locations = query_locations - self.frame_diagonal = np.sqrt(frame_dimensions[0]**2 + frame_dimensions[1]**2) - self.proximity_values = None - self.normalization_method = normalization_method - - def extract(self): - from scipy.spatial.distance import cdist - - proximities = [] - for anchor, query in zip(self.anchor_locations, self.query_locations): - if len(anchor) == 0 or len(query) == 0: - proximities.append(self.frame_diagonal if self.normalization_method == 'frame' else float('inf')) - else: - distances = cdist(anchor.points, query.points) - min_distance = np.min(distances) - proximities.append(min_distance) - - proximities = np.array(proximities) - - if self.normalization_method == 'frame': - self.proximity_values = 1 - np.clip(proximities / self.frame_diagonal, 0, 1) - elif self.normalization_method == 'minmax': - min_proximity = np.min(proximities) - max_proximity = np.max(proximities) - if max_proximity > min_proximity: - self.proximity_values = 1 - (proximities - min_proximity) / (max_proximity - min_proximity) - else: - self.proximity_values = np.ones_like(proximities) - else: - raise ValueError(f"Unknown normalization method: {self.normalization_method}") - - self.smooth_proximities() - return self - - def smooth_proximities(self, window_size=5): - kernel = np.ones(window_size) / window_size - self.proximity_values = np.convolve(self.proximity_values, kernel, mode='same') - - def get_value_at_frame(self, frame_index): - return self.proximity_values[frame_index] - -class Location: - def __init__(self, x, y, z=None): - x = np.asarray(x).reshape(-1) - y = np.asarray(y).reshape(-1) - - if z is not None: - z = np.asarray(z).reshape(-1) - # Ensure z has the same length as x and y, with a small tolerance for floating-point imprecision - if abs(len(z) - len(x)) <= 1: # Allow for off-by-one errors - z = z[:len(x)] if len(z) > len(x) else np.pad(z, (0, len(x) - len(z)), 'constant', constant_values=np.nan) - elif len(z) != len(x): - raise ValueError(f"Mismatch in dimensions: x,y have length {len(x)}, but z has length {len(z)}") - self.points = np.column_stack((x, y, z)) - else: - self.points = np.column_stack((x, y)) - - def __len__(self): - return len(self.points) - - def __getitem__(self, idx): - return self.points[idx] - - @property - def x(self): - return self.points[:, 0] - - @property - def y(self): - return self.points[:, 1] - - @property - def z(self): - return self.points[:, 2] if self.points.shape[1] > 2 else None - - @classmethod - def from_tensor(cls, tensor): - return cls(tensor[:, 0], tensor[:, 1], tensor[:, 2] if tensor.shape[1] > 2 else None) - - def to_tensor(self): - return torch.from_numpy(self.points) \ No newline at end of file diff --git a/nodes/flex/proximity_feature_extractor.py b/nodes/flex/proximity_feature_extractor.py deleted file mode 100644 index e8d75b6..0000000 --- a/nodes/flex/proximity_feature_extractor.py +++ /dev/null @@ -1,279 +0,0 @@ -from .proximity_feature import Location, ProximityFeature -from .feature_extractors import FirstFeature -from ... import RyanOnTheInside -from .feature_pipe import FeaturePipe -import numpy as np -import cv2 -import torch -from scipy.spatial.distance import cdist # Add this import - -class ProximityFeatureNode(FirstFeature): - @classmethod - def INPUT_TYPES(cls): - return { - "required": { - "video_frames": ("IMAGE",), - "frame_rate": ("FLOAT", {"default": 30.0, "min": 1.0, "max": 120.0, "step": 0.1}), - "anchor_locations": ("LOCATION",), - "query_locations": ("LOCATION",), - "normalization_method": (["frame", "minmax"],), - } - } - - RETURN_TYPES = ("FEATURE", "FEATURE_PIPE") - RETURN_NAMES = ("proximity_feature", "feature_pipe") - FUNCTION = "create_feature" - - def create_feature(self, video_frames, frame_rate, anchor_locations, query_locations, normalization_method): - feature_pipe = FeaturePipe(frame_rate, video_frames) - frame_dimensions = (video_frames.shape[2], video_frames.shape[1]) # width, height - - proximity_feature = ProximityFeature( - name="proximity_feature", - anchor_locations=anchor_locations, - query_locations=query_locations, - frame_rate=frame_rate, - frame_count=len(video_frames), - frame_dimensions=frame_dimensions, - normalization_method=normalization_method - ) - - # Ensure anchor and query points have the same number of columns - anchor_dim = anchor_locations[0].points.shape[1] - query_dim = query_locations[0].points.shape[1] - - if anchor_dim != query_dim: - if anchor_dim == 3: - for location in anchor_locations: - location.points = location.points[:, :2] - elif query_dim == 3: - for location in query_locations: - location.points = location.points[:, :2] - print("Depth ignored. Depth must be included for all points or no points.") - - # self.start_progress(len(video_frames), desc="Calculating proximity") - proximity_feature.extract() - # self.end_progress() - - return (proximity_feature, feature_pipe) - -class ProximityFeatureInput(RyanOnTheInside): - CATEGORY="RyanOnTheInside/Proximity" - -class LocationFromMask(ProximityFeatureInput): - @classmethod - def INPUT_TYPES(cls): - return { - "required": { - "masks": ("MASK",), - "method": (["mask_boundary", "mask_center"],), - }, - "optional": { - "depth_maps": ("IMAGE",), - } - } - - RETURN_TYPES = ("LOCATION",) - FUNCTION = "generate_locations" - - def generate_locations(self, masks, method, depth_maps=None): - locations = [] - for i in range(masks.shape[0]): - mask = masks[i].cpu().numpy() - if np.sum(mask) == 0: # Check if the mask is empty - print(f"No mask found in frame {i}.") - locations.append(Location(np.array([-1]), np.array([-1]), np.array([-1]) if depth_maps is not None else None)) - continue - - if method == "mask_boundary": - contours, _ = cv2.findContours(mask.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - if contours: - boundary = np.vstack(contours).squeeze() - else: - boundary = np.array([[0, 0]]) # Fallback to a single point - elif method == "mask_center": - y, x = np.where(mask > 0.5) - if len(x) > 0 and len(y) > 0: - center_x, center_y = np.mean(x), np.mean(y) - boundary = np.array([[center_x, center_y]]) - else: - boundary = np.array([[mask.shape[1] / 2, mask.shape[0] / 2]]) - - if depth_maps is not None: - depth_map = depth_maps[i].cpu().numpy() - # Ensure boundary points are within the depth map dimensions - valid_points = (boundary[:, 0] < depth_map.shape[1]) & (boundary[:, 1] < depth_map.shape[0]) - boundary = boundary[valid_points] - - if boundary.size == 0: # Check if boundary is empty after filtering - print(f"No valid boundary points in frame {i}.") - locations.append(Location(np.array([-1]), np.array([-1]), np.array([-1]))) - continue - - # Extract depth values for valid boundary points - z = np.mean(depth_map[boundary[:, 1].astype(int), boundary[:, 0].astype(int)], axis=-1) - - location = Location(boundary[:, 0], boundary[:, 1], z) - else: - location = Location(boundary[:, 0], boundary[:, 1]) - - locations.append(location) - - return (locations,) - -class LocationFromPoint(ProximityFeatureInput): - @classmethod - def INPUT_TYPES(cls): - return { - "required": { - "x": ("FLOAT", {"default": 0.0,"min":0.0,"step":0.01}), - "y": ("FLOAT", {"default": 0.0,"min":0.0,"step":0.01}), - "z": ("FLOAT", {"default": 0.0, "max":1.0, "min":0.0,"step":0.01}), - "batch_count": ("INT", {"default": 1, "min": 1}), - } - } - - RETURN_TYPES = ("LOCATION",) - FUNCTION = "generate_locations" - - def generate_locations(self, x, y, batch_count, z): - locations = [] - for _ in range(batch_count): - location = Location(np.array([x]), np.array([y]), np.array([z])) - locations.append(location) - - return (locations,) - -class LocationTransform(ProximityFeatureInput): - @classmethod - def INPUT_TYPES(cls): - return { - "required": { - "locations": ("LOCATION",), - "feature": ("FEATURE",), - "transformation_type": (["translate", "scale"],), - "transformation_value": ("FLOAT", {"default": 1.0}), - } - } - - RETURN_TYPES = ("LOCATION",) - FUNCTION = "transform_locations" - - def transform_locations(self, locations, feature, transformation_type, transformation_value): - transformed_locations = [] - for frame_index, location in enumerate(locations): - feature_value = feature.get_value_at_frame(frame_index) - if transformation_type == "translate": - new_x = location.x + feature_value * transformation_value - new_y = location.y + feature_value * transformation_value - if location.z is not None: - new_z = location.z + feature_value * transformation_value - transformed_location = Location(new_x, new_y, new_z) - else: - transformed_location = Location(new_x, new_y) - elif transformation_type == "scale": - new_x = location.x * feature_value * transformation_value - new_y = location.y * feature_value * transformation_value - if location.z is not None: - new_z = location.z * feature_value * transformation_value - transformed_location = Location(new_x, new_y, new_z) - else: - transformed_location = Location(new_x, new_y) - transformed_locations.append(transformed_location) - - return (transformed_locations,) - - - -class ProximityVisualizer(RyanOnTheInside): - @classmethod - def INPUT_TYPES(cls): - return { - "required": { - "video_frames": ("IMAGE",), - "anchor_locations": ("LOCATION",), - "query_locations": ("LOCATION",), - "proximity_feature": ("FEATURE",), - "anchor_color": ("STRING", {"default": "(255,0,0)"}), - "query_color": ("STRING", {"default": "(0,255,0)"}), - "line_color": ("STRING", {"default": "(0,0,255)"}), - "text_color": ("STRING", {"default": "(255,255,255)"}), - "font_scale": ("FLOAT", {"default": 0.5, "min": 0.1, "max": 2.0, "step": 0.1}), - } - } - - RETURN_TYPES = ("IMAGE",) - FUNCTION = "visualize_proximity" - - def visualize_proximity(self, video_frames, anchor_locations, query_locations, proximity_feature, - anchor_color, query_color, line_color, text_color, font_scale): - anchor_color = self.parse_color(anchor_color) - query_color = self.parse_color(query_color) - line_color = self.parse_color(line_color) - text_color = self.parse_color(text_color) - - output_frames = [] - height, width = video_frames.shape[1:3] # Extract height and width from video_frames - - # Calculate the frame diagonal from the video frames - frame_diagonal = np.sqrt(width**2 + height**2) - scale_factor = frame_diagonal / proximity_feature.frame_diagonal - for frame_index in range(len(video_frames)): - frame = video_frames[frame_index].cpu().numpy() - frame = (frame * 255).astype(np.uint8) - frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) - - anchor = anchor_locations[frame_index] - query = query_locations[frame_index] - - # Calculate scaling factors based on the frame diagonal - - - # Draw all anchor and query points (using only x and y coordinates) - for point in anchor: - scaled_point = (int(point[0] * scale_factor), int(point[1] * scale_factor)) - cv2.circle(frame, scaled_point, 2, anchor_color, -1) - for point in query: - scaled_point = (int(point[0] * scale_factor), int(point[1] * scale_factor)) - cv2.circle(frame, scaled_point, 2, query_color, -1) - - # Find the closest pair of points - if len(anchor) > 0 and len(query) > 0: - distances = cdist(anchor.points[:, :2], query.points[:, :2]) # Use only x and y for distance calculation - min_idx = np.unravel_index(distances.argmin(), distances.shape) - closest_anchor = anchor[min_idx[0]] - closest_query = query[min_idx[1]] - - # Draw line between closest points (using only x and y coordinates) - cv2.line(frame, - (int(closest_anchor[0] * scale_factor), int(closest_anchor[1] * scale_factor)), - (int(closest_query[0] * scale_factor), int(closest_query[1] * scale_factor)), - line_color, 2) - - # Display coordinates of closest points - anchor_text = f"Anchor: ({int(closest_anchor[0] * scale_factor)}, {int(closest_anchor[1] * scale_factor)}" - query_text = f"Query: ({int(closest_query[0] * scale_factor)}, {int(closest_query[1] * scale_factor)}" - - if anchor.z is not None and query.z is not None: - anchor_text += f", {closest_anchor[2]:.2f})" - query_text += f", {closest_query[2]:.2f})" - else: - anchor_text += ") [2D]" - query_text += ") [2D]" - - cv2.putText(frame, anchor_text, (10, 60), cv2.FONT_HERSHEY_SIMPLEX, font_scale, text_color, 1) - cv2.putText(frame, query_text, (10, 90), cv2.FONT_HERSHEY_SIMPLEX, font_scale, text_color, 1) - - # Add proximity value to the frame - proximity_value = proximity_feature.get_value_at_frame(frame_index) - cv2.putText(frame, f"Proximity: {proximity_value:.2f}", (10, 30), - cv2.FONT_HERSHEY_SIMPLEX, 1, text_color, 2) - - frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - output_frames.append(frame) - - output_tensor = torch.from_numpy(np.stack(output_frames)).float() / 255.0 - return (output_tensor,) - - def parse_color(self, color_string): - return tuple(map(int, color_string.strip("()").split(","))) \ No newline at end of file diff --git a/nodes/images/image_base.py b/nodes/images/flex_image_base.py similarity index 100% rename from nodes/images/image_base.py rename to nodes/images/flex_image_base.py diff --git a/nodes/images/flex_images.py b/nodes/images/flex_images.py index 012ab7a..a3dfca4 100644 --- a/nodes/images/flex_images.py +++ b/nodes/images/flex_images.py @@ -1,7 +1,7 @@ import cv2 import torch import numpy as np -from .image_base import FlexImageBase +from .flex_image_base import FlexImageBase from scipy.ndimage import gaussian_filter import torch.nn.functional as F from .image_utils import transform_image diff --git a/nodes/latents/flex_latent_base.py b/nodes/latents/flex_latent_base.py new file mode 100644 index 0000000..5ed0905 --- /dev/null +++ b/nodes/latents/flex_latent_base.py @@ -0,0 +1,105 @@ +import numpy as np +import torch +from abc import abstractmethod +from ... import RyanOnTheInside +from ..flex.flex_base import FlexBase + +class FlexLatentBase(RyanOnTheInside, FlexBase): + @classmethod + def INPUT_TYPES(cls): + return { + **super().INPUT_TYPES(), + "required": { + **super().INPUT_TYPES()["required"], + "latents": ("LATENT",), + }, + # Optional inputs are inherited from FlexBase + } + + CATEGORY = "RyanOnTheInside/FlexLatent" + RETURN_TYPES = ("LATENT",) + FUNCTION = "apply_effect" + + def apply_effect( + self, + latents, + strength, + feature_threshold, + feature_param, + feature_mode, + opt_feature=None, + opt_feature_pipe=None, + **kwargs + ): + if (opt_feature is None) != (opt_feature_pipe is None): + raise ValueError( + "Both opt_feature and opt_feature_pipe must be provided together, or neither should be provided." + ) + + latents_np = latents["samples"].cpu().numpy() + num_frames = latents_np.shape[0] + self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}") + + result = [] + for i in range(num_frames): + latent = latents_np[i] + feature_value = 1.0 # Default feature value + apply_effect = True # Default to applying the effect + + if opt_feature is not None and opt_feature_pipe is not None: + feature_value = opt_feature.get_value_at_frame(i) + apply_effect = feature_value >= feature_threshold + + kwargs['frame_index'] = i + kwargs['feature_value'] = feature_value # Add feature_value to kwargs + kwargs['strength'] = strength # Add strength to kwargs + kwargs['feature_param'] = feature_param # Add feature_param to kwargs + kwargs['feature_mode'] = feature_mode # Add feature_mode to kwargs + + if apply_effect: + processed_latent = self.process_latent( + latent, + **kwargs + ) + else: + processed_latent = latent + + result.append(processed_latent) + self.update_progress() + + self.end_progress() + + result_np = np.stack(result) + result_tensor = torch.from_numpy(result_np).float() + + return ({"samples": result_tensor},) + + def process_latent( + self, + latent: np.ndarray, + **kwargs + ) -> np.ndarray: + feature_value = kwargs['feature_value'] + strength = kwargs['strength'] + feature_param = kwargs['feature_param'] + feature_mode = kwargs['feature_mode'] + + # Modulate the selected parameter + for param_name in self.get_modifiable_params(): + if param_name in kwargs and param_name == feature_param: + kwargs[param_name] = self.modulate_param( + param_name, + kwargs[param_name], + feature_value, + strength, + feature_mode + ) + + # Call the subclass's implementation + return self.apply_effect_internal(latent, **kwargs) + + @abstractmethod + def apply_effect_internal(self, latent: np.ndarray, **kwargs) -> np.ndarray: + """To be implemented by subclasses.""" + pass + diff --git a/nodes/latents/latent_base.py b/nodes/latents/flex_latents.py similarity index 59% rename from nodes/latents/latent_base.py rename to nodes/latents/flex_latents.py index a7da4d9..ce74ec9 100644 --- a/nodes/latents/latent_base.py +++ b/nodes/latents/flex_latents.py @@ -1,94 +1,4 @@ -# custom_nodes/ComfyUI_RyanOnTheInside/nodes/latents/latent_base.py - -import numpy as np -import torch -from abc import ABC, abstractmethod -from comfy.utils import ProgressBar -from ... import RyanOnTheInside - -#NOTE: this is a work in progress, it sucks - -class FlexLatentBase(RyanOnTheInside, ABC): - @classmethod - def INPUT_TYPES(cls): - return { - "required": { - "latents": ("LATENT",), - "feature": ("FEATURE",), - "feature_pipe": ("FEATURE_PIPE",), - "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), - "feature_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), - "feature_param": (cls.get_modifiable_params(), {"default": cls.get_modifiable_params()[0]}), - "feature_mode": (["relative", "absolute"], {"default": "relative"}), - } - } - - CATEGORY = "RyanOnTheInside/FlexLatent" - RETURN_TYPES = ("LATENT",) - FUNCTION = "apply_effect" - - def __init__(self): - self.progress_bar = None - - def start_progress(self, total_steps, desc="Processing"): - self.progress_bar = ProgressBar(total_steps) - - def update_progress(self): - if self.progress_bar: - self.progress_bar.update(1) - - def end_progress(self): - self.progress_bar = None - - @classmethod - @abstractmethod - def get_modifiable_params(cls): - """Return a list of parameter names that can be modulated.""" - return [] - - def modulate_param(self, param_name, param_value, feature_value, strength, mode): - if mode == "relative": - return param_value * (1 + (feature_value - 0.5) * strength) - else: # absolute - return param_value * feature_value * strength - - def apply_effect(self, latents, feature, feature_pipe, strength, feature_threshold, feature_param, feature_mode, **kwargs): - num_frames = feature_pipe.frame_count - latents_np = latents["samples"].cpu().numpy() - - self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}") - - result = [] - for i in range(num_frames): - latent = latents_np[i] - feature_value = feature.get_value_at_frame(i) - kwargs['frame_index'] = i - kwargs['feature_value'] = feature_value - kwargs['strength'] = strength - kwargs['feature_param'] = feature_param - kwargs['feature_mode'] = feature_mode - - if feature_value >= feature_threshold: - processed_latent = self.process_latent(latent, **kwargs) - else: - processed_latent = latent - - result.append(processed_latent) - self.update_progress() - - self.end_progress() - - result_np = np.stack(result) - result_tensor = torch.from_numpy(result_np).float() - - return ({"samples": result_tensor},) - - @abstractmethod - def process_latent(self, latent: np.ndarray, **kwargs) -> np.ndarray: - """Process the latent using subclass-specific logic.""" - -# custom_nodes/ComfyUI_RyanOnTheInside/nodes/latents/flex_latent_interpolate.py - +from .flex_latent_base import FlexLatentBase import numpy as np class FlexLatentInterpolate(FlexLatentBase): @@ -105,26 +15,14 @@ class FlexLatentInterpolate(FlexLatentBase): def get_modifiable_params(cls): return ["None"] - def process_latent(self, latent: np.ndarray, **kwargs) -> np.ndarray: + def apply_effect_internal(self, latent: np.ndarray, **kwargs) -> np.ndarray: feature_value = kwargs['feature_value'] strength = kwargs['strength'] - feature_param = kwargs['feature_param'] - feature_mode = kwargs['feature_mode'] latent_2 = kwargs['latent_2'] interpolation_mode = kwargs['interpolation_mode'] + frame_index = kwargs['frame_index'] - # Modulate any parameters if needed - if feature_param in self.get_modifiable_params(): - modulated_value = self.modulate_param( - feature_param, - kwargs.get(feature_param, 0), - feature_value, - strength, - feature_mode - ) - kwargs[feature_param] = modulated_value - - latent_2_np = latent_2["samples"].cpu().numpy()[kwargs['frame_index']] + latent_2_np = latent_2["samples"].cpu().numpy()[frame_index] # Perform interpolation t = np.clip(feature_value * strength, 0.0, 1.0) @@ -154,31 +52,6 @@ class FlexLatentInterpolate(FlexLatentBase): coef2 = np.sin(t * omega) / sin_omega result = coef1 * latent1 + coef2 * latent2 return result.reshape(latent1.shape) - -# custom_nodes/ComfyUI_RyanOnTheInside/nodes/latents/embedding_guided_latent_interpolate.py - - def spherical_interpolation(self, latent1, latent2, t): - # Flatten the latents - latent1_flat = latent1.flatten() - latent2_flat = latent2.flatten() - - # Compute dot product and norms - dot = np.dot(latent1_flat, latent2_flat) - norm1 = np.linalg.norm(latent1_flat) - norm2 = np.linalg.norm(latent2_flat) - denominator = norm1 * norm2 + 1e-8 # Avoid division by zero - omega = np.arccos(np.clip(dot / denominator, -1.0, 1.0)) - - if np.isclose(omega, 0): - return latent1 - else: - sin_omega = np.sin(omega) - coef1 = np.sin((1 - t) * omega) / sin_omega - coef2 = np.sin(t * omega) / sin_omega - result = coef1 * latent1 + coef2 * latent2 - return result.reshape(latent1.shape) - -import numpy as np class EmbeddingGuidedLatentInterpolate(FlexLatentBase): @classmethod @@ -196,28 +69,15 @@ class EmbeddingGuidedLatentInterpolate(FlexLatentBase): def get_modifiable_params(cls): return ["interpolation_mode", "None"] - def process_latent(self, latent: np.ndarray, **kwargs) -> np.ndarray: + def apply_effect_internal(self, latent: np.ndarray, **kwargs) -> np.ndarray: feature_value = kwargs['feature_value'] strength = kwargs['strength'] - feature_param = kwargs['feature_param'] - feature_mode = kwargs['feature_mode'] latent_2 = kwargs['latent_2'] embedding_1 = kwargs['embedding_1'] embedding_2 = kwargs['embedding_2'] interpolation_mode = kwargs['interpolation_mode'] frame_index = kwargs['frame_index'] - # Modulate any parameters if needed - if feature_param in self.get_modifiable_params(): - modulated_value = self.modulate_param( - feature_param, - kwargs.get(feature_param, 0), - feature_value, - strength, - feature_mode - ) - kwargs[feature_param] = modulated_value - latent_2_np = latent_2["samples"].cpu().numpy()[frame_index] embedding_1_np = embedding_1.cpu().numpy()[frame_index] embedding_2_np = embedding_2.cpu().numpy()[frame_index] @@ -261,7 +121,6 @@ class EmbeddingGuidedLatentInterpolate(FlexLatentBase): coef2 = np.sin(t * omega) / sin_omega result = coef1 * latent1 + coef2 * latent2 return result.reshape(latent1.shape) - class FlexLatentBlend(FlexLatentBase): @classmethod @@ -278,7 +137,7 @@ class FlexLatentBlend(FlexLatentBase): def get_modifiable_params(cls): return ["blend_strength", "None"] - def process_latent(self, latent: np.ndarray, **kwargs) -> np.ndarray: + def apply_effect_internal(self, latent: np.ndarray, **kwargs) -> np.ndarray: feature_value = kwargs['feature_value'] strength = kwargs['strength'] feature_param = kwargs['feature_param'] @@ -323,7 +182,7 @@ class FlexLatentBlend(FlexLatentBase): else: # Default to Add if mode is unrecognized return latent1 + latent2 - + class FlexLatentNoise(FlexLatentBase): @classmethod def INPUT_TYPES(cls): @@ -338,7 +197,7 @@ class FlexLatentNoise(FlexLatentBase): def get_modifiable_params(cls): return ["noise_level", "None"] - def process_latent(self, latent: np.ndarray, **kwargs) -> np.ndarray: + def apply_effect_internal(self, latent: np.ndarray, **kwargs) -> np.ndarray: feature_value = kwargs['feature_value'] strength = kwargs['strength'] feature_param = kwargs['feature_param'] @@ -355,7 +214,7 @@ class FlexLatentNoise(FlexLatentBase): strength, feature_mode ) - # Ensure noise_level remains within [0, 1] + # Ensure noise_level remains within [0.0, 1.0] noise_level = np.clip(noise_level, 0.0, 1.0) # Generate noise @@ -369,4 +228,5 @@ class FlexLatentNoise(FlexLatentBase): # Add noise to the latent result = latent + noise - return result \ No newline at end of file + return result + From 9cf6d2f20d97465ec1b66338cc035e3ea9fc3301 Mon Sep 17 00:00:00 2001 From: ryanontheinstide Date: Sat, 26 Oct 2024 14:20:19 -0400 Subject: [PATCH 10/10] accurate wording --- nodes/masks/flex_mask_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes/masks/flex_mask_base.py b/nodes/masks/flex_mask_base.py index bcbab02..7f257ec 100644 --- a/nodes/masks/flex_mask_base.py +++ b/nodes/masks/flex_mask_base.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod import numpy as np import torch -#TODO SOON TO UPDATE TO SUB FLEXBASE. UNTIL THEN REFER TO ANYOTHER TARGET (images, audio, etc) FOR EXAMPLES ON HOW TO EXTEND +#TODO EVENTUALLY TO UPDATE TO SUB FLEXBASE. UNTIL THEN REFER TO ANYOTHER TARGET (images, audio, etc) FOR EXAMPLES ON HOW TO EXTEND class FlexMaskBase(MaskBase): feature_threshold_default=0.0 @classmethod