From 37f99600d84a48640d090ab47c7d5f417eaeb780 Mon Sep 17 00:00:00 2001 From: RyanOnTheInside <7623207+ryanontheinside@users.noreply.github.com> Date: Tue, 3 Sep 2024 16:12:30 -0400 Subject: [PATCH] Squash --- __init__.py | 6 ++ node_configs.py | 45 +++++++++- nodes/flex/feature_modulation.py | 43 +++++++++- nodes/masks/flex_masks.py | 137 +++++++++++++++++++++++++++++++ nodes/utility_nodes.py | 4 +- pyproject.toml | 2 +- readme.md | 11 ++- 7 files changed, 238 insertions(+), 10 deletions(-) diff --git a/__init__.py b/__init__.py index 9bdcb18..dd60669 100644 --- a/__init__.py +++ b/__init__.py @@ -131,6 +131,8 @@ from .nodes.masks.flex_masks import ( FlexMaskWavePropagation, FlexMaskEmanatingRings, FlexMaskRandomShapes, + FlexMaskDepthChamber, + # FlexMaskDepthChamberRelative, #NOTE work in progress ) @@ -156,6 +158,7 @@ from .nodes.flex.feature_modulation import ( FeatureOscillator, FeatureScaler, FeatureSmoothing, + FeatureFade, ) @@ -212,6 +215,8 @@ NODE_CLASS_MAPPINGS = { "FlexMaskWavePropagation": FlexMaskWavePropagation, "FlexMaskEmanatingRings": FlexMaskEmanatingRings, "FlexMaskRandomShapes": FlexMaskRandomShapes, + "FlexMaskDepthChamber": FlexMaskDepthChamber, + # "FlexMaskDepthChamberRelative": FlexMaskDepthChamberRelative, #audio "AudioSeparator": AudioSeparator, @@ -237,6 +242,7 @@ NODE_CLASS_MAPPINGS = { "FeatureOscillator": FeatureOscillator, "FeatureScaler": FeatureScaler, "FeatureSmoothing": FeatureSmoothing, + "FeatureFade": FeatureFade, #images 'FlexImageEdgeDetect': FlexImageEdgeDetect, diff --git a/node_configs.py b/node_configs.py index e8fef35..6faa6dd 100644 --- a/node_configs.py +++ b/node_configs.py @@ -322,6 +322,20 @@ add_node_config("FlexMaskBase", { """ }) +add_node_config("FlexMaskDepthChamber", { + "TOP_DESCRIPTION": "Applies a depth-based mask modulation using a depth map and specified front and back depth values.", + "ADDITIONAL_INFO": """ +- `depth_map`: Input depth map (IMAGE type) +- `z_front`: Front depth value for the mask (0.0 to 1.0). Default is 1.0. +- `z_back`: Back depth value for the mask (0.0 to 1.0). Default is 0.0. +- `feature_param`: Parameter to modulate based on the feature. Options are "none", "z_front", "z_back", "both". +- `feature_mode`: Mode of feature modulation. Options are "squeeze" and "expand". + +This node creates a mask based on the depth values in the input depth map. The mask is modulated by the specified front and back depth values, and can be further adjusted using a feature input to dynamically change the depth range. +""" +}) + + add_node_config("FlexMaskMorph", { "TOP_DESCRIPTION": "Applies morphological operations to the mask, modulated by a selected feature.", "ADDITIONAL_INFO": """ @@ -755,6 +769,18 @@ add_node_config("FeatureOscillator", { """ }) +add_node_config("FeatureFade", { + "TOP_DESCRIPTION": "Fades between two features based on a fader value or a control feature.", + "ADDITIONAL_INFO": """ +- `feature1`: First input feature to fade from (FEATURE type) +- `feature2`: Second input feature to fade to (FEATURE type) +- `fader`: Static fader value to control the blend between feature1 and feature2 (0.0 to 1.0). 0.0 is 100 percent feature1, 1.0 is 100 percent feature2. +- `control_feature`: Optional feature to dynamically control the fader value (FEATURE type). If provided, the fader value will be modulated by this feature. + +Shoutout @cyncratic +""" +}) + add_node_config("FeatureToWeightsStrategy", { "TOP_DESCRIPTION": "Converts a FEATURE input into a WEIGHTS_STRATEGY for use with IPAdapter nodes.", "ADDITIONAL_INFO": """ @@ -859,7 +885,7 @@ add_node_config("UtilityNode",{ "BASE_DESCRIPTION":"Various Utils" }) -add_node_config("ImageInterval", { +add_node_config("ImageIntervalSelect", { "TOP_DESCRIPTION": "Selects images from a sequence at specified intervals.", "ADDITIONAL_INFO": """ - `image`: Input image sequence (IMAGE type) @@ -869,7 +895,7 @@ add_node_config("ImageInterval", { """ }) -add_node_config("ImageChunk", { +add_node_config("ImageChunks", { "TOP_DESCRIPTION": "Concatenates images into a grid.", "ADDITIONAL_INFO": """ - `image`: Input image sequence (IMAGE type) @@ -880,8 +906,8 @@ add_node_config("ImageChunk", { """ }) -add_node_config("VideoChunk", { - "TOP_DESCRIPTION": "Chunks images into grids for video processing.", +add_node_config("VideoChunks", { + "TOP_DESCRIPTION": "Chunks images into grids.", "ADDITIONAL_INFO": """ - `image`: Input image sequence (IMAGE type) - `chunk_size`: Number of images per grid (default: 4, min: 1) @@ -892,6 +918,17 @@ add_node_config("VideoChunk", { """ }) +add_node_config("SwapDevice", { + "TOP_DESCRIPTION": "Transfers the image and mask tensors to the specified device (CPU or CUDA).", + "ADDITIONAL_INFO": """ +- `device`: The target device to transfer the tensors to. Options are "cpu" or "cuda". +- `image`: (Optional) The image tensor to transfer. If not provided, a zero tensor is created. +- `mask`: (Optional) The mask tensor to transfer. If not provided, a zero tensor is created. + +This node checks if the specified device is available and transfers the image and mask tensors to that device. If the device is not available, it raises a ValueError. +""" +}) + add_node_config("ImageShuffle", { "TOP_DESCRIPTION": "Shuffles images in groups of specified size.", "ADDITIONAL_INFO": """ diff --git a/nodes/flex/feature_modulation.py b/nodes/flex/feature_modulation.py index 185c79c..0a2c796 100644 --- a/nodes/flex/feature_modulation.py +++ b/nodes/flex/feature_modulation.py @@ -105,7 +105,7 @@ class FeatureMixer(FeatureModulationBase): "release": ("FLOAT", {"default": 1.0, "min": 0.01, "max": 1.0, "step": 0.01}), "smoothing": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), "feature_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), - "rise_detection_threshold": ("FLOAT", {"default": 0.3, "min": 0.0, "max": 1.0, "step": 0.05}), + "rise_detection_threshold": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.05}), "rise_smoothing_factor": ("FLOAT", {"default": 0.5, "min": 0.1, "max": 2.0, "step": 0.05}), **super().INPUT_TYPES()["required"], # Include the invert_output option } @@ -338,3 +338,44 @@ class FeatureOscillator(FeatureModulationBase): processed_feature = self.create_processed_feature(feature, blended, "Oscillated", invert_output) return (processed_feature, self.visualize(processed_feature)) + + +#NOTE separated from FeatureMath for ease of use. +class FeatureFade(FeatureModulationBase): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "feature1": ("FEATURE",), + "feature2": ("FEATURE",), + "fader": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + **super().INPUT_TYPES()["required"], + }, + "optional": { + "control_feature": ("FEATURE",), + } + } + + RETURN_TYPES = ("FEATURE", "IMAGE") + RETURN_NAMES = ("FEATURE", "FEATURE_VISUALIZATION") + + def modulate(self, feature1, feature2, fader, invert_output, control_feature=None): + values1 = [feature1.get_value_at_frame(i) for i in range(feature1.frame_count)] + values2 = [feature2.get_value_at_frame(i) for i in range(feature2.frame_count)] + + # Ensure both features have the same length + min_length = min(len(values1), len(values2)) + values1 = values1[:min_length] + values2 = values2[:min_length] + + if control_feature: + control_values = [control_feature.get_value_at_frame(i) for i in range(control_feature.frame_count)] + control_values = control_values[:min_length] + fader_values = [(v - min(control_values)) / (max(control_values) - min(control_values)) if max(control_values) > min(control_values) else 0.5 for v in control_values] + else: + fader_values = [fader] * min_length + + combined = [(1 - f) * v1 + f * v2 for v1, v2, f in zip(values1, values2, fader_values)] + + processed_feature = self.create_processed_feature(feature1, combined, "Faded", invert_output) + return (processed_feature, self.visualize(processed_feature)) \ No newline at end of file diff --git a/nodes/masks/flex_masks.py b/nodes/masks/flex_masks.py index 8355307..06589e7 100644 --- a/nodes/masks/flex_masks.py +++ b/nodes/masks/flex_masks.py @@ -591,6 +591,143 @@ class FlexMaskRandomShapes(FlexMaskBase): easing_function=easing_function, shape_type=shape_type, feature_param=feature_param, **kwargs),) + +import torch +class FlexMaskDepthChamber(FlexMaskBase): + @classmethod + def INPUT_TYPES(cls): + return { + **super().INPUT_TYPES(), + "required": { + **super().INPUT_TYPES()["required"], + "depth_map": ("IMAGE",), + "z_front": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), + "z_back": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), + "feature_param": (["none", "z_front", "z_back", "both"],), + "feature_mode": (["squeeze", "expand"],), + } + } + + def process_mask(self, mask: np.ndarray, feature_value: float, strength: float, + depth_map: torch.Tensor, z_front: float, z_back: float, feature_param: str, + feature_mode: str, **kwargs) -> np.ndarray: + frame_index = kwargs.get('frame_index', 0) + depth_map_frame = depth_map[frame_index].cpu().numpy() + + depth_map_frame = depth_map_frame[:, :, 0] + + # Adjust z_front and z_back based on feature_mode and feature_param + if feature_param != "none": + if feature_mode == "squeeze": + if feature_param in ["z_front", "both"]: + z_front = z_front - (z_front - z_back) * strength * feature_value / 2 if z_front > z_back else z_front + (z_back - z_front) * strength * feature_value / 2 + if feature_param in ["z_back", "both"]: + z_back = z_back + (z_front - z_back) * strength * feature_value / 2 if z_back < z_front else z_back - (z_back - z_front) * strength * feature_value / 2 + elif feature_mode == "expand": + if feature_param in ["z_front", "both"]: + z_front = min(1.0, z_front + (z_front - z_back) * strength * feature_value / 2) if z_front > z_back else max(0.0, z_front - (z_back - z_front) * strength * feature_value / 2) + if feature_param in ["z_back", "both"]: + z_back = max(0.0, z_back - (z_front - z_back) * strength * feature_value / 2) if z_back < z_front else min(1.0, z_back + (z_back - z_front) * strength * feature_value / 2) + + # Create the depth mask + if z_back < z_front: + depth_mask = (depth_map_frame >= z_back) & (depth_map_frame <= z_front) + else: + depth_mask = (depth_map_frame >= z_back) | (depth_map_frame <= z_front) + + depth_mask_resized = cv2.resize(depth_mask.astype(np.float32), (mask.shape[1], mask.shape[0])) + + return depth_mask_resized + + def main_function(self, masks, feature, feature_pipe, strength, feature_threshold, + invert, subtract_original, grow_with_blur, depth_map, z_front, z_back, + feature_param, feature_mode, **kwargs): + return (self.apply_mask_operation(masks, feature, feature_pipe, strength, + feature_threshold, invert, subtract_original, + grow_with_blur, depth_map=depth_map, z_front=z_front, z_back=z_back, + feature_param=feature_param, feature_mode=feature_mode, + **kwargs),) + +from typing import List +import torch.nn.functional as F +class FlexMaskDepthChamberRelative(FlexMaskBase): + @classmethod + def INPUT_TYPES(cls): + return { + **super().INPUT_TYPES(), + "required": { + **super().INPUT_TYPES()["required"], + "depth_map": ("IMAGE",), + "z1": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + "z2": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + "feature_param": (["none", "z1", "z2", "both"],), + "feature_mode": (["squeeze", "expand"],), + } + } + + def calculate_roi_size(self, mask: torch.Tensor) -> float: + # Calculate the bounding box of the mask + y_indices, x_indices = torch.where(mask > 0) + if len(y_indices) == 0 or len(x_indices) == 0: + return 0.0 + height = y_indices.max().item() - y_indices.min().item() + width = x_indices.max().item() - x_indices.min().item() + return height * width + + def calculate_reference_size(self, masks: List[torch.Tensor]) -> float: + # Calculate the mean or median size of the ROI across all frames + sizes = [self.calculate_roi_size(mask) for mask in masks] + return torch.median(torch.tensor(sizes)).item() # or torch.mean(torch.tensor(sizes)).item() + + def process_mask(self, mask: torch.Tensor, feature_value: float, strength: float, + depth_map: torch.Tensor, z1: float, z2: float, feature_param: str, + feature_mode: str, reference_size: float, **kwargs) -> torch.Tensor: + frame_index = kwargs.get('frame_index', 0) + depth_map_frame = depth_map[frame_index, :, :, 0] + + # Calculate the ROI size for the current frame + roi_size = self.calculate_roi_size(mask) + + if feature_param == "z1": + z1 = z1 * (roi_size / reference_size) + elif feature_param == "z2": + z2 = z2 * (roi_size / reference_size) + elif feature_param == "both": + z1 = z1 * (roi_size / reference_size) + z2 = z2 * (roi_size / reference_size) + + # Ensure z1 is less than z2 + z1, z2 = min(z1, z2), max(z1, z2) + + if feature_mode == "squeeze": + depth_mask = (depth_map_frame >= z1) & (depth_map_frame <= z2) + elif feature_mode == "expand": + depth_mask = (depth_map_frame < z1) | (depth_map_frame > z2) + + depth_mask_resized = F.interpolate(depth_mask.unsqueeze(0).unsqueeze(0).float(), size=mask.shape[-2:], mode='nearest').squeeze(0).squeeze(0) + + output_mask = mask.float() * depth_mask_resized + + return output_mask + + def main_function(self, masks, feature, feature_pipe, strength, feature_threshold, + invert, subtract_original, grow_with_blur, depth_map, z1, z2, + feature_param, feature_mode, **kwargs): + reference_size = self.calculate_reference_size(masks) + + output_masks = [] + for frame_index, mask in enumerate(masks): + output_mask = self.process_mask(mask, feature_value=1.0, strength=1.0, + depth_map=depth_map, z1=z1, z2=z2, + feature_param=feature_param, feature_mode=feature_mode, + reference_size=reference_size, frame_index=frame_index) + output_masks.append(output_mask) + + # Stack the list of tensors into a single tensor + output_masks_tensor = torch.stack(output_masks) + + return (output_masks_tensor,) + #TODO # class FlexMaskVoronoiShape(FlexMaskBase): # @classmethod diff --git a/nodes/utility_nodes.py b/nodes/utility_nodes.py index 46898fa..430a3a2 100644 --- a/nodes/utility_nodes.py +++ b/nodes/utility_nodes.py @@ -3,9 +3,9 @@ import torch import torchvision from .. import RyanOnTheInside from abc import ABC - + class UtilityNode(RyanOnTheInside, ABC): - CATEGORY="UTILITY" + CATEGORY="RyanOnTheInside/Utility" class ImageIntervalSelect(UtilityNode): diff --git a/pyproject.toml b/pyproject.toml index 1ae9ce5..24b178d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui_ryanonyheinside" description = "Custom nodes introducing particle simulations, optical flow, audio manipulation & reactivity, and temporal masks" -version = "1.5.1" +version = "1.5.3" license = {file = "LICENSE"} dependencies = ["opencv-python==4.10.0","scipy","pillow","librosa","pymunk==6.8.1","matplotlib","openunmix","mido"] diff --git a/readme.md b/readme.md index e377d00..dbe2a60 100644 --- a/readme.md +++ b/readme.md @@ -1,7 +1,8 @@ # ComfyUI RyanOnTheInside Node Pack ## Overview -RyanOnTheInside node pack introduces dynamic features to ComfyUI: +### RyanOnTheInside node pack introduces dynamic features to ComfyUI: + **Dynamic Reactivity**: Almost all nodes in this pack can be made to react to audio, MIDI, motion, time, color, depth, brightness, and more, allowing for incredibly dynamic and responsive workflows.** - 🎆 **Particle Systems**: Create mesmerizing, fluid-like effects - 🎵 **Audio & MIDI Processing**: Separate instruments and create audio-reactive visuals @@ -9,6 +10,9 @@ RyanOnTheInside node pack introduces dynamic features to ComfyUI: - 🌊 **Optical Flow**: Generate masks based on motion in videos - ⏳ **Temporal Effects**: Apply time-based transformations to masks and images + + + @@ -29,6 +33,9 @@ RyanOnTheInside node pack introduces dynamic features to ComfyUI: *Examples showcasing various effects using particle emitters, vortices, and other node features* ##### 🆕 Recent Updates +- 9/3/24 - **FlexMaskDepthChamber**: Mask anything within a given depth range +- 9/3/24 - **FeatureFade**: another feature modifier to allow mixing of features, controled by features. Think mixing depth and motion features according to a kick drum or somthing. Ridiculous. +- 9/1/24 - **Utility Nodes**: Added many utility nodes for batch manipulation and more - 8/31/24 - **FlexMaskRandomShapes**: Create dynamic shapes. Cooler than it sounds! - 8/30/24 - **FlexMaskEmanatingRings**: Create dynamic, expanding ring patterns emanating from mask edges. - 8/30/24 - **FlexMaskWavePropagation**: Simulate wave-like distortions propagating from mask boundaries. @@ -50,7 +57,7 @@ For a range of example and tutorial workflows showcasing the capabilities of thi ## ❓ Help and Documentation -For detailed information on each and every node, click the ❓ icon present in the top-right corner of every node. +For detailed information on each and every node, click the ❓ icon present in the top-right corner of the node. ![Help and Documentation](./assets/help.png)
RyanOnTheInside particle demo header