From 5ee2ed45a98fab422b8be76b4f1c404cc2eac045 Mon Sep 17 00:00:00 2001 From: ryanontheinstide Date: Sun, 18 Aug 2024 21:37:29 -0400 Subject: [PATCH] Audio reactivity & manipulation --- __init__.py | 15 ++- nodes/audio/audio_nodes.py | 179 +++++++++++++++++++++++++-- nodes/audio/audio_utils.py | 2 +- nodes/masks/audio_masks.py | 70 +---------- nodes/masks/mask_base.py | 74 ++++++++++- nodes/masks/particle_system_masks.py | 8 -- pyproject.toml | 6 +- readme.md | 157 +++++++++++++++++++---- requirements.txt | 1 + 9 files changed, 390 insertions(+), 122 deletions(-) diff --git a/__init__.py b/__init__.py index 7126e84..e615e38 100644 --- a/__init__.py +++ b/__init__.py @@ -8,6 +8,9 @@ from .nodes.masks.temporal_masks import ( from .nodes.audio.audio_nodes import ( AudioSeparator, AudioFeatureVisualizer, + FrequencyFilterCustom, + FrequencyFilterPreset, + AudioFilter, ) from .nodes.masks.audio_masks import ( AudioMaskMorph, @@ -66,7 +69,10 @@ NODE_CLASS_MAPPINGS = { "AudioMaskTransform":AudioMaskTransform, "AudioMaskMath": AudioMaskMath, "AudioSeparator": AudioSeparator, - "AudioFeatureVisualizer":AudioFeatureVisualizer, + #"AudioFeatureVisualizer":AudioFeatureVisualizer, + "FrequencyFilterCustom": FrequencyFilterCustom, + "FrequencyFilterPreset": FrequencyFilterPreset, + "AudioFilter":AudioFilter, "MovingShape": MovingShape, @@ -79,7 +85,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "MaskTransform":"Temporal Mask Transform | RyanOnTheInside", "MaskMath":"Temporal Mask Math | RyanOnTheInside", "MaskRings":"Temporal Mask Rings | RyanOnTheInside", - "MaskWarp":"Mask Warp | RyanOnTheInside", + "MaskWarp":"Temporal Mask Warp | RyanOnTheInside", "OpticalFlowMaskModulation": "Optical Flow Mask Modulation | RyanOnTheInside", @@ -102,7 +108,10 @@ NODE_DISPLAY_NAME_MAPPINGS = { "AudioMaskTransform":"Audio Mask Transform | RyanOnTheInside", "AudioMaskMath": "Audio Mask Math | RyanOnTheInside", "AudioSeparator": "Audio Separator | RyanOnTheInside", - "AudioFeatureVisualizer": "Audio Feature Visualizer | RyanOnTheInside" , + #"AudioFeatureVisualizer": "Audio Feature Visualizer | RyanOnTheInside" , + "Frequency Filter Custom": "Frequency Filter Custom | RyanOnTheInside", + "Frequency Filter Preset": "Frequency Filter Preset | RyanOnTheInside", + "AudioFilter": "Audio Filter | RyanOnTheInside", "MovingShape": "Moving Shape | RyanOnTheInside", "TextMaskNode":"Text Mask Node | RyanOnTheInside" diff --git a/nodes/audio/audio_nodes.py b/nodes/audio/audio_nodes.py index ee24475..cbea72a 100644 --- a/nodes/audio/audio_nodes.py +++ b/nodes/audio/audio_nodes.py @@ -6,6 +6,9 @@ import matplotlib.pyplot as plt from scipy.ndimage import zoom import cv2 from .audio_utils import AudioVisualizer +from scipy import signal +import logging + class AudioSeparator: @classmethod @@ -22,6 +25,7 @@ class AudioSeparator: RETURN_NAMES = ("audio", "video_frames","drums_audio", "vocals_audio", "bass_audio", "other_audio") FUNCTION = "process_audio" + def __init__(self): self.separator = openunmix.umxl(targets=['drums', 'vocals', 'bass', 'other'], device='cpu') @@ -29,32 +33,24 @@ class AudioSeparator: waveform = audio['waveform'] sample_rate = audio['sample_rate'] - # Extract dimensions from video_frames num_frames, height, width, _ = video_frames.shape - print(f"Video frames shape: {video_frames.shape}") - # Ensure the waveform is in the correct shape (nb_channels, nb_samples) if waveform.dim() == 3: waveform = waveform.squeeze(0) # Remove batch dimension if present if waveform.dim() == 1: waveform = waveform.unsqueeze(0) # Add channel dimension if mono if waveform.shape[0] != 2: waveform = waveform.repeat(2, 1) # Duplicate mono to stereo if necessary - print(f"Waveform shape after preprocessing: {waveform.shape}") - - # Add a batch dimension + waveform = waveform.unsqueeze(0) - print(f"Waveform shape before separation: {waveform.shape}") # Separate the audio using Open-Unmix estimates = self.separator(waveform) - print(f"Estimates shape: {estimates.shape}") # Create isolated audio objects for each target isolated_audio = {} for i, target in enumerate(['drums', 'vocals', 'bass', 'other']): target_waveform = estimates[:, i, :, :] # Shape: (1, 2, num_samples) - print(f"{target} audio shape: {target_waveform.shape}") isolated_audio[target] = { 'waveform': target_waveform, @@ -70,7 +66,167 @@ class AudioSeparator: isolated_audio['bass'], isolated_audio['other'], ) - + +class AudioFilter: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "audio": ("AUDIO",), + "filters": ("FREQUENCY_FILTER",), + }, + } + + RETURN_TYPES = ("AUDIO",) + FUNCTION = "apply_filters" + + def apply_filters(self, audio, filters): + audio_np = audio['waveform'].cpu().numpy().squeeze(0) + sample_rate = audio['sample_rate'] + + filtered_channels = [] + for channel in audio_np: + filtered_channel = channel + for filter_params in filters: + filtered_channel = self.apply_single_filter(filtered_channel, filter_params, sample_rate) + filtered_channels.append(filtered_channel) + + filtered_audio = np.stack(filtered_channels) + + max_val = np.max(np.abs(filtered_audio)) + if max_val > 1.0: + filtered_audio = filtered_audio / max_val + + filtered_tensor = torch.from_numpy(filtered_audio).unsqueeze(0).float() + + return ({"waveform": filtered_tensor, "sample_rate": sample_rate},) + + def apply_single_filter(self, audio, filter_params, sample_rate): + filter_type = filter_params['type'] + order = filter_params['order'] + cutoff = filter_params['cutoff'] + + if filter_type == 'lowpass': + b, a = signal.butter(order, cutoff, btype='low', fs=sample_rate) + elif filter_type == 'highpass': + b, a = signal.butter(order, cutoff, btype='high', fs=sample_rate) + elif filter_type == 'bandpass': + low = cutoff - 50 + high = cutoff + 50 + b, a = signal.butter(order, [low, high], btype='band', fs=sample_rate) + else: + raise ValueError(f"Unsupported filter type: {filter_type}") + + return signal.lfilter(b, a, audio) + +class FrequencyFilterPreset: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "preset": ([ + "isolate_kick_drum", + "isolate_snare_drum", + "isolate_hi_hat", + "isolate_bass", + "isolate_vocals", + "remove_rumble", + "brighten_mix", + "remove_hiss" + ],), + }, + "optional": { + "previous_filter": ("FREQUENCY_FILTER",), + }, + } + + RETURN_TYPES = ("FREQUENCY_FILTER",) + FUNCTION = "create_preset_filter_chain" + + def create_preset_filter_chain(self, preset, previous_filter=None): + new_filters = self.get_preset_filters(preset) + + if previous_filter: + if isinstance(previous_filter, list): + return (previous_filter + new_filters,) + else: + return ([previous_filter] + new_filters,) + else: + return (new_filters,) + + def get_preset_filters(self, preset): + if preset == "isolate_kick_drum": + return [ + {"type": "highpass", "order": 2, "cutoff": 30}, + {"type": "lowpass", "order": 4, "cutoff": 200} + ] + elif preset == "isolate_snare_drum": + return [ + {"type": "bandpass", "order": 4, "cutoff": 200}, + {"type": "bandpass", "order": 4, "cutoff": 5000} + ] + elif preset == "isolate_hi_hat": + return [ + {"type": "highpass", "order": 4, "cutoff": 8000} + ] + elif preset == "isolate_bass": + return [ + {"type": "lowpass", "order": 4, "cutoff": 250} + ] + elif preset == "isolate_vocals": + return [ + {"type": "bandpass", "order": 2, "cutoff": 300}, + {"type": "bandpass", "order": 2, "cutoff": 4000} + ] + elif preset == "remove_rumble": + return [ + {"type": "highpass", "order": 4, "cutoff": 40} + ] + elif preset == "brighten_mix": + return [ + {"type": "highpass", "order": 2, "cutoff": 6000} + ] + elif preset == "remove_hiss": + return [ + {"type": "lowpass", "order": 6, "cutoff": 7500} + ] + else: + raise ValueError(f"Unknown preset: {preset}") + +class FrequencyFilterCustom: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "filter_type": (["lowpass", "highpass", "bandpass"],), + "order": ("INT", {"default": 4, "min": 1, "max": 10, "step": 1}), + "cutoff": ("FLOAT", {"default": 1000, "min": 20, "max": 20000, "step": 1}), + }, + "optional": { + "previous_filter": ("FREQUENCY_FILTER",), + }, + } + + RETURN_TYPES = ("FREQUENCY_FILTER",) + FUNCTION = "create_filter" + + def create_filter(self, filter_type, order, cutoff, previous_filter=None): + filter_params = { + "type": filter_type, + "order": order, + "cutoff": cutoff, + } + + if previous_filter: + if isinstance(previous_filter, list): + previous_filter.append(filter_params) + return (previous_filter,) + else: + return ([previous_filter, filter_params],) + else: + return ([filter_params],) + + class AudioFeatureVisualizer: @classmethod def INPUT_TYPES(cls): @@ -95,9 +251,7 @@ class AudioFeatureVisualizer: CATEGORY = "audio" def visualize_audio_feature(self, audio, video_frames, visualization_type, frame_rate): - # Extract dimensions from video_frames num_frames, height, width, _ = video_frames.shape - print(f"Video frames shape: {video_frames.shape}") visualizer = AudioVisualizer(audio, num_frames, height, width, frame_rate) @@ -116,6 +270,5 @@ class AudioFeatureVisualizer: else: raise ValueError(f"Unsupported visualization type: {visualization_type}") - print(f"Output mask shape: {mask.shape}") return (mask,) \ No newline at end of file diff --git a/nodes/audio/audio_utils.py b/nodes/audio/audio_utils.py index b2e0ea7..ef2bf12 100644 --- a/nodes/audio/audio_utils.py +++ b/nodes/audio/audio_utils.py @@ -32,7 +32,7 @@ class BaseAudioProcessor: end_sample = int(end_time * self.sample_rate) return self.audio[start_sample:end_sample] -class AudioVisualizer(BaseAudioProcessor): +class AudioVisualizer(BaseAudioProcessor): def create_spectrogram(self): print(f"create_spectrogram input shapes: audio={self.audio.shape}, num_frames={self.num_frames}, height={self.height}, width={self.width}") diff --git a/nodes/masks/audio_masks.py b/nodes/masks/audio_masks.py index 8be3392..e43e5d9 100644 --- a/nodes/masks/audio_masks.py +++ b/nodes/masks/audio_masks.py @@ -1,74 +1,8 @@ -from .mask_base import MaskBase -from .mask_utils import morph_mask, normalize_array, warp_mask, transform_mask, combine_masks -from ..audio.audio_utils import AudioFeatureExtractor +from .mask_base import AudioMaskBase +from .mask_utils import morph_mask, warp_mask, transform_mask, combine_masks import numpy as np import torch -class AudioMaskBase(MaskBase): - @classmethod - def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), - "required": { - **super().INPUT_TYPES()["required"], - "audio": ("AUDIO",), - "video_frames": ("IMAGE",), - "audio_feature": (["amplitude_envelope", "rms_energy", "spectral_centroid", "onset_detection", "chroma_features"],), - "feature_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), - "frame_rate": ("FLOAT", {"default": 30, "min": 0.1, "max": 120, "step": 0.1}), - } - } - - RETURN_TYPES = ("MASK","AUDIO") - FUNCTION = "main_function" - - def __init__(self): - super().__init__() - - def initialize(self, audio, audio_feature, num_frames, frame_rate): - self.audio_feature_extractor = AudioFeatureExtractor(audio, num_frames, frame_rate, feature_type=audio_feature) - - def process_mask(self, mask: np.ndarray, strength: float, audio: dict, video_frames: torch.Tensor, audio_feature: str, feature_threshold: float, frame_rate: float, **kwargs) -> np.ndarray: - # Extract audio feature - feature = self.audio_feature_extractor.extract() - - # Normalize feature to [0, 1] range - normalized_feature = normalize_array(feature) - - # Ensure the feature length matches the number of video frames - num_frames = mask.shape[0] - if len(normalized_feature) != num_frames: - normalized_feature = np.interp(np.linspace(0, 1, num_frames), np.linspace(0, 1, len(normalized_feature)), normalized_feature) - - # Apply the audio-driven effect to each frame - processed_masks = [] - for i in range(num_frames): - if normalized_feature[i] < feature_threshold: - processed_masks.append(mask[i]) - else: - frame_strength = strength * (1 + normalized_feature[i]) - processed_mask = self.process_single_frame(mask[i], frame_strength, frame_index=i, **kwargs) - processed_masks.append(processed_mask) - - return np.stack(processed_masks) - - def process_single_frame(self, mask: np.ndarray, strength: float, frame_index: int = None, **kwargs) -> np.ndarray: - """ - Process a single frame of the mask. - This method should be implemented by subclasses. - """ - raise NotImplementedError("Subclasses must implement process_single_frame method") - - def apply_audio_driven_mask(self, masks, audio, video_frames, strength, audio_feature, feature_threshold, frame_rate, **kwargs): - processed_masks = self.process_mask(masks.numpy(), strength, audio, video_frames, audio_feature, feature_threshold, frame_rate, **kwargs) - return self.apply_mask_operation(torch.from_numpy(processed_masks), masks, strength, **kwargs) - - def main_function(self, masks, audio, video_frames, strength, audio_feature, feature_threshold, frame_rate, **kwargs): - num_frames = masks.shape[0] - self.initialize(audio, audio_feature, num_frames, frame_rate) - - return self.apply_audio_driven_mask(masks, audio, video_frames, strength, audio_feature, feature_threshold, frame_rate, **kwargs), audio - class AudioMaskMorph(AudioMaskBase): @classmethod def INPUT_TYPES(cls): diff --git a/nodes/masks/mask_base.py b/nodes/masks/mask_base.py index 578c1c3..ba728b4 100644 --- a/nodes/masks/mask_base.py +++ b/nodes/masks/mask_base.py @@ -18,7 +18,7 @@ import random from typing import List, Tuple import pymunk import cv2 - +from ..audio.audio_utils import AudioFeatureExtractor class MaskBase(ABC): @classmethod @@ -33,7 +33,7 @@ class MaskBase(ABC): } } - CATEGORY = "/RyanOnTheInside/masks/" + CATEGORY="RyanOnTheInside/Masks" def __init__(self): self.pre_processors = [] @@ -150,7 +150,9 @@ class TemporalMaskBase(MaskBase, ABC): "palindrome": ("BOOLEAN", {"default": False}), } } - + + CATEGORY="RyanOnTheInside/TemporalMasks" + def __init__(self): super().__init__() @@ -235,7 +237,7 @@ class ParticleSystemMaskBase(MaskBase, ABC): RETURN_TYPES = ("MASK", "IMAGE") FUNCTION = "main_function" - CATEGORY = "/RyanOnTheInside/masks/" + CATEGORY="RyanOnTheInside/ParticleSystemMasks" def __init__(self): super().__init__() @@ -812,7 +814,7 @@ class OpticalFlowMaskBase(MaskBase, ABC): } } - CATEGORY = "/RyanOnTheInside/masks/" + CATEGORY="RyanOnTheInside/OpticalFlowMasks" def __init__(self): super().__init__() @@ -860,5 +862,67 @@ class OpticalFlowMaskBase(MaskBase, ABC): processed_masks = np.stack(result) return self.apply_mask_operation(processed_masks, masks, strength, **kwargs) +class AudioMaskBase(MaskBase): + @classmethod + def INPUT_TYPES(cls): + return { + **super().INPUT_TYPES(), + "required": { + "frame_rate": ("FLOAT", {"default": 30, "min": 0.1, "max": 120, "step": 0.1}), + **super().INPUT_TYPES()["required"], + "audio": ("AUDIO",), + "video_frames": ("IMAGE",), + "audio_feature": (["amplitude_envelope", "rms_energy", "spectral_centroid", "onset_detection", "chroma_features"],), + "feature_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), + + } + } + + RETURN_TYPES = ("MASK","AUDIO") + FUNCTION = "main_function" + CATEGORY="RyanOnTheInside/AudioMasks" + + def __init__(self): + super().__init__() + + def initialize(self, audio, audio_feature, num_frames, frame_rate): + self.audio_feature_extractor = AudioFeatureExtractor(audio, num_frames, frame_rate, feature_type=audio_feature) + + def process_mask(self, mask: np.ndarray, strength: float, audio: dict, video_frames: torch.Tensor, audio_feature: str, feature_threshold: float, frame_rate: float, **kwargs) -> np.ndarray: + feature = self.audio_feature_extractor.extract() + + normalized_feature = normalize_array(feature) + + num_frames = mask.shape[0] + if len(normalized_feature) != num_frames: + normalized_feature = np.interp(np.linspace(0, 1, num_frames), np.linspace(0, 1, len(normalized_feature)), normalized_feature) + + processed_masks = [] + for i in range(num_frames): + if normalized_feature[i] < feature_threshold: + processed_masks.append(mask[i]) + else: + frame_strength = strength * (1 + normalized_feature[i]) + processed_mask = self.process_single_frame(mask[i], frame_strength, frame_index=i, **kwargs) + processed_masks.append(processed_mask) + + return np.stack(processed_masks) + + def process_single_frame(self, mask: np.ndarray, strength: float, frame_index: int = None, **kwargs) -> np.ndarray: + """ + Process a single frame of the mask. + This method should be implemented by subclasses. + """ + raise NotImplementedError("Subclasses must implement process_single_frame method") + + def apply_audio_driven_mask(self, masks, audio, video_frames, strength, audio_feature, feature_threshold, frame_rate, **kwargs): + processed_masks = self.process_mask(masks.numpy(), strength, audio, video_frames, audio_feature, feature_threshold, frame_rate, **kwargs) + return self.apply_mask_operation(torch.from_numpy(processed_masks), masks, strength, **kwargs) + + def main_function(self, masks, audio, video_frames, strength, audio_feature, feature_threshold, frame_rate, **kwargs): + num_frames = masks.shape[0] + self.initialize(audio, audio_feature, num_frames, frame_rate) + + return self.apply_audio_driven_mask(masks, audio, video_frames, strength, audio_feature, feature_threshold, frame_rate, **kwargs), audio diff --git a/nodes/masks/particle_system_masks.py b/nodes/masks/particle_system_masks.py index 88a3b69..37dee2b 100644 --- a/nodes/masks/particle_system_masks.py +++ b/nodes/masks/particle_system_masks.py @@ -22,7 +22,6 @@ class ParticleEmissionMask(ParticleSystemMaskBase): RETURN_TYPES = ("MASK","IMAGE") FUNCTION = "main_function" - CATEGORY = "Masks/Particle Systems" def process_single_mask(self, mask: np.ndarray, frame_index: int, emission_strength: float, **kwargs) -> Tuple[np.ndarray, np.ndarray]: @@ -78,7 +77,6 @@ class Vortex: RETURN_TYPES = ("VORTEX",) FUNCTION = "create_vortex" - CATEGORY = "RyanOnTheInside/ParticleSystems" def create_vortex(self, x, y, strength, radius, inward_factor, movement_speed, color, draw, previous_vortex=None): vortex = { @@ -119,7 +117,6 @@ class GravityWell: RETURN_TYPES = ("GRAVITY_WELL",) FUNCTION = "create_gravity_well" - CATEGORY = "RyanOnTheInside/ParticleSystems" def create_gravity_well(self, x, y, strength, radius, type, color, draw, previous_well=None): well = { @@ -164,7 +161,6 @@ class ParticleEmitter: RETURN_TYPES = ("PARTICLE_EMITTER",) FUNCTION = "create_emitter" - CATEGORY = "RyanOnTheInside/ParticleSystems" def create_emitter(self, emitter_x, emitter_y, particle_direction, particle_spread, particle_size, particle_speed, emission_rate, color, initial_plume, @@ -211,7 +207,6 @@ class SpringJointSetting: RETURN_TYPES = ("SPRING_JOINT_SETTING",) FUNCTION = "create_setting" - CATEGORY = "RyanOnTheInside/ParticleSystems" def create_setting(self, stiffness, damping, rest_length, max_distance): return ({ @@ -237,7 +232,6 @@ class EmitterMovement: RETURN_TYPES = ("EMITTER_MOVEMENT",) FUNCTION = "create_movement" - CATEGORY = "RyanOnTheInside/ParticleSystems" def create_movement(self, emitter_x_frequency, emitter_y_frequency, direction_frequency, emitter_x_amplitude, emitter_y_amplitude, direction_amplitude): @@ -274,7 +268,6 @@ class StaticBody: RETURN_TYPES = ("STATIC_BODY",) FUNCTION = "create_static_body" - CATEGORY = "RyanOnTheInside/ParticleSystems" def create_static_body(self, shape_type, x1, y1, x2, y2, elasticity, friction, draw, color, previous_body=None): body = { @@ -313,7 +306,6 @@ class ParticleModulationBase: } } - CATEGORY = "RyanOnTheInside/ParticleSystems" def create_modulation(self, start_frame, end_frame, effect_duration, temporal_easing, palindrome, previous_modulation=None): modulation = { diff --git a/pyproject.toml b/pyproject.toml index 7e64a35..eed5c21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,9 @@ [project] name = "comfyui_ryanonyheinside" -description = "Custom nodes introducing particle simulations, optical flow, and temporal masks" -version = "1.1.0" +description = "Custom nodes introducing particle simulations, optical flow, audio manipulation & reactivity, and temporal masks" +version = "1.2.0" license = {file = "LICENSE"} -dependencies = ["opencv-python==4.10.0","scipy","pillow","librosa","pymunk==6.8.1","matplotlib"] +dependencies = ["opencv-python==4.10.0","scipy","pillow","librosa","pymunk==6.8.1","matplotlib","openunmix"] [project.urls] Repository = "https://github.com/ryanontheinside/ComfyUI_RyanOnTheInside" diff --git a/readme.md b/readme.md index 9aab703..d47813d 100644 --- a/readme.md +++ b/readme.md @@ -1,10 +1,11 @@ # ComfyUI RyanOnTheInside Node Pack ## Overview -RyanOnTheInside node pack introduces the following (so far): -- Particle Systems -- Optical Flow -- Temporal Masks +RyanOnTheInside node pack introduces the following: +- Particle Systems - dynamic particle systems in Comfy! +- Audio Masks - Create reactive masks from audio! Even separate vocals, drums, bass, etc! +- Optical Flow - Masks generated by the motion in a video! +- Temporal Masks - Apply effects to masks over tiiiiime! @@ -23,11 +24,24 @@ RyanOnTheInside node pack introduces the following (so far):
-*Some examples using particle emitters and vortices* +*Examples showcasing various effects using particle emitters, vortices, and other node features* + +## Table of Contents +- [Installation](#installation) +- [Requirements](#requirements) +- [Features](#features) + - [Particle System Masks](#particle-system-masks) + - [Audio Masks](#audio-masks) + - [Optical Flow Masks](#optical-flow-masks) + - [Temporal Masks](#temporal-masks) + - [Miscellaneous Nodes](#miscellaneous-nodes) +- [Contributing](#contributing) +- [License](#license) +- [Support](#support) ## Installation -Install via the ComfyUI Manager, or... +Install via the ComfyUI Manager by searching for RyanOnTheInside, or... 1. Navigate to your ComfyUI's `custom_nodes` directory 2. Clone the repository: @@ -44,7 +58,6 @@ Install via the ComfyUI Manager, or... ``` 5. Restart ComfyUI if it's currently running and refresh your browser - ## Requirements See `requirements.txt` for a list of dependencies. @@ -60,11 +73,13 @@ All mask nodes in this system share the following parameters: - `Subtract Original`: Amount of the original mask to subtract from the result (0.0 to 1.0). Useful for creating edge effects or isolating changes. - `Grow with Blur`: Expands the mask edges using a blur effect (0.0 to 10.0). Higher values create softer, more expanded edges. -These parameters provide a consistent base for all mask operations, allowing for fine-tuned control over the mask processing across various node types. There are currently three categories of masks... +These parameters provide a consistent base for all mask operations, allowing for consistent control over the mask processing across various node types. -## Particle System Masks +
+

Particle System Masks

 + +Create dynamic, fluid-like effects through particle simulation. This section includes nodes for particle emission, force fields, and complex particle behaviors. -Create dynamic, fluid-like effects through particle simulation: - Supports N particle emitters, each with independent settings including movement, particle (spread, speed, size, COLOR, etc) - Supports N force fields (Gravity Well (repel and attract), and Vortex), each with independent settings - Each simulation space has basic gravity and wind settings @@ -72,9 +87,9 @@ Create dynamic, fluid-like effects through particle simulation: - Static bodies for particle interactions - Spring joint settings for more complex particle behaviors - Particle System Demo +Particle System Demo showing various effects like object-aware particles, plume effects, and vortex interactions - *A few examples showcasing object-aware particles, initial plume effects, vortex interactions, gravity wells, and directional emitters.* +*Examples showcasing object-aware particles, initial plume effects, vortex interactions, gravity wells, and directional emitters.* #### General Parameters for particle system These settings apply to all particle system mask nodes, and generally affect the simulation space level. @@ -197,18 +212,110 @@ These are optional inputs for a simulation space. They can be chained together t - `color`: Color of the vortex visualization (RGB tuple) - `draw`: Visibility of the vortex (0.0 to 1.0) -## Optical Flow Masks +
-Motion-based effects: +
+

Audio Masks

 + +Create dynamic, audio-reactive mask effects. This section covers nodes for audio-driven mask generation and manipulation. + +- Create separate masks for vocals, bass, drums, other! +- Audio-driven morphological operations (erosion, dilation, opening, closing) +- Audio-reactive warping effects (perlin noise, radial, swirl) +- Audio-based mask transformations (translation, rotation, scaling) +- Mathematical operations between masks driven by audio features +- Supports various audio features for mask modulation + +Audio Mask Examples demonstrating morphing, warping, transformation, and mathematical operations + +*Examples of audio-driven mask effects: morphing, warping, transformation, and mathematical operations* + +### Common Parameters for all AudioMask nodes +- `masks`: Input mask sequence to be processed +- `audio`: Input audio for driving the mask generation +- `video_frames`: Corresponding video frames +- `strength`: Overall strength of the effect (0.0 to 1.0) +- `audio_feature`: Audio feature to use for mask modulation (amplitude_envelope, rms_energy, spectral_centroid, onset_detection, chroma_features) +- `feature_threshold`: Threshold for audio feature detection (0.0 to 1.0) +- `frame_rate`: Frame rate of the video (0.1 to 120.0 fps) + +### Audio Mask Nodes + +#### AudioMaskMorph +Applies morphological operations to masks based on audio features: +- `morph_type`: Type of morphological operation ("erode", "dilate", "open", "close") +- `max_kernel_size`: Maximum size of the morphological kernel (3 to 21, odd numbers only) +- `max_iterations`: Maximum number of times to apply the operation (1 to 50) + +#### AudioMaskWarp +Applies warping effects to masks driven by audio: +- `warp_type`: Type of warping effect ("perlin", "radial", "swirl") +- `frequency`: Frequency of the warping effect (0.01 to 1.0) +- `max_amplitude`: Maximum amplitude of the warping effect (0.1 to 500.0) +- `octaves`: Number of octaves for noise-based warps (1 to 8) + +#### AudioMaskTransform +Applies geometric transformations to masks based on audio features: +- `transform_type`: Type of transformation ("translate", "rotate", "scale") +- `max_x_value`: Maximum X-axis transformation value (-1000.0 to 1000.0) +- `max_y_value`: Maximum Y-axis transformation value (-1000.0 to 1000.0) + +#### AudioMaskMath +Performs mathematical operations between two masks, modulated by audio: +- `mask_b`: Second mask to combine with the input mask +- `combination_method`: Mathematical operation to apply ("add", "subtract", "multiply", "minimum", "maximum") + +### Audio Processing and Visualization Nodes + +#### AudioSeparator +Separates audio into different components: +- `audio`: Input audio to be processed +- `video_frames`: Corresponding video frames +- `frame_rate`: Frame rate of the video + +Outputs: original audio, video frames, drums audio, vocals audio, bass audio, other audio + +#### AudioFilter +Applies frequency filters to audio: +- `audio`: Input audio to be filtered +- `filters`: Frequency filters to be applied (FREQUENCY_FILTER type) + +#### FrequencyFilterPreset +Creates preset filter chains for common audio processing tasks: +- `preset`: Preset to use (e.g., "isolate_kick_drum", "isolate_vocals", "remove_rumble") +- `previous_filter`: (Optional) Previous filter chain to append to + +#### FrequencyFilterCustom +Creates custom frequency filters: +- `filter_type`: Type of filter ("lowpass", "highpass", "bandpass") +- `order`: Filter order (1 to 10) +- `cutoff`: Cutoff frequency (20 to 20000 Hz) +- `previous_filter`: (Optional) Previous filter chain to append to + +#### AudioFeatureVisualizer +Visualizes various audio features: +- `audio`: Input audio to visualize +- `video_frames`: Corresponding video frames +- `visualization_type`: Type of visualization ("waveform", "spectrogram", "mfcc", "chroma", "tonnetz", "spectral_centroid") +- `frame_rate`: Frame rate of the video + +These audio processing and visualization nodes enable complex audio analysis, filtering, and feature extraction, allowing for creative audio-driven effects and visualizations within the ComfyUI environment. + +
+ +
+

Optical Flow Masks

 + +Create motion-based effects using optical flow analysis. This section includes nodes for generating masks based on movement in video sequences. - Multiple optical flow algorithms (Farneback, Lucas-Kanade, Pyramidal Lucas-Kanade) - Particle simulations reactive to optical flow - work in progress - Directional flow isolation and emphasis - in progress - Advanced blending and modulation options - Particle System Demo +Optical Flow Demo showing motion-generated trails and effects - *Motion generated trails* +*Motion generated trails using optical flow analysis* ### Optical Flow Mask Nodes @@ -258,9 +365,12 @@ This node will generate a particle system animation driven by optical flow, crea - `Initial Velocity`: Starting speed of newly emitted particles (0.1 to 5.0) --> -## Temporal Masks +
-Implement time-based transformations and animations: +
+

Temporal Masks

 + +Implement time-based transformations and animations on masks. This section covers nodes for creating dynamic mask effects over time. - Morphing and transformation effects (erosion, dilation, translation, rotation, scaling) - Various mask combination methods @@ -268,9 +378,9 @@ Implement time-based transformations and animations: - Warp effects using Perlin noise, radial distortion, and swirling patterns - Customizable easing functions and palindrome support - Temporal Masks Examples +Temporal Masks Examples showing various time-based effects - *Examples of various combinations of temporal mask effects including morphing, transformations, math , and warping.* +*Examples of various combinations of temporal mask effects including morphing, transformations, math, and warping.* - `Start Frame`: The frame number where the effect begins (0 to 1000). @@ -321,8 +431,12 @@ Applies various warping effects to the mask. All these nodes inherit the common parameters from MaskBase and TemporalMaskBase, allowing for precise control over the timing and strength of the effects across multiple frames. +
-### Miscellaneous Nodes +
+

Miscellaneous Nodes

 + +Additional utility nodes for creating specific mask effects and animations. #### MovingShape @@ -367,6 +481,7 @@ Parameters: This node creates both a mask and an image with the specified text, allowing for various text-based effects and animations. The text automatically wraps to fit within the specified maximum width. +
## Contributing diff --git a/requirements.txt b/requirements.txt index cf04099..2a2fa89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ pillow librosa pymunk==6.8.1 matplotlib +openunmix \ No newline at end of file