diff --git a/__init__.py b/__init__.py index 2e41f9b..dff4080 100644 --- a/__init__.py +++ b/__init__.py @@ -1,4 +1,4 @@ -from .node_configs import CombinedMeta +from .node_configs.node_configs import CombinedMeta from collections import OrderedDict #allows for central management and inheritance of class variables for help documentation @@ -51,16 +51,17 @@ from .nodes.masks.temporal_masks import ( from .nodes.audio.audio_nodes import ( AudioSeparator, DownloadOpenUnmixModel, + # DownloadCREPEModel, AudioFeatureVisualizer, FrequencyFilterCustom, FrequencyFilterPreset, AudioFilter, + EmptyMaskFromAudio, + EmptyImageFromAudio, ) from .nodes.flex.feature_extractors import( - - AudioFeatureExtractor, TimeFeatureNode, DepthFeatureNode, ColorFeatureNode, @@ -69,18 +70,30 @@ from .nodes.flex.feature_extractors import( AreaFeatureNode, ) -from .nodes.flex.midi_feature_extractor import( +from .nodes.flex.feature_extractors_audio import( + AudioFeatureExtractor, + PitchRangeNode, + PitchRangePresetNode, + PitchRangeByNoteNode, + PitchFeatureExtractor, +) + +from .nodes.flex.feature_extractors_midi import( MIDILoadAndExtract, ) -from .nodes.flex.proximity_feature_extractor import( +from .nodes.flex.feature_extractors_proximity import( LocationFromMask, ProximityFeatureNode, - ProximityVisualizer, LocationFromPoint, LocationTransform, ) +from .nodes.flex.visualizers import( + ProximityVisualizer, + EffectVisualizer, + PitchVisualizer, +) @@ -103,11 +116,6 @@ from .nodes.masks.particle_system_masks import ( ParticleSpeedModulation, ) -# from .nodes.masks.opacity_masks import( -# DepthBasedMaskOpacity, -# FlexDepthBasedMaskOpacity, -# ) - from .nodes.masks.mask_utility_nodes import ( _mfc, TextMaskNode, @@ -146,6 +154,11 @@ from .nodes.masks.flex_masks import ( ) +from .nodes.masks.flex_masks_normal import ( + FlexMaskNormalLighting, + +) + from .nodes.images.flex_images import ( FlexImageEdgeDetect, FlexImagePosterize, @@ -175,6 +188,7 @@ from .nodes.flex.feature_modulation import ( PreviewFeature, FeatureMath, FeatureRebase, + FeatureTruncateOrExtend, ) @@ -232,18 +246,26 @@ NODE_CLASS_MAPPINGS = { "FlexMaskEmanatingRings": FlexMaskEmanatingRings, "FlexMaskRandomShapes": FlexMaskRandomShapes, "FlexMaskDepthChamber": FlexMaskDepthChamber, + #"FlexMaskNormalLighting": FlexMaskNormalLighting, # "FlexMaskDepthChamberRelative": FlexMaskDepthChamberRelative, #audio "AudioSeparator": AudioSeparator, "DownloadOpenUnmixModel": DownloadOpenUnmixModel, + # "DownloadCREPEModel": DownloadCREPEModel, "AudioFeatureVisualizer": AudioFeatureVisualizer, "FrequencyFilterCustom": FrequencyFilterCustom, "FrequencyFilterPreset": FrequencyFilterPreset, "AudioFilter": AudioFilter, + "EmptyMaskFromAudio": EmptyMaskFromAudio, + "EmptyImageFromAudio": EmptyImageFromAudio, #features "AudioFeatureExtractor": AudioFeatureExtractor, + "PitchFeatureExtractor": PitchFeatureExtractor, + "PitchRange": PitchRangeNode, + "PitchRangePreset": PitchRangePresetNode, + "PitchRangeByNoteNode": PitchRangeByNoteNode, "MIDILoadAndExtract": MIDILoadAndExtract, "TimeFeatureNode": TimeFeatureNode, "DepthFeatureNode": DepthFeatureNode, @@ -252,7 +274,6 @@ NODE_CLASS_MAPPINGS = { "MotionFeatureNode": MotionFeatureNode, "LocationFromMask": LocationFromMask, "ProximityFeatureNode": ProximityFeatureNode, - "ProximityVisualizer": ProximityVisualizer, "LocationFromPoint": LocationFromPoint, "LocationTransform": LocationTransform, "AreaFeatureNode": AreaFeatureNode, @@ -270,6 +291,8 @@ NODE_CLASS_MAPPINGS = { "FeatureMath": FeatureMath, "PreviewFeature": PreviewFeature, "FeatureRebase": FeatureRebase, + "FeatureTruncateOrExtend": FeatureTruncateOrExtend, + #images 'FlexImageEdgeDetect': FlexImageEdgeDetect, "FlexImagePosterize": FlexImagePosterize, @@ -285,6 +308,10 @@ NODE_CLASS_MAPPINGS = { # "FlexDepthBasedMaskOpacity": FlexDepthBasedMaskOpacity, # "DepthBasedMaskOpacity": DepthBasedMaskOpacity, + #visulizers + "ProximityVisualizer": ProximityVisualizer, + "EffectVisualizer": EffectVisualizer, + "PitchVisualizer": PitchVisualizer, #garb "DyeImage": DyeImage, @@ -328,18 +355,17 @@ NODE_DISPLAY_NAME_MAPPINGS = { "ParticleSizeModulation": "Particle Size Modulation", "ParticleSpeedModulation":"Particle Speed Modulation", - "AudioMaskMorph":"Audio Mask Morph", - "AudioMaskWarp": "Audio Mask Warp", - "AudioMaskTransform":"Audio Mask Transform", - "AudioMaskMath": "Audio Mask Math", + "AudioSeparator": "Audio Separator", "AudioFeatureVisualizer": "Audio Feature Visualizer" , "Frequency Filter Custom": "Frequency Filter Custom", "Frequency Filter Preset": "Frequency Filter Preset", "AudioFilter": "Audio Filter", - + + "MIDILoadAndExtract": "MIDI Load & Feature Extract", + "PitchRangeByNoteNode": "Pitch Range By Note", "AudioFeatureExtractor": "Audio Feature & Extractor", "TimeFeatureNode": "Time Feature", "DepthFeatureNode":"Depth Feature", diff --git a/node_configs/node_configs.py b/node_configs/node_configs.py new file mode 100644 index 0000000..250d611 --- /dev/null +++ b/node_configs/node_configs.py @@ -0,0 +1,1241 @@ +#NOTE: this abstraction allows for the documentation to be both centrally managed and inherited +from abc import ABCMeta +class NodeConfigMeta(type): + def __new__(cls, name, bases, attrs): + new_class = super().__new__(cls, name, bases, attrs) + if name in NODE_CONFIGS: + for key, value in NODE_CONFIGS[name].items(): + setattr(new_class, key, value) + return new_class + +class CombinedMeta(NodeConfigMeta, ABCMeta): + pass + +def add_node_config(node_name, config): + NODE_CONFIGS[node_name] = config + +NODE_CONFIGS = {} + +add_node_config("MaskBase", { + "BASE_DESCRIPTION": """ +##Parameters +- `Masks`: Input mask or sequence of masks to be processed. (you can pass in a blank mask if you want) +- `Strength`: Controls the intensity of the effect (0.0 to 1.0). Higher values make the mask operation more pronounced. +- `Invert`: When enabled, reverses the mask, turning black areas white and vice versa. +- `Subtract Original`: Removes a portion of the original mask from the result (0.0 to 1.0). Higher values create more pronounced edge effects. +- `Grow with Blur`: Expands the mask edges (0.0 to 10.0). Higher values create softer, more expanded edges. +""" +}) + +add_node_config("TemporalMaskBase", { + "ADDITIONAL_INFO": """ +- `Start Frame`: The frame number where the effect begins (0 to 1000). +- `End Frame`: The frame number where the effect ends (0 to 1000). If set to 0, continues until the last frame. +- `Effect Duration`: Number of frames over which the effect is applied (0 to 1000). If 0, uses (End Frame - Start Frame). +- `Temporal Easing`: Controls how the effect strength changes over time, affecting the smoothness of transitions. + - Options: "ease_in_out", "linear", "bounce", "elastic", "none" +- `Palindrome`: When enabled, the effect plays forward then reverses within the specified duration, creating a back-and-forth motion. +""" +}) + +add_node_config("ParticleSystemMaskBase", { + "ADDITIONAL_INFO": """ +- `particle_count`: Total number of particles in the system (1 to 10000). More particles create denser effects. +- `particle_lifetime`: How long each particle exists in seconds (0.1 to 10.0). Longer lifetimes create more persistent effects. +- `wind_strength`: Power of the wind effect (-100.0 to 100.0). Positive values blow right, negative left. +- `wind_direction`: Angle of the wind in degrees (0.0 to 360.0). 0 is right, 90 is up, etc. +- `gravity`: Strength of downward pull (-1000.0 to 1000.0). Negative values make particles float up. +- `start_frame`: Frame to begin the particle effect (0 to 1000). +- `end_frame`: Frame to stop the particle effect (0 to 1000). +- `respect_mask_boundary`: When enabled, particles stay within the mask's shape. + +Optional inputs: +- `emitters`: Particle emitter configurations (PARTICLE_EMITTER type). Define where particles originate. +- `vortices`: Optional vortex configurations (VORTEX type). Create swirling effects. +- `wells`: Optional gravity well configurations (GRAVITY_WELL type). Create areas that attract or repel particles. +- `well_strength_multiplier`: Amplifies the power of gravity wells (0.0 to 10.0). Higher values create stronger attraction/repulsion. +""" +}) + +add_node_config("ParticleEmissionMask", { + "TOP_DESCRIPTION": "This is the main node for particle simulations. It creates dynamic, fluid-like effects through particle simulation. Supports multiple particle emitters, force fields (Gravity Well, Vortex), and allows for complex particle behaviors including boundary-respecting particles and static body interactions.", + "ADDITIONAL_INFO": """ +- `emission_strength`: Strength of particle emission effect (0.0 to 1.0), basically opacity +- `draw_modifiers`: Visibility of vortices and gravity wells (0.0 to 1.0) +""" +}) + +add_node_config("OpticalFlowMaskBase", { + "ADDITIONAL_INFO": """ +- `Images`: Input image sequence for optical flow calculation +- `Masks`: Input mask sequence to be processed +- `Strength`: Overall intensity of the effect (0.0 to 1.0). Higher values create more pronounced motion-based effects. +- `Flow Method`: Technique used for optical flow calculation. Each method has different speed/accuracy tradeoffs. + - Options: "Farneback", "LucasKanade", "PyramidalLK" +- `Flow Threshold`: Minimum motion required to trigger the effect (0.0 to 1.0). Higher values ignore subtle movements. +- `Magnitude Threshold`: Relative threshold for flow magnitude (0.0 to 1.0). Higher values focus on areas of stronger motion. +""" +}) + +add_node_config("OpticalFlowMaskModulation", { + "TOP_DESCRIPTION": "This is currently the main Optical Flow node. Use it to make motion based effects.", + "ADDITIONAL_INFO": """ +- `Modulation Strength`: Intensity of the modulation effect (0.0 to 5.0). Higher values create more pronounced motion trails. +- `Blur Radius`: Smoothing applied to the flow magnitude (0 to 20 pixels). Larger values create smoother trails. +- `Trail Length`: Number of frames for the trail effect (1 to 20). Longer trails last longer. +- `Decay Factor`: Rate of trail decay over time (0.1 to 1.0). Lower values make trails fade faster. +- `Decay Style`: Method of trail decay. + - Options: "fade" (opacity reduction), "thickness" (width reduction) +- `Max Thickness`: Maximum trail thickness for thickness-based decay (1 to 50 pixels). Larger values create thicker trails. +""" +}) + +add_node_config("OpticalFlowDirectionMask", { + "TOP_DESCRIPTION": "***WORK IN PROGRESS***" +}) + +add_node_config("OpticalFlowParticleSystem", { + "TOP_DESCRIPTION": "***WORK IN PROGRESS***" +}) + +add_node_config("MaskTransform", { + "TOP_DESCRIPTION": "Applies geometric transformations to the mask over time.", + "ADDITIONAL_INFO": """ +- `Transform Type`: The type of transformation to apply. + - Options: "translate", "rotate", "scale" +- `X Value`: Horizontal component of the transformation (-1000 to 1000). Positive values move right, negative left. +- `Y Value`: Vertical component of the transformation (-1000 to 1000). Positive values move up, negative down. +""" +}) + +add_node_config("MaskMorph", { + "TOP_DESCRIPTION": "Applies morphological operations to the mask, changing its shape over time.", + "ADDITIONAL_INFO": """ +- `Morph Type`: The type of morphological operation to apply. + - Options: "erode", "dilate", "open", "close" +- `Max Kernel Size`: Maximum size of the morphological kernel (3 to 21, odd numbers only). Larger values create more pronounced effects. +- `Max Iterations`: Maximum number of times to apply the operation (1 to 50). More iterations create more extreme effects. +""" +}) + +add_node_config("MaskMath", { + "TOP_DESCRIPTION": "Combines two masks using various mathematical operations.", + "ADDITIONAL_INFO": """ +- `Mask B`: Second mask to combine with the input mask. +- `Combination Method`: Mathematical operation to apply. + - Options: "add", "subtract", "multiply", "minimum", "maximum" +""" +}) + +add_node_config("MaskRings", { + "TOP_DESCRIPTION": "Creates concentric ring patterns based on the distance from the mask edges.", + "ADDITIONAL_INFO": """ +- `Num Rings`: Number of rings to generate (1 to 50). More rings create more detailed patterns. +- `Max Ring Width`: Maximum width of each ring as a fraction of the total distance (0.01 to 0.5). Larger values create wider rings. +""" +}) + +add_node_config("MaskWarp", { + "TOP_DESCRIPTION": "Applies various warping effects to the mask, creating distortions and movement.", + "ADDITIONAL_INFO": """ +- `Warp Type`: The type of warping effect to apply. Each creates a different distortion pattern. + - Options: "perlin" (noise-based), "radial" (circular), "swirl" (spiral) +- `Frequency`: Controls the scale of the warping effect (0.01 to 1.0). Higher values create more rapid changes in the warp pattern. +- `Amplitude`: Controls the strength of the warping effect (0.1 to 500.0). Higher values create more extreme distortions. +- `Octaves`: For noise-based warps, adds detail at different scales (1 to 8). More octaves create more complex, detailed patterns. +""" +}) + +add_node_config("MIDILoadAndExtract", { + "TOP_DESCRIPTION": """ + Loads a MIDI file and extracts specified features for mask modulation. To use this, select the notes on the piano that you want to use to control modulations. + Many of the different types of information in the notes can be chosen as the driving feature.""", + "ADDITIONAL_INFO": """ +- `midi_file`: Path to the MIDI file to load and analyze +- `track_selection`: Which track(s) to analyze ("all" or specific track number) +- `attribute`: MIDI attribute to extract (e.g., "Note On/Off", "Pitchbend", "Pitch", "Aftertouch") +- `frame_rate`: Frame rate of the video to sync MIDI data with +- `video_frames`: Corresponding video frames (IMAGE type) +- `chord_only`: When true, only considers full chords (BOOLEAN) +- `notes`: IGNORE THIS. Certain limitations prevent me from hiding it completely. Love, Ryan +""" +}) + +add_node_config("AudioFilter", { + "TOP_DESCRIPTION": "Applies frequency filters to audio for targeted sound processing.", + "ADDITIONAL_INFO": """ +- `audio`: Input audio to be filtered (AUDIO type) +- `filters`: Frequency filters to be applied (FREQUENCY_FILTER type). These determine which frequencies are emphasized or reduced. +""" +}) + +add_node_config("FrequencyFilterPreset", { + "TOP_DESCRIPTION": "Creates preset filter chains for common audio processing tasks, simplifying complex audio manipulations.", + "ADDITIONAL_INFO": """ +- `preset`: Preset to use (e.g., "isolate_kick_drum" emphasizes low frequencies, "isolate_vocals" focuses on mid-range, "remove_rumble" cuts low frequencies) + +Optional inputs: +- `previous_filter`: Previous filter chain to append to, allowing for cumulative effects +""" +}) + +add_node_config("FrequencyFilterCustom", { + "TOP_DESCRIPTION": "Creates custom frequency filters.", + "ADDITIONAL_INFO": """ +- `filter_type`: Type of filter ("lowpass", "highpass", "bandpass") +- `order`: Filter order (1 to 10) +- `cutoff`: Cutoff frequency (20 to 20000 Hz) + +Optional inputs: +- `previous_filter`: Previous filter chain to append to +""" +}) + +add_node_config("AudioFeatureVisualizer", { + "TOP_DESCRIPTION": "***WORK IN PROGESS*** Visualizes various audio features, creating visual representations of sound characteristics.", + "ADDITIONAL_INFO": """ +- `audio`: Input audio to visualize (AUDIO type) +- `video_frames`: Corresponding video frames to overlay visualizations on (IMAGE type) +- `visualization_type`: Type of visualization to generate: + - "waveform": Shows amplitude over time + - "spectrogram": Displays frequency content over time + - "mfcc": Mel-frequency cepstral coefficients, useful for speech recognition + - "chroma": Represents pitch classes, useful for harmonic analysis + - "tonnetz": Tonal space representation + - "spectral_centroid": Shows the "center of mass" of the spectrum over time +- `frame_rate`: Frame rate of the video for synchronization +""" +}) + +add_node_config("AudioSeparator", { + "TOP_DESCRIPTION": "Separates an input audio track into its component parts using the Open-Unmix model.", + "ADDITIONAL_INFO": """ +- `audio`: Input audio to be separated (AUDIO type) +- `video_frames`: Corresponding video frames (IMAGE type) +- `frame_rate`: Frame rate of the video for synchronization + +Outputs: +- Original audio +- Isolated drums track +- Isolated vocals track +- Isolated bass track +- Isolated other instruments track +- FeaturePipe containing frame information + +This node uses the Open-Unmix model to separate the input audio into four stems: drums, vocals, bass, and other instruments. Each separated track is returned as an individual AUDIO type output, along with the original audio and a FeaturePipe for further processing. +""" +}) + +add_node_config("SpringJointSetting", { + "TOP_DESCRIPTION": "Defines the behavior of spring joints attached to particles.", + "ADDITIONAL_INFO": """ +- `stiffness`: Stiffness of the spring (0.0 to 1000.0). Higher values create stronger connections. +- `damping`: Damping factor of the spring (0.0 to 100.0). Higher values create more resistance to motion. +- `rest_length`: Rest length of the spring (0.0 to 100.0). Longer springs allow for more stretching. +- `max_distance`: Maximum distance the spring can stretch (0.0 to 500.0). Larger values allow for more elasticity. +""" +}) + +add_node_config("StaticBody", { + "TOP_DESCRIPTION": "Defines static bodies in the simulation that particles can interact with (think walls, barrier, ramps, etc.).", + "ADDITIONAL_INFO": """ +- `shape_type`: Type of shape ("segment" or "polygon") +- `x1`, `y1`, `x2`, `y2`: Coordinates defining the shape +- `elasticity`: Bounciness of the static body (0.0 to 1.0). Higher values create more bouncy collisions. +- `friction`: Friction of the static body (0.0 to 1.0). Higher values create more resistance to motion. +- `draw`: Whether to visualize the static body and how thick +- `color`: Color of the static body (RGB tuple) +""" +}) + +add_node_config("GravityWell", { + "TOP_DESCRIPTION": "An optional input for a simulation space. These can be chained together to add many to a simulation.", + "ADDITIONAL_INFO": """ +- `x`: X-coordinate of the gravity well (0.0 to 1.0) +- `y`: Y-coordinate of the gravity well (0.0 to 1.0) +- `strength`: Strength of the gravity well. Higher values create stronger attraction or repulsion. +- `radius`: Radius of effect for the gravity well. Larger values affect a wider area. +- `type`: Type of gravity well ('attract' or 'repel'). Attract pulls particles in, repel pushes them away. +- `color`: Color of the gravity well visualization (RGB tuple) +- `draw`: Thickness of the gravity well visualization (0.0 to 1.0) +""" +}) + +add_node_config("Vortex", { + "TOP_DESCRIPTION": "An optional input for a simulation space. These can be chained together to add many to a simulation", + "ADDITIONAL_INFO": """ +- `x`: X-coordinate of the vortex center (0.0 to 1.0) +- `y`: Y-coordinate of the vortex center (0.0 to 1.0) +- `strength`: Strength of the vortex effect (0.0 to 1000.0). Higher values create stronger swirling motion. +- `radius`: Radius of effect for the vortex (10.0 to 500.0). Larger values create wider swirling areas. +- `inward_factor`: Factor controlling how quickly particles are pulled towards the center (0.0 to 1.0). Higher values create tighter spirals. +- `movement_speed`: Speed of movement of the vortex object (0.0 to 10.0). Higher values make the vortex move faster in the simulation space. +- `color`: Color of the vortex visualization (RGB tuple) +- `draw`: Thickness of the vortex visualization (0.0 to 1.0) +""" +}) + +add_node_config("ParticleModulationBase", { + "TOP_DESCRIPTION": "Base class for particle modulation settings.", + "ADDITIONAL_INFO": """ +- `start_frame`: Frame to start the modulation effect (0 to 1000) +- `end_frame`: Frame to end the modulation effect (0 to 1000) +- `effect_duration`: Duration of the modulation effect in frames (0 to 1000) +- `temporal_easing`: Easing function for the modulation effect ("ease_in_out", "linear", "bounce", "elastic", "none") +- `palindrome`: Whether to reverse the modulation effect after completion (True/False) +- `random`: Selects a random value between 0 and the chosen target value and applies it per particle +- `feature`: Optionally, pass in a feature (like audio or motion) to drive the modulation of the particles +""" +}) + +add_node_config("ParticleSizeModulation", { + "TOP_DESCRIPTION": "Modulates particle size over time.", + "ADDITIONAL_INFO": """ +- `target_size`: Target size for particles at the end of the modulation (0.0 to 400.0) +""" +}) + +add_node_config("ParticleSpeedModulation", { + "TOP_DESCRIPTION": "Modulates particle speed over time.", + "ADDITIONAL_INFO": """ +- `target_speed`: Target speed for particles at the end of the modulation (0.0 to 1000.0) +""" +}) + +add_node_config("ParticleColorModulation", { + "TOP_DESCRIPTION": "Modulates particle color over time.", + "ADDITIONAL_INFO": """ +- `target_color`: Target color for particles at the end of the modulation (RGB tuple) +""" +}) + +add_node_config("FlexMaskBase", { + "BASE_DESCRIPTION": """ +- `feature`: The feature used to modulate the mask operation (FEATURE type) +- `feature_pipe`: The feature pipe containing frame information (FEATURE_PIPE type) +- `feature_threshold`: Threshold for feature activation (0.0 to 1.0) + +""" +}) + +add_node_config("FlexMaskDepthChamber", { + "TOP_DESCRIPTION": "Applies a depth-based mask modulation using a depth map and specified front and back depth values.", + "ADDITIONAL_INFO": """ +- `depth_map`: Input depth map (IMAGE type) +- `z_front`: Front depth value for the mask (0.0 to 1.0). Default is 1.0. +- `z_back`: Back depth value for the mask (0.0 to 1.0). Default is 0.0. +- `feature_param`: Parameter to modulate based on the feature. Options are "none", "z_front", "z_back", "both". +- `feature_mode`: Mode of feature modulation. + +This node creates a mask based on the depth values in the input depth map. The mask is modulated by the specified front and back depth values, and can be further adjusted using a feature input to dynamically change the depth range. +""" +}) + + +add_node_config("FlexMaskMorph", { + "TOP_DESCRIPTION": "Applies morphological operations to the mask, modulated by a selected feature.", + "ADDITIONAL_INFO": """ +- `morph_type`: The type of morphological operation to apply. + - Options: "erode", "dilate", "open", "close" +- `max_kernel_size`: Maximum size of the morphological kernel (3 to 21, odd numbers only). Larger values create more pronounced effects. +- `max_iterations`: Maximum number of times to apply the operation (1 to 50). More iterations create more extreme effects. + +The strength of the morphological operation is determined by the selected feature's value at each frame. +""" +}) + +add_node_config("FlexMaskWarp", { + "TOP_DESCRIPTION": "Applies warping effects to the mask, modulated by a selected feature.", + "ADDITIONAL_INFO": """ +- `warp_type`: The type of warping effect to apply. Each creates a different distortion pattern. + - Options: "perlin" (noise-based), "radial" (circular), "swirl" (spiral) +- `frequency`: Controls the scale of the warping effect (0.01 to 1.0). Higher values create more rapid changes in the warp pattern. +- `max_amplitude`: Maximum amplitude of the warping effect (0.1 to 500.0). Higher values create more extreme distortions. +- `octaves`: For noise-based warps, adds detail at different scales (1 to 8). More octaves create more complex, detailed patterns. + +The intensity of the warping effect is determined by the selected feature's value at each frame. +""" +}) + +add_node_config("FlexMaskTransform", { + "TOP_DESCRIPTION": "Applies geometric transformations to the mask, modulated by a selected feature.", + "ADDITIONAL_INFO": """ +- `transform_type`: The type of transformation to apply. + - Options: "translate", "rotate", "scale" +- `max_x_value`: Maximum horizontal component of the transformation (-1000.0 to 1000.0). Positive values move right, negative left. +- `max_y_value`: Maximum vertical component of the transformation (-1000.0 to 1000.0). Positive values move up, negative down. + +The extent of the transformation is determined by the selected feature's value at each frame. +""" +}) + +add_node_config("FlexMaskMath", { + "TOP_DESCRIPTION": "Performs mathematical operations between two masks, modulated by a selected feature.", + "ADDITIONAL_INFO": """ +- `mask_b`: Second mask to combine with the input mask. +- `combination_method`: Mathematical operation to apply. + - Options: "add", "subtract", "multiply", "minimum", "maximum" + +The strength of the combination is determined by the selected feature's value at each frame. +""" +}) + +add_node_config("FlexMaskOpacity", { + "TOP_DESCRIPTION": "Applies opacity modulation to the mask based on a selected feature.", + "ADDITIONAL_INFO": """ +- `max_opacity`: Maximum opacity to apply to the mask (0.0 to 1.0). Higher values allow for more opaque masks. + +The actual opacity applied is determined by the product of max_opacity, feature value, and strength. + +This node is useful for creating masks that fade in and out based on the selected feature, allowing for smooth transitions and effects that respond to various inputs like audio, time, or other extracted features. +""" +}) + +add_node_config("FlexMaskVoronoiScheduled", { + "TOP_DESCRIPTION": "Generates a Voronoi noise mask with parameters modulated by a selected feature according to a specified formula.", + "ADDITIONAL_INFO": """ +- `distance_metric`: Method used to calculate distances in the Voronoi diagram. Options include various mathematical norms and custom patterns. +- `scale`: Base scale of the Voronoi cells (0.1 to 10.0). Larger values create bigger cells. +- `detail`: Number of Voronoi cells (10 to 1000). More cells create more intricate patterns. +- `randomness`: Degree of randomness in cell placement (0.0 to 5.0). Higher values create more chaotic patterns. +- `seed`: Random seed for reproducible results (0 to 2^64 - 1). +- `x_offset`: Horizontal offset of the Voronoi pattern (-1000.0 to 1000.0). +- `y_offset`: Vertical offset of the Voronoi pattern (-1000.0 to 1000.0). +- `feature_param`: Which parameter to modulate based on the feature ("scale", "detail", "randomness", "seed", "x_offset", "y_offset"). +- `formula`: Mathematical formula used to map the feature value to the feature parameter. Options include "Linear", "Quadratic", "Cubic", "Sinusoidal", and "Exponential". +- `a` and `b`: Parameters for fine-tuning the chosen formula (0.1 to 10.0). + +Credit for the heavy lifting for this node goes to https://github.com/alanhuang67/ +""" +}) + +add_node_config("FlexMaskBinary", { + "TOP_DESCRIPTION": "Applies binary thresholding to the mask, modulated by a selected feature.", + "ADDITIONAL_INFO": """ +- `threshold`: Base threshold value for binarization (0.0 to 1.0). Pixels above this value become white, below become black. +- `method`: Thresholding method to use. + - Options: "simple" (basic threshold), "adaptive" (local adaptive threshold), "hysteresis" (double threshold with connectivity), "edge" (Canny edge detection) +- `max_smoothing`: Maximum amount of Gaussian smoothing to apply (0 to 51, odd values only). Higher values create smoother masks. +- `max_edge_enhancement`: Maximum strength of edge enhancement (0.0 to 10.0). Higher values create more pronounced edges. +- `feature_param`: Which parameter to modulate based on the feature value. + - Options: "threshold" (adjusts threshold), "smoothing" (adjusts smoothing), "edge_enhancement" (adjusts edge enhancement), "none" (no modulation) + +The binary mask operation is applied with strength determined by the selected feature's value at each frame. This node is useful for creating sharp, high-contrast masks that can be dynamically adjusted based on various inputs like audio, time, or other extracted features. +""" +}) + +add_node_config("FlexMaskEmanatingRings", { + "TOP_DESCRIPTION": "Creates dynamic, expanding ring patterns that emanate from the edges of the input mask.", + "ADDITIONAL_INFO": """ +- `num_rings`: Number of concentric rings to generate (1 to 50). More rings create a denser, more complex pattern. +- `max_ring_width`: Maximum width of each ring as a fraction of the total distance (0.01 to 0.9). Larger values create wider, more prominent rings that overlap more. +- `wave_speed`: Speed at which the rings expand outward (0.01 to 0.5). Higher values create faster-moving, more dynamic patterns. +- `feature_param`: Determines which aspect of the effect is modulated by the input feature. + - Options: "num_rings" (varies ring count), "ring_width" (adjusts ring thickness), "wave_speed" (changes expansion rate), "all" (modulates all parameters) + +This node creates a mesmerizing effect of rings expanding from the edges of the input mask. The rings start thick at the mask boundary and thin out as they move outward, creating a pulsating, wave-like appearance. The effect can be subtle and smooth or bold and dynamic depending on the parameter settings. + +The feature input can be used to dynamically adjust the effect over time, allowing for rhythmic pulsing (e.g., synced to audio) or gradual evolution of the pattern. When the feature value is below the threshold, the animation continues but no new rings are generated, creating a smooth transition effect. +""" +}) + +add_node_config("FlexMaskRandomShapes", { + "TOP_DESCRIPTION": "Generates dynamic masks with randomly placed shapes, modulated by a selected feature.", + "ADDITIONAL_INFO": """ +- `max_num_shapes`: Maximum number of shapes to generate (1 to 100). The actual number is modulated by the feature if 'num_shapes' is selected as the feature parameter. +- `max_shape_size`: Maximum size of shapes as a fraction of the frame size (0.01 to 1.0). The actual size is modulated by the feature if 'shape_size' is selected as the feature parameter. +- `appearance_duration`: Number of frames over which shapes appear (1 to 100). Modulated by the feature if selected as the feature parameter. +- `disappearance_duration`: Number of frames over which shapes disappear (1 to 100). Modulated by the feature if selected as the feature parameter. +- `appearance_method`: How shapes appear and disappear. + - Options: "grow" (shapes grow/shrink), "pop" (shapes appear/disappear suddenly), "fade" (shapes fade in/out) +- `easing_function`: Determines the rate of change for appearance/disappearance. + - Options: "linear", "ease_in_out", "bounce", "elastic" +- `shape_type`: Type of shape to generate. Includes various geometric shapes and a "random" option. +- `feature_param`: Aspect of the effect modulated by the input feature. + - Options: "num_shapes", "shape_size", "appearance_duration", "disappearance_duration" + +This node creates a dynamic mask with randomly placed shapes that appear and disappear over time. The number, size, and timing of the shapes can be modulated by the input feature, creating effects that respond to various inputs like audio, time, or other extracted features. The shapes can grow, pop, or fade in and out, with different easing functions for smooth or bouncy transitions. +""" +}) + +add_node_config("FlexMaskWavePropagation", { + "TOP_DESCRIPTION": "Good luck with this one...Simulates wave-like abstract distortions propagating from the edges of the input mask.", + "ADDITIONAL_INFO": """ +- `wave_speed`: Controls the rate of wave propagation (0.1 to 100.0). Higher values create faster-moving, more rapidly evolving patterns. +- `wave_amplitude`: Determines the intensity of the wave effect (0.1 to 2.0). Larger values create more pronounced, exaggerated distortions. +- `wave_decay`: Rate at which waves fade out over time (0.9 to 10.0). Lower values cause waves to dissipate quickly, while higher values allow waves to persist and interact more. +- `wave_frequency`: Frequency of the wave oscillations (0.01 to 10.0). Higher values create more rapid, ripple-like effects, while lower values produce smoother, more gradual undulations. +- `max_wave_field`: Maximum allowed intensity for the wave field (10.0 to 10000.0). This parameter prevents the effect from becoming too extreme over time. + +This node creates a dynamic, fluid-like effect where waves seem to emanate from the edges of the input mask. The waves propagate outward, interacting with each other and creating complex, evolving patterns. The effect can range from subtle, water-like ripples to intense, psychedelic distortions depending on the parameter settings. + +The wave propagation is particularly sensitive to the interplay between `wave_speed`, `wave_amplitude`, and `wave_decay`. High speed with low decay can create a turbulent, chaotic effect, while lower speed with higher decay produces a more serene, flowing appearance. + +The feature input modulates the intensity of new waves being generated, allowing for dynamic control over the effect's strength. This can be used to create pulsating effects synchronized with audio or other time-varying inputs. +""" +}) + +add_node_config("FeatureExtractorBase", { + "BASE_DESCRIPTION": """ + Features are used to modulate other RyanOnTheInside nodes. + + You can replace this feature with any of the others, and it will work. + + Available features include Audio, Motion, MIDI, Pitch, Proximity, Depth, Time, Color, Brightness, and more. + +### Parameters: +- `frame_rate`: Frame rate of the video +- `frame_count`: Total number of frames +""" +}) + +add_node_config("TimeFeatureNode", { + "TOP_DESCRIPTION": "Produces a feature that changes over time based on the selected effect type. This can be used to create dynamic, time-varying mask modulations.", + "ADDITIONAL_INFO": """ +- `effect_type`: Type of time-based pattern to apply. + - Options: "smooth" (gradual), "accelerate" (speeds up), "pulse" (rhythmic), "sawtooth" (repeating ramp), "bounce" (up and down) +- `speed`: How quickly the effect progresses (0.1 to 10.0, default: 1.0). Higher values create faster changes. +- `offset`: Shifts the starting point of the effect (0.0 to 1.0, default: 0.0). Useful for staggering multiple effects. + + +""" +}) + +add_node_config("AudioFeatureNode", { + "TOP_DESCRIPTION": "Analyzes the input audio to extract the specified feature. This feature can then be used to modulate masks based on audio characteristics.", + "ADDITIONAL_INFO": """ +- `audio`: Input audio to analyze (AUDIO type) +- `feature_type`: Type of audio feature to extract. + - Options: "amplitude_envelope", "rms_energy", "spectral_centroid", "onset_detection", "chroma_features" + +""" +}) + +add_node_config("DepthFeatureNode", { + "TOP_DESCRIPTION": "Analyzes the input depth maps to extract the specified depth-related feature. This feature can be used to modulate masks based on depth information in the scene.", + "ADDITIONAL_INFO": """ +- `depth_maps`: Input depth maps to analyze (IMAGE type) +- `feature_type`: Type of depth feature to extract. + - Options: "mean_depth", "depth_variance", "depth_range", "gradient_magnitude", "foreground_ratio", "midground_ratio", "background_ratio" + +""" +}) + +add_node_config("ColorFeatureNode", { + "TOP_DESCRIPTION": "Extracts color-related features from video frames for mask modulation.", + "ADDITIONAL_INFO": """ +- `feature_type`: Type of color feature to extract + - Options: "dominant_color" (most prevalent color), "color_variance" (variation in colors), "saturation" (color intensity), "red_ratio" (proportion of red), "green_ratio" (proportion of green), "blue_ratio" (proportion of blue) + +Analyzes the input video frames to extract the specified color-related feature. This feature can be used to modulate masks based on color information in the scene, creating effects that respond to color changes over time. +""" +}) + +add_node_config("BrightnessFeatureNode", { + "TOP_DESCRIPTION": "Extracts brightness-related features from video frames for mask modulation.", + "ADDITIONAL_INFO": """ +- `feature_type`: Type of brightness feature to extract + - Options: "mean_brightness" (average brightness), "brightness_variance" (variation in brightness), "dark_ratio" (proportion of dark areas), "mid_ratio" (proportion of mid-tone areas), "bright_ratio" (proportion of bright areas) + +Analyzes the input video frames to extract the specified brightness-related feature. This feature can be used to modulate masks based on lighting changes in the scene, allowing for effects that respond to overall brightness or specific tonal ranges. +""" +}) + +add_node_config("MotionFeatureNode", { + "TOP_DESCRIPTION": "Extracts motion-related features from video frames for mask modulation.", + "ADDITIONAL_INFO": """ +- `feature_type`: Type of motion feature to extract + - Options: "mean_motion" (average motion), "max_motion" (peak motion), "motion_direction" (overall direction), "horizontal_motion" (left-right movement), "vertical_motion" (up-down movement), "motion_complexity" (intricacy of motion) +- `flow_method`: Technique used for optical flow calculation + - Options: "Farneback" (dense flow), "LucasKanade" (sparse flow), "PyramidalLK" (multi-scale sparse flow) +- `flow_threshold`: Minimum motion magnitude to consider (0.0 to 10.0). Higher values ignore subtle movements. +- `magnitude_threshold`: Relative threshold for motion magnitude (0.0 to 1.0). Higher values focus on areas of stronger motion. + +Analyzes the input video frames to extract the specified motion-related feature using optical flow techniques. This feature can be used to modulate masks based on movement in the scene, creating effects that respond to motion intensity, direction, or complexity. +""" +}) + +add_node_config("AudioFeatureExtractor", { + "TOP_DESCRIPTION": "Analyzes the input audio to extract the specified feature. The resulting feature can be used to modulate masks based on audio characteristics.", + "ADDITIONAL_INFO": """ +- `audio`: Input audio to analyze (AUDIO type) +- `feature_pipe`: Feature pipe for frame information (FEATURE_PIPE type) +- `feature_type`: Type of audio feature to extract + - Options: "amplitude_envelope", "rms_energy", "spectral_centroid", "onset_detection", "chroma_features" + +""" +}) + +add_node_config("PitchFeatureExtractor", { + "TOP_DESCRIPTION": "Extracts pitch-related features from audio input.", + "ADDITIONAL_INFO": """ +- `audio`: Input audio to analyze (AUDIO type) +- `feature_pipe`: Feature pipe containing frame information (FEATURE_PIPE type) +- `feature_type`: Type of pitch feature to extract: + - "pitch_filtered": Filtered pitch values + - "pitch_direction": Direction of pitch changes + - "vibrato_signal": Vibrato signal + - "vibrato_intensity": Intensity of vibrato +- `window_size`: Size of the analysis window (0 for default) +- `pitch_tolerance`: Tolerance for pitch detection (0.0 to 1.0) +- `pitch_range_collections`: (Optional) Collections of pitch ranges to consider (PITCH_RANGE_COLLECTION type) + +This node extracts various pitch-related features from the input audio, which can be used for further analysis or mask modulation. +""" +}) + +add_node_config("PitchRangeByNoteNode", { + "TOP_DESCRIPTION": "Creates pitch ranges based on specified MIDI notes.", + "ADDITIONAL_INFO": """ +- `chord_only`: If true, only detects when all specified notes are present simultaneously (BOOLEAN) +- `pitch_tolerance_percent`: Tolerance percentage for pitch detection (0.0 to 100.0) +- `notes`: IGNORE THIS. Certain limitations prevent me from hiding it completely. Love, Ryan +- `previous_range_collection`: (Optional) Previous pitch range collection to append to (PITCH_RANGE_COLLECTION type) + +This node creates pitch ranges based on specified MIDI notes, which can be used for targeted pitch detection in audio analysis. +""" +}) + +add_node_config("PitchRangePresetNode", { + "TOP_DESCRIPTION": "Creates preset pitch ranges for common vocal ranges.", + "ADDITIONAL_INFO": """ +- `preset`: Preset vocal range to use: + - Options: "Bass", "Baritone", "Tenor", "Alto", "Mezzo-soprano", "Soprano", "Contralto" +- `previous_range_collection`: (Optional) Previous pitch range collection to append to (PITCH_RANGE_COLLECTION type) + +This node provides preset pitch ranges corresponding to common vocal ranges, which can be used for voice-specific audio analysis. +""" +}) + +add_node_config("PitchRangeNode", { + "TOP_DESCRIPTION": "Creates a custom pitch range for audio analysis.", + "ADDITIONAL_INFO": """ +- `min_pitch`: Minimum frequency of the pitch range in Hz (20.0 to 2000.0) +- `max_pitch`: Maximum frequency of the pitch range in Hz (20.0 to 2000.0) +- `previous_range_collection`: (Optional) Previous pitch range collection to append to (PITCH_RANGE_COLLECTION type) + +This node allows you to create a custom pitch range by specifying the minimum and maximum frequencies. This can be useful for targeting specific frequency ranges in audio analysis, such as isolating particular instruments or vocal ranges. + +The created pitch range can be combined with other pitch ranges or used independently in pitch-related feature extraction nodes. +""" +}) + +add_node_config("EmptyImageFromAudio", { + "TOP_DESCRIPTION": "Creates an empty image sequence based on audio input.", + "ADDITIONAL_INFO": """ +- `audio`: Input audio (AUDIO type) +- `frame_rate`: Frame rate of the output image sequence (0.1 to 120 fps) +- `height`: Height of the output images (16 to 4096 pixels) +- `width`: Width of the output images (16 to 4096 pixels) + +This node generates an empty image sequence dimensions and frame rate, based on the duration of the input audio. +It's useful for creating a blank canvas for further image processing or visualization that matches the length of an audio track. +""" +}) + +add_node_config("EmptyMaskFromAudio", { + "TOP_DESCRIPTION": "Creates an empty mask sequence based on audio input.", + "ADDITIONAL_INFO": """ +- `audio`: Input audio (AUDIO type) +- `frame_rate`: Frame rate of the output mask sequence (0.1 to 120 fps) +- `height`: Height of the output masks (16 to 4096 pixels) +- `width`: Width of the output masks (16 to 4096 pixels) + +This node generates an empty mask sequence with the specified dimensions and frame rate, based on the duration of the input audio. +It's useful for creating a blank mask for further processing or effects that match the length of an audio track. +""" +}) + +add_node_config("TimeFeatureNode", { + "TOP_DESCRIPTION": "Generates time-based features for mask modulation.", + "ADDITIONAL_INFO": """ +- `video_frames`: Input video frames (IMAGE type) +- `frame_rate`: Frame rate of the video +- `effect_type`: Type of time-based pattern to apply + - Options: "smooth" (gradual), "accelerate" (speeds up), "pulse" (rhythmic), "sawtooth" (repeating ramp), "bounce" (up and down) +- `speed`: How quickly the effect progresses (0.1 to 10.0, default: 1.0). Higher values create faster changes. +- `offset`: Shifts the starting point of the effect (0.0 to 1.0, default: 0.0). Useful for staggering multiple effects. + +Generates a feature that changes over time based on the selected effect type. This can be used to create dynamic, time-varying mask modulations. +""" +}) + +add_node_config("DepthFeatureNode", { + "TOP_DESCRIPTION": "Extracts depth-related features from depth maps for mask modulation.", + "ADDITIONAL_INFO": """ +- `video_frames`: Input video frames (IMAGE type) +- `frame_rate`: Frame rate of the video +- `depth_maps`: Input depth maps to analyze (IMAGE type) +- `feature_type`: Type of depth feature to extract + - Options: "mean_depth", "depth_variance", "depth_range", "gradient_magnitude", "foreground_ratio", "midground_ratio", "background_ratio" + +Analyzes the input depth maps to extract the specified depth-related feature. This feature can be used to modulate masks based on depth information in the scene. +""" +}) + +add_node_config("AreaFeatureNode", { + "TOP_DESCRIPTION": "Extracts area-related features from mask sequences for mask modulation.", + "ADDITIONAL_INFO": """ +- `masks`: Input mask sequence to analyze (MASK type) +- `feature_type`: Type of area feature to extract + - Options: + - "total_area" (sum of pixels above threshold) + - "largest_contour" (area of the largest contiguous region) + - "bounding_box" (area of the bounding box containing the largest region) +- `threshold`: Threshold value for considering pixels as part of the area (0.0 to 1.0) + +This node analyzes the input mask sequence to extract the specified area-related feature. The resulting feature can be used to modulate masks based on changes in area over time, allowing for effects that respond to the size or extent of masked regions in the scene. +""" +}) + +add_node_config("ProximityFeatureNode", { + "TOP_DESCRIPTION": "Calculates a proximity feature based on the distance between anchor and query locations in video frames.", + "ADDITIONAL_INFO": """ +- `video_frames`: Input video frames (IMAGE type) +- `frame_rate`: Frame rate of the video (FLOAT type, default: 30.0, min: 1.0, max: 120.0, step: 0.1) +- `anchor_locations`: Locations of anchor points (LOCATION type) +- `query_locations`: Locations of query points (LOCATION type) +- `distance_metric`: Distance metric to use for calculation (Options: "euclidean", "manhattan", "chebyshev") + +This node calculates a proximity feature based on the specified distance metric between anchor and query locations in the input video frames. The resulting feature can be used to modulate other effects based on spatial relationships. +""" +}) + +add_node_config("LocationFromMask", { + "TOP_DESCRIPTION": "This is for use with proximity features. This generates locations from mask inputs using various methods.", + "ADDITIONAL_INFO": """ +- `masks`: Input masks (MASK type) +- `method`: Method to use for location extraction (Options: "mask_center", "mask_boundary", "mask_top_left", "mask_bottom_right") +- `depth_maps`: (Optional) Input depth maps. The depth map provides a value for the z coordinate of every location. If no depth map is provided, the value defaults to .5. The z coordinate is far less granular than x and y, as all we have are relative normalized depth per frame (0 to 1). + +This node generates locations from the input masks using the specified method. The locations can be used as anchor or query points for proximity calculations or other spatially dependent effects. +""" +}) + +add_node_config("LocationFromPoint", { + "TOP_DESCRIPTION": "Generates locations from specified x, y, and z coordinates.", + "ADDITIONAL_INFO": """ +- `x`: X-coordinate of the location (FLOAT, default: 0.0, min: 0.0, step: 0.01) +- `y`: Y-coordinate of the location (FLOAT, default: 0.0, min: 0.0, step: 0.01) +- `batch_count`: Number of locations to generate (INT, default: 1, min: 1) +- `z`: Z-coordinate of the location (FLOAT, default: 0.0, min: 0.0, max: 1.0, step: 0.01) + +This node generates a batch of locations based on the specified x, y, and z coordinates. +""" +}) + +add_node_config("LocationTransform", { + "TOP_DESCRIPTION": "Transforms locations based on a feature and specified transformation type.", + "ADDITIONAL_INFO": """ +- `locations`: Input locations to be transformed (LOCATION type) +- `feature`: Feature used to modulate the transformation (FEATURE type) +- `transformation_type`: Type of transformation to apply ("translate" or "scale") +- `transformation_value`: Value of the transformation (FLOAT, default: 1.0) + +This node transforms the input locations based on the specified transformation type and value, modulated by the input feature. +""" +}) + +add_node_config("EmitterMovement", { + "TOP_DESCRIPTION": """These parameters work together to create complex, periodic movements for particle emitters. +By adjusting frequencies and amplitudes, you can achieve various patterns like circles, +figure-eights, or more chaotic motions. The direction parameters add extra dynamism by +altering the angle of particle emission over time.""", + "ADDITIONAL_INFO": """ +Position Control: +- `emitter_x_frequency`: How quickly the emitter moves horizontally (0.0 to 10.0). Higher values create faster side-to-side motion. +- `emitter_x_amplitude`: Maximum horizontal distance the emitter moves (0.0 to 0.5). Larger values create wider movements. +- `emitter_y_frequency`: How quickly the emitter moves vertically (0.0 to 10.0). Higher values create faster up-and-down motion. +- `emitter_y_amplitude`: Maximum vertical distance the emitter moves (0.0 to 0.5). Larger values create taller movements. +Direction Control: +- `direction_frequency`: How quickly the emission angle changes (0.0 to 10.0). Higher values create more rapid direction changes. +- `direction_amplitude`: Maximum angle change in degrees (0.0 to 180.0). Larger values allow for more extreme direction shifts. + +Feature Modulation: +- `feature`: Optional feature to modulate the movement (FEATURE type) +- `feature_param`: Parameter to be modulated by the feature ("emitter_x_frequency", "emitter_y_frequency", or "direction_frequency") + + +""" +}) + +add_node_config("ParticleEmitter", { + "TOP_DESCRIPTION": "This node creates a particle emitter with the specified properties. It can be used in conjunction with particle system mask nodes to create complex particle effects. They can be chained together to add many to a given simulation.", + "ADDITIONAL_INFO": """ +- `emitter_x`: X-coordinate of the emitter (0.0 to 1.0, left to right) +- `emitter_y`: Y-coordinate of the emitter (0.0 to 1.0, up to down) +- `particle_direction`: Direction of particle emission in degrees (0.0 to 360.0, clockwise) +- `particle_spread`: Spread angle of particle emission in degrees (0.0 to 360.0, clockwise) +- `particle_size`: Size of emitted particles (1.0 to 400.0) +- `particle_speed`: Speed of emitted particles (1.0 to 1000.0) +- `emission_rate`: Rate of particle emission (0.1 to 100.0) +- `color`: Color of emitted particles (RGB string) +- `initial_plume`: Initial burst of particles (0.0 to 1.0) +- `start_frame`: Frame to start the emission (0 to 10000) +- `end_frame`: Frame to end the emission (0 to 10000) +- `emission_radius`: Defaulting to 0 (a point), this value changes the radius of the area from which the particles are emitted. The open 'mouth' of the emitter. + +Optional inputs: +- `emitter_movement`: Movement settings for the emitter (EMITTER_MOVEMENT type) +- `spring_joint_setting`: Spring joint configuration for particles (SPRING_JOINT_SETTING type) +- `particle_modulation`: Modulation settings for particle properties over time (PARTICLE_MODULATION type) +""" +}) + +add_node_config("MovingShape", { + "TOP_DESCRIPTION": "Generate animated mask sequences featuring a moving shape with customizable parameters.", + "ADDITIONAL_INFO": """ +- `frame_width`: Width of each frame (1-3840 pixels) +- `frame_height`: Height of each frame (1-2160 pixels) +- `num_frames`: Number of frames in the sequence (1-120) +- `rgb`: Color of the shape in RGB format, e.g., "(255,255,255)" +- `shape`: Shape type ("square", "circle", or "triangle") +- `shape_width_percent`: Width of the shape as a percentage of frame width (0-100%) +- `shape_height_percent`: Height of the shape as a percentage of frame height (0-100%) +- `shape_start_position_x`: Starting X position of the shape (-100 to 100) +- `shape_start_position_y`: Starting Y position of the shape (-100 to 100) +- `shape_end_position_x`: Ending X position of the shape (-100 to 100) +- `shape_end_position_y`: Ending Y position of the shape (-100 to 100) +- `movement_type`: Type of movement ("linear", "ease_in_out", "bounce", or "elastic") +- `grow`: Growth factor of the shape during animation (0-100) +- `palindrome`: Whether to reverse the animation sequence (True/False) +- `delay`: Number of static frames at the start (0-60) + +This node creates a mask sequence with a moving shape, allowing for various animations and transformations. +""" +}) + +add_node_config("TextMaskNode", { + "TOP_DESCRIPTION": "Generate mask and image sequences featuring customizable text.", + "ADDITIONAL_INFO": """ +- `width`: Width of the output image (1-8192 pixels) +- `height`: Height of the output image (1-8192 pixels) +- `text`: The text to be rendered +- `font`: Font to be used (selectable from system fonts) +- `font_size`: Size of the font (1-1000) +- `font_color`: Color of the text in RGB format, e.g., "(255,255,255)" +- `background_color`: Color of the background in RGB format, e.g., "(0,0,0)" +- `x_position`: Horizontal position of the text (0.0-1.0, where 0.5 is center) +- `y_position`: Vertical position of the text (0.0-1.0, where 0.5 is center) +- `rotation`: Rotation angle of the text (0-360 degrees) +- `max_width_ratio`: Maximum width of text as a ratio of image width (0.1-1.0) +- `batch_size`: Number of images to generate in the batch (1-10000) +""" +}) + +add_node_config("_mfc", { + "TOP_DESCRIPTION": "Basic mask from color.", + "ADDITIONAL_INFO": """ +This is an abstract base class that provides common functionality for mask function components. +It is not meant to be used directly but serves as a foundation for other mask-related nodes. + +Key features: +- Implements common methods for mask operations +- Provides a structure for derived classes to follow +- Ensures consistency across different mask function components +""" +}) + +add_node_config("DownloadOpenUnmixModel", { + "TOP_DESCRIPTION": "Downloads and loads Open Unmix models for audio classification", + "ADDITIONAL_INFO": """ +-umxl (default) trained on private stems dataset of compressed stems. Note, that the weights are only licensed for non-commercial use (CC BY-NC-SA 4.0). + +-umxhq trained on MUSDB18-HQ which comprises the same tracks as in MUSDB18 but un-compressed which yield in a full bandwidth of 22050 Hz. + + +""" +}) + +add_node_config("FeatureMixer", { + "TOP_DESCRIPTION": "Advanced feature modulation node for fine-tuning and shaping feature values.", + "ADDITIONAL_INFO": """ +- `feature`: Input feature to be processed (FEATURE type) +- `base_gain`: Overall amplification of the feature values (0.0 to 10.0). Higher values increase the overall intensity. +- `floor`: Minimum value for the processed feature (0.0 to 1.0). Prevents values from going below this threshold. +- `ceiling`: Maximum value for the processed feature (0.0 to 10.0). Caps values at this upper limit. +- `peak_sharpness`: Sharpness of peaks in the feature curve (0.1 to 10.0). Higher values create more pronounced peaks. +- `valley_sharpness`: Sharpness of valleys in the feature curve (0.1 to 10.0). Higher values create deeper valleys. +- `attack`: Speed at which the envelope follower responds to increasing values (0.01 to 1.0). Lower values create slower attack. +- `release`: Speed at which the envelope follower responds to decreasing values (0.01 to 1.0). Lower values create slower release. +- `smoothing`: Amount of smoothing applied to the final curve (0.0 to 1.0). Higher values create smoother transitions. + +This node provides extensive control over feature modulation, allowing for complex shaping of feature values over time. It combines multiple processing stages including gain, waveshaping, envelope following, and smoothing to create highly customized feature curves for mask modulation. + +Outputs: +- Processed FEATURE +- Visualization of the processed feature (IMAGE type) +""" +}) + +add_node_config("FeatureRebase", { + "TOP_DESCRIPTION": "Rebases feature values within specified thresholds.", + "ADDITIONAL_INFO": """ +- `feature`: Input feature to be rebased (FEATURE type) +- `lower_threshold`: Lower threshold for feature values (FLOAT, default: 0.0, min: 0.0, max: 1.0, step: 0.01) +- `upper_threshold`: Upper threshold for feature values (FLOAT, default: 1.0, min: 0.0, max: 1.0, step: 0.01) +- `invert_output`: Whether to invert the output feature values (BOOLEAN, default: False) + +This node rebases the input feature values within the specified thresholds and normalizes them. +""" +}) + +add_node_config("FeatureMath", { + "TOP_DESCRIPTION": "Performs mathematical operations between a feature's values and a float value.", + "ADDITIONAL_INFO": """ +- `feature`: Input feature (FEATURE type) +- `y`: Input value (FLOAT type) +- `operation`: Mathematical operation to perform ("add", "subtract", "multiply", "divide", "max", "min"). Determines how the feature's values are combined with y. + +This node takes a feature and performs the specified operation between its values and the float value y, returning the processed feature and its visualization. +""" +}) + + +add_node_config("FeatureScaler", { + "TOP_DESCRIPTION": "Scales and transforms feature values using various mathematical functions.", + "ADDITIONAL_INFO": """ + - `feature`: Input feature to be scaled (FEATURE type) + - `scale_type`: Type of scaling to apply ("linear", "logarithmic", "exponential", "inverse") + - `min_output`: Minimum output value after scaling (0.0 to 1.0) + - `max_output`: Maximum output value after scaling (0.0 to 1.0) + - `exponent`: Exponent for exponential scaling (0.1 to 10.0) + """ +}) + +add_node_config("FeatureCombine", { + "TOP_DESCRIPTION": "Performs mathematical operations between two features.", + "ADDITIONAL_INFO": """ + - `feature1`: First input feature (FEATURE type) + - `feature2`: Second input feature (FEATURE type) + - `operation`: Mathematical operation to perform ("add", "subtract", "multiply", "divide", "max", "min"). Determines how the two features are combined. + - `weight1`: Weight applied to feature1 (0.0 to 1.0). Higher values give more importance to feature1 in the operation. + - `weight2`: Weight applied to feature2 (0.0 to 1.0). Higher values give more importance to feature2 in the operation. + """ +}) + +add_node_config("FeatureSmoothing", { + "TOP_DESCRIPTION": "Applies various smoothing techniques to a feature.", + "ADDITIONAL_INFO": """ + - `feature`: Input feature to be smoothed (FEATURE type) + - `smoothing_type`: Type of smoothing to apply ("moving_average", "exponential", "gaussian") + - `window_size`: Size of the smoothing window for moving average and gaussian (3 to 21, odd numbers only). Larger values create smoother transitions but may reduce responsiveness. + - `alpha`: Smoothing factor for exponential smoothing (0.0 to 1.0). Higher values make the feature respond more quickly to changes, while lower values create a more gradual, smoothed effect. + - `sigma`: Standard deviation for gaussian smoothing (0.1 to 5.0). Higher values create a more pronounced smoothing effect. + """ +}) + +add_node_config("FeatureOscillator", { + "TOP_DESCRIPTION": "Generates oscillating patterns based on the input feature.", + "ADDITIONAL_INFO": """ + - `feature`: Input feature to base oscillation on (FEATURE type) + - `oscillator_type`: Type of oscillation ("sine", "square", "sawtooth", "triangle") + - `frequency`: Frequency of oscillation (0.1 to 10.0) + - `amplitude`: Amplitude of oscillation (0.0 to 1.0) + - `phase_shift`: Phase shift of oscillation (0.0 to 2π) + - `blend`: Blend factor between original feature and oscillation (0.0 to 1.0) + """ +}) + +add_node_config("FeatureFade", { + "TOP_DESCRIPTION": "Fades between two features based on a fader value or a control feature.", + "ADDITIONAL_INFO": """ +- `feature1`: First input feature to fade from (FEATURE type) +- `feature2`: Second input feature to fade to (FEATURE type) +- `fader`: Static fader value to control the blend between feature1 and feature2 (0.0 to 1.0). 0.0 is 100 percent feature1, 1.0 is 100 percent feature2. +- `control_feature`: Optional feature to dynamically control the fader value (FEATURE type). If provided, the fader value will be modulated by this feature. + +Shoutout @cyncratic +""" +}) + +add_node_config("FeatureTruncateOrExtend", { + "TOP_DESCRIPTION": "Adjusts the length of a feature to match a target feature pipe, either by truncating or extending it.", + "ADDITIONAL_INFO": """ +- `feature`: Input feature to be adjusted (FEATURE type) +- `target_feature_pipe`: Target feature pipe to match the length (FEATURE type) +- `fill_method`: Method to use when extending the feature: + - "zeros": Fills with 0's + - "ones": Fills with 1's + - "average": Fills with the average value of the source feature + - "random": Fills with random values between 0 and 1 + - "repeat": Repeats the source values from the beginning +- `invert_output`: Whether to invert the output feature values (True/False) + +This node adjusts the length of the input feature to match the length of the target feature pipe. If the input feature is longer, it will be truncated. If it's shorter, it will be extended using the specified fill method. + +The "repeat" fill method is particularly useful for maintaining patterns or rhythms when extending the feature. + +Use cases: +1. Adapting audio-extracted features for shorter or longer video animations +2. Synchronizing features of different lengths for complex animations +3. Creating looping patterns by repeating shorter features + +Outputs: +- Adjusted FEATURE +- Visualization of the adjusted feature (IMAGE type) +""" +}) + + +add_node_config("FeatureToWeightsStrategy", { + "TOP_DESCRIPTION": "Converts a FEATURE input into a WEIGHTS_STRATEGY for use with IPAdapter nodes.", + "ADDITIONAL_INFO": """ +- `feature`: Input feature to be converted (FEATURE type) + +This node takes a FEATURE input and converts it into a WEIGHTS_STRATEGY that can be used with IPAdapter nodes. It creates a custom weights strategy based on the feature values for each frame. +This node is particularly useful for creating dynamic, feature-driven animations with IPAdapter, where the strength of the adaptation can vary over time based on extracted features from audio, video, or other sources. + +""" +}) + +NODE_CONFIGS["FlexImageBase"] = { + "BASE_DESCRIPTION": """ +- `images`: Input image sequence (IMAGE type) +- `feature`: Feature used to modulate the effect (FEATURE type) +- `feature_pipe`: Feature pipe containing frame information (FEATURE_PIPE type) +- `strength`: Overall strength of the effect (0.0 to 1.0) +- `feature_threshold`: Minimum feature value to apply the effect (0.0 to 1.0) +- `feature_mode`: How the feature modulates the parameter ("relative" or "absolute"). Relative mode adjusts the parameter based on its current value, while absolute mode directly sets the parameter to the feature value. +""" +} + +add_node_config("FlexImageEdgeDetect", { + "TOP_DESCRIPTION": "Applies edge detection to the image using the Canny algorithm.", + "ADDITIONAL_INFO": """ +- `low_threshold`: Lower bound for the hysteresis thresholding (0 to 255). +- `high_threshold`: Upper bound for the hysteresis thresholding (0 to 255). +- `feature_param`: Parameter to modulate based on the feature. Options are "low_threshold", "high_threshold", "None". + +This node detects edges in the image using the Canny edge detection algorithm. The thresholds control the sensitivity of the edge detection. +""" +}) + +add_node_config("FlexImagePosterize", { + "TOP_DESCRIPTION": "Applies a posterization effect to the image, reducing the number of colors.", + "ADDITIONAL_INFO": """ +- `max_levels`: Maximum number of color levels per channel (2 to 256). +- `dither_strength`: Intensity of dithering effect (0.0 to 1.0). +- `channel_separation`: Degree of separation between color channels (0.0 to 1.0). +- `gamma`: Gamma correction applied before posterization (0.1 to 2.2). +- `feature_param`: Parameter to modulate based on the feature. Options are "max_levels", "dither_strength", "channel_separation", "gamma", "None". + +This node reduces the number of colors in the image, creating a posterized effect. Dithering can be applied to reduce banding. +""" +}) + +add_node_config("FlexImageKaleidoscope", { + "TOP_DESCRIPTION": "Creates a kaleidoscope effect by mirroring and rotating segments of the image.", + "ADDITIONAL_INFO": """ +- `segments`: Number of mirror segments (2 to 32). +- `center_x`: X-coordinate of the effect center (0.0 to 1.0). +- `center_y`: Y-coordinate of the effect center (0.0 to 1.0). +- `zoom`: Zoom factor for the effect (0.1 to 2.0). +- `rotation`: Rotation angle of the effect (0.0 to 360.0 degrees). +- `precession`: Rate of rotation change over time (-1.0 to 1.0). +- `speed`: Speed of the effect animation (0.1 to 5.0). +- `feature_param`: Parameter to modulate based on the feature. Options are "segments", "zoom", "rotation", "precession", "speed", "None". + +This node creates a kaleidoscope effect by mirroring and rotating segments of the image. +""" +}) + +add_node_config("FlexImageColorGrade", { + "TOP_DESCRIPTION": "Applies color grading to the image using a Look-Up Table (LUT).", + "ADDITIONAL_INFO": """ +- `intensity`: Strength of the color grading effect (0.0 to 1.0). +- `mix`: Blend factor between original and graded image (0.0 to 1.0). +- `lut_file`: Path to the LUT file (optional). +- `feature_param`: Parameter to modulate based on the feature. Options are "intensity", "mix", "None". + +This node applies color grading to the image using a LUT. The intensity and mix parameters control the strength and blend of the effect. +""" +}) + +add_node_config("FlexImageGlitch", { + "TOP_DESCRIPTION": "Creates a glitch effect by applying horizontal shifts and color channel separation.", + "ADDITIONAL_INFO": """ +- `shift_amount`: Magnitude of horizontal shift (0.0 to 1.0). +- `scan_lines`: Number of scan lines to add (0 to 100). +- `color_shift`: Amount of color channel separation (0.0 to 1.0). +- `feature_param`: Parameter to modulate based on the feature. Options are "shift_amount", "scan_lines", "color_shift", "None". + +This node creates a glitch effect by shifting pixels horizontally and separating color channels. +""" +}) + +add_node_config("FlexImageChromaticAberration", { + "TOP_DESCRIPTION": "Simulates chromatic aberration by shifting color channels.", + "ADDITIONAL_INFO": """ +- `shift_amount`: Magnitude of color channel shift (0.0 to 0.1). +- `angle`: Angle of the shift effect (0.0 to 360.0 degrees). +- `feature_param`: Parameter to modulate based on the feature. Options are "shift_amount", "angle", "None". + +This node simulates chromatic aberration by shifting the red and blue color channels in opposite directions. +""" +}) + +add_node_config("FlexImagePixelate", { + "TOP_DESCRIPTION": "Applies a pixelation effect to the image.", + "ADDITIONAL_INFO": """ +- `pixel_size`: Size of each pixelated block (1 to 100 pixels). +- `feature_param`: Parameter to modulate based on the feature. Options are "pixel_size", "None". + +This node reduces the resolution of the image by applying a pixelation effect. +""" +}) + +add_node_config("FlexImageBloom", { + "TOP_DESCRIPTION": "Adds a bloom effect to bright areas of the image.", + "ADDITIONAL_INFO": """ +- `threshold`: Brightness threshold for the bloom effect (0.0 to 1.0). +- `blur_amount`: Amount of blur applied to the bloom (0.0 to 50.0). +- `intensity`: Strength of the bloom effect (0.0 to 1.0). +- `feature_param`: Parameter to modulate based on the feature. Options are "threshold", "blur_amount", "intensity", "None". + +This node adds a bloom effect to bright areas of the image, creating a glowing effect. +""" +}) + +add_node_config("FlexImageTiltShift", { + "TOP_DESCRIPTION": "Creates a tilt-shift effect, simulating a shallow depth of field.", + "ADDITIONAL_INFO": """ +- `blur_amount`: Strength of the blur effect (0.0 to 50.0). +- `focus_position_x`: X-coordinate of the focus center (0.0 to 1.0). +- `focus_position_y`: Y-coordinate of the focus center (0.0 to 1.0). +- `focus_width`: Width of the focus area (0.0 to 1.0). +- `focus_height`: Height of the focus area (0.0 to 1.0). +- `focus_shape`: Shape of the focus area ("rectangle" or "ellipse"). +- `feature_param`: Parameter to modulate based on the feature. Options are "blur_amount", "focus_position_x", "focus_position_y", "focus_width", "focus_height", "None". + +This node creates a tilt-shift effect, simulating a shallow depth of field by blurring areas outside the focus region. +""" +}) + +add_node_config("FlexImageParallax", { + "TOP_DESCRIPTION": "Applies a parallax effect to the image using a depth map.", + "ADDITIONAL_INFO": """ +- `shift_x`: Horizontal shift factor for the parallax effect (-1.0 to 1.0). Positive values shift right, negative values shift left. +- `shift_y`: Vertical shift factor for the parallax effect (-1.0 to 1.0). Positive values shift up, negative values shift down. +- `depth_map`: Input depth map (IMAGE type). The depth map is used to determine the amount of shift for each pixel. +- `feature_param`: Parameter to modulate based on the feature. Options are "shift_x", "shift_y", "None". + +This node creates a parallax effect by shifting pixels in the image based on the corresponding values in the depth map. The shift factors `shift_x` and `shift_y` control the direction and magnitude of the parallax effect. The depth map should be provided as an image, where the intensity of each pixel represents the depth value. +""" +}) + +add_node_config("UtilityNode",{ + "BASE_DESCRIPTION":"Various Utils" +}) + +add_node_config("ImageIntervalSelect", { + "TOP_DESCRIPTION": "Selects images from a sequence at specified intervals.", + "ADDITIONAL_INFO": """ +- `image`: Input image sequence (IMAGE type) +- `interval`: Interval at which to select images (1 to 100000) +- `start_at`: Starting index for selection (0 to 100000) +- `end_at`: Ending index for selection (0 to 100000) +""" +}) + +add_node_config("ImageIntervalSelectPercentage", { + "TOP_DESCRIPTION": "Selects images from a sequence at specified percentage intervals.", + "ADDITIONAL_INFO": """ +- `image`: Input image sequence (IMAGE type) +- `interval_percentage`: Interval at which to select images as a percentage of the total sequence length (1 to 100) +- `start_percentage`: Starting percentage for selection (0 to 100) +- `end_percentage`: Ending percentage for selection (0 to 100) +""" +}) + +add_node_config("ImageChunks", { + "TOP_DESCRIPTION": "Concatenates images into a grid.", + "ADDITIONAL_INFO": """ +- `image`: Input image sequence (IMAGE type) +- `padding`: Padding between images in the grid (default: 0) +- `normalize`: Whether to normalize the images (default: False) +- `scale_each`: Whether to scale each image individually (default: False) +- `pad_value`: Value for padding (default: 0) +""" +}) + +add_node_config("VideoChunks", { + "TOP_DESCRIPTION": "Chunks images into grids.", + "ADDITIONAL_INFO": """ +- `image`: Input image sequence (IMAGE type) +- `chunk_size`: Number of images per grid (default: 4, min: 1) +- `padding`: Padding between images in the grid (default: 2) +- `normalize`: Whether to normalize the images (default: False) +- `scale_each`: Whether to scale each image individually (default: False) +- `pad_value`: Value for padding (default: 0) +""" +}) + +add_node_config("SwapDevice", { + "TOP_DESCRIPTION": "Transfers the image and mask tensors to the specified device (CPU or CUDA).", + "ADDITIONAL_INFO": """ +- `device`: The target device to transfer the tensors to. Options are "cpu" or "cuda". +- `image`: (Optional) The image tensor to transfer. If not provided, a zero tensor is created. +- `mask`: (Optional) The mask tensor to transfer. If not provided, a zero tensor is created. + +This node checks if the specified device is available and transfers the image and mask tensors to that device. If the device is not available, it raises a ValueError. +""" +}) + +add_node_config("ImageShuffle", { + "TOP_DESCRIPTION": "Shuffles images in groups of specified size.", + "ADDITIONAL_INFO": """ +- `image`: Input image sequence (IMAGE type) +- `shuffle_size`: Number of images per shuffle group (default: 4, min: 1) +""" +}) + +add_node_config("ImageDifference", { + "TOP_DESCRIPTION": "Computes the difference between consecutive images.", + "ADDITIONAL_INFO": """ +- `image`: Input image sequence (IMAGE type) +""" +}) + +add_node_config("EffectVisualizer", { + "TOP_DESCRIPTION": "Visualizes feature values on video frames.", + "ADDITIONAL_INFO": """ +- `video_frames`: Input video frames (IMAGE type) +- `feature`: Feature to visualize (FEATURE type) +- `text_color`: Color for text overlay (RGB string, e.g., "(255,255,255)") +- `font_scale`: Scale factor for the font size (0.1 to 2.0) +""" +}) + +add_node_config("ProximityVisualizer", { + "TOP_DESCRIPTION": "Visualizes proximity relationships between anchor and query locations on video frames.", + "ADDITIONAL_INFO": """ +- `anchor_locations`: Locations of anchor points (LOCATION type) +- `query_locations`: Locations of query points (LOCATION type) +- `feature`: Proximity feature to visualize (FEATURE type) +- `anchor_color`: Color for anchor points (RGB string, e.g., "(255,0,0)") +- `query_color`: Color for query points (RGB string, e.g., "(0,255,0)") +- `line_color`: Color for the line connecting closest points (RGB string, e.g., "(0,0,255)") + + +The visualization helps in understanding spatial relationships and proximity-based effects in the video sequence. +""" +}) + +add_node_config("PitchVisualizer", { + "TOP_DESCRIPTION": "Visualizes pitch-related information on video frames.", + "ADDITIONAL_INFO": """ +- `video_frames`: Input video frames (IMAGE type) +- `feature`: Pitch feature to visualize (FEATURE type) +- `text_color`: Color of the text overlay (RGB string, e.g., "(255,255,255)") +- `font_scale`: Scale factor for the font size (0.1 to 2.0) + +This node overlays pitch-related information on video frames, including: +- Feature value +- Pitch value in Hz +- Confidence value of the pitch detection +- Approximate musical note + +The information is displayed as text on each frame, allowing for easy visualization of pitch characteristics alongside the video content. +""" +}) \ No newline at end of file diff --git a/nodes/audio/audio_nodes.py b/nodes/audio/audio_nodes.py index e5204ca..c7c618f 100644 --- a/nodes/audio/audio_nodes.py +++ b/nodes/audio/audio_nodes.py @@ -14,6 +14,11 @@ from ... import RyanOnTheInside class AudioNodeBase(RyanOnTheInside): CATEGORY= "RyanOnTheInside/Audio" + @staticmethod + def create_empty_tensor(audio, frame_rate, height, width, channels): + audio_duration = audio['waveform'].shape[-1] / audio['sample_rate'] + num_frames = int(audio_duration * frame_rate) + return torch.zeros((num_frames, height, width, channels), dtype=torch.float32) class DownloadOpenUnmixModel(AudioNodeBase): @classmethod @@ -167,6 +172,44 @@ class AudioFilter(AudioNodeBase): return signal.lfilter(b, a, audio) +# class DownloadCREPEModel(AudioNodeBase): +# @classmethod +# def INPUT_TYPES(cls): +# return { +# "required": { +# "model_name": (["tiny", "small", "medium", "large", "full"], {"default": "medium"}), +# } +# } + +# RETURN_TYPES = ("CREPE_MODEL",) +# FUNCTION = "download_and_load_model" +# CATEGORY = "RyanOnTheInside/Audio" + +# def download_and_load_model(self, model_name): +# try: +# import crepe +# except ImportError: +# raise ImportError("""To use this node please +# pip install crepe tensorflow +# :) +# """) + +# download_path = os.path.join(folder_paths.models_dir, "crepe") +# os.makedirs(download_path, exist_ok=True) + +# model_file = f"crepe_{model_name}.json" +# model_path = os.path.join(download_path, model_file) + +# if not os.path.exists(model_path): +# print(f"Downloading CREPE {model_name} model...") +# model = crepe.core.build_and_load_model(model_name) +# print(f"Model downloaded and loaded.") +# else: +# print(f"Loading model from: {model_path}") +# model = crepe.core.build_and_load_model(model_name) + +# return (model,) + class FrequencyFilterPreset(AudioNodeBase): @classmethod def INPUT_TYPES(cls): @@ -318,4 +361,42 @@ class AudioFeatureVisualizer(AudioNodeBase): raise ValueError(f"Unsupported visualization type: {visualization_type}") - return (mask,) \ No newline at end of file + return (mask,) + +class EmptyImageFromAudio(AudioNodeBase): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "audio": ("AUDIO",), + "frame_rate": ("FLOAT", {"default": 30, "min": 0.1, "max": 120, "step": 0.1}), + "height": ("INT", {"default": 512, "min": 16, "max": 4096, "step": 1}), + "width": ("INT", {"default": 512, "min": 16, "max": 4096, "step": 1}), + } + } + + RETURN_TYPES = ("IMAGE",) + FUNCTION = "create_empty_image" + + def create_empty_image(self, audio, frame_rate, height, width): + empty_image = self.create_empty_tensor(audio, frame_rate, height, width, channels=3) + return (empty_image,) + +class EmptyMaskFromAudio(AudioNodeBase): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "audio": ("AUDIO",), + "frame_rate": ("FLOAT", {"default": 30, "min": 0.1, "max": 120, "step": 0.1}), + "height": ("INT", {"default": 512, "min": 16, "max": 4096, "step": 1}), + "width": ("INT", {"default": 512, "min": 16, "max": 4096, "step": 1}), + } + } + + RETURN_TYPES = ("MASK",) + FUNCTION = "create_empty_mask" + + def create_empty_mask(self, audio, frame_rate, height, width): + empty_mask = self.create_empty_tensor(audio, frame_rate, height, width, channels=1) + return (empty_mask,) \ No newline at end of file diff --git a/nodes/flex/feature_externals.py b/nodes/flex/feature_externals.py index c9b9cd3..f667653 100644 --- a/nodes/flex/feature_externals.py +++ b/nodes/flex/feature_externals.py @@ -2,7 +2,7 @@ import torch import torch.nn.functional as F import numpy as np from ... import RyanOnTheInside -from ... import RyanOnTheInside +import cv2 class FlexExternalModulator(RyanOnTheInside): CATEGORY = "RyanOnTheInside/FlexExternalMod" @@ -42,11 +42,6 @@ class FeatureToWeightsStrategy(FlexExternalModulator): return (weights_strategy,) - -import torch -import torch.nn.functional as F -import cv2 - class DepthShapeModifier(FlexExternalModulator): @classmethod def INPUT_TYPES(cls): diff --git a/nodes/flex/feature_extractors.py b/nodes/flex/feature_extractors.py index fa46237..fd7d08d 100644 --- a/nodes/flex/feature_extractors.py +++ b/nodes/flex/feature_extractors.py @@ -1,11 +1,26 @@ from .feature_pipe import FeaturePipe from ... import RyanOnTheInside -from .features import AudioFeature, TimeFeature, DepthFeature, ColorFeature, BrightnessFeature, MotionFeature, AreaFeature -from .proximity_feature import Location +from .features import TimeFeature, DepthFeature, ColorFeature, BrightnessFeature, MotionFeature, AreaFeature, BaseFeature +from abc import ABC, abstractmethod from tqdm import tqdm from comfy.utils import ProgressBar +import typing + +class FeatureExtractorBase(RyanOnTheInside, ABC): + @classmethod + @abstractmethod + def feature_type(cls) -> type[BaseFeature]: + pass + + @classmethod + def INPUT_TYPES(cls): + feature_class = cls.feature_type() + return { + "required": { + "extraction_method": (feature_class.get_extraction_methods(), {"default": feature_class.get_extraction_methods()[0]}), + } + } -class FeatureExtractorBase(RyanOnTheInside): def __init__(self): self.progress_bar = None self.tqdm_bar = None @@ -34,31 +49,19 @@ class FeatureExtractorBase(RyanOnTheInside): self.total_steps = 0 CATEGORY="RyanOnTheInside/FlexFeatures" -class AudioFeatureExtractor(FeatureExtractorBase): - @classmethod - def INPUT_TYPES(cls): - return { - "required": { - "audio": ("AUDIO",), - "feature_pipe": ("FEATURE_PIPE",), - "feature_type": (["amplitude_envelope", "spectral_centroid", "onset_detection", "chroma_features"],), - } - } - - RETURN_TYPES = ("FEATURE", "FEATURE_PIPE") - FUNCTION = "extract_feature" - - - def extract_feature(self, audio, feature_pipe, feature_type): - feature = AudioFeature(feature_type, audio, feature_pipe.frame_count, feature_pipe.frame_rate, feature_type) - feature.extract() - return (feature, feature_pipe) - +# class FirstFeature(FeatureExtractorBase): + + @classmethod + def feature_type(cls) -> type[BaseFeature]: + pass + @classmethod def INPUT_TYPES(cls): - return { + return { + **super().INPUT_TYPES(), "required": { + **super().INPUT_TYPES()["required"], "video_frames": ("IMAGE",), "frame_rate": ("FLOAT", {"default": 30.0, "min": 1.0, "max": 120.0, "step": 0.1}), } @@ -68,13 +71,16 @@ class FirstFeature(FeatureExtractorBase): FUNCTION = "create_feature" class TimeFeatureNode(FirstFeature): + @classmethod + def feature_type(cls) -> type[BaseFeature]: + return TimeFeature + @classmethod def INPUT_TYPES(cls): return { **super().INPUT_TYPES(), "required": { **super().INPUT_TYPES()["required"], - "effect_type": (["smooth", "accelerate", "pulse", "sawtooth", "bounce"],), "speed": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 10.0, "step": 0.1}), "offset": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), } @@ -82,14 +88,18 @@ class TimeFeatureNode(FirstFeature): - def create_feature(self, effect_type, speed, offset, video_frames, frame_rate): + def create_feature(self, extraction_method, speed, offset, video_frames, frame_rate): feature_pipe = FeaturePipe(frame_rate, video_frames) time_feature = TimeFeature("time_effect", feature_pipe.frame_rate, feature_pipe.frame_count, - effect_type=effect_type, speed=speed, offset=offset) + effect_type=extraction_method, speed=speed, offset=offset) time_feature.extract() return (time_feature, feature_pipe) class DepthFeatureNode(FirstFeature): + @classmethod + def feature_type(cls) -> type[BaseFeature]: + return DepthFeature + @classmethod def INPUT_TYPES(cls): return { @@ -97,58 +107,66 @@ class DepthFeatureNode(FirstFeature): "required": { **super().INPUT_TYPES()["required"], "depth_maps": ("IMAGE",), - "feature_type": (["mean_depth", "depth_variance", "depth_range", "gradient_magnitude", "foreground_ratio", "midground_ratio", "background_ratio"],), } } - def create_feature(self, depth_maps, frame_rate, video_frames, feature_type): + def create_feature(self, depth_maps, frame_rate, video_frames, extraction_method): feature_pipe = FeaturePipe(frame_rate, video_frames) - depth_feature = DepthFeature("depth_feature", feature_pipe.frame_rate, feature_pipe.frame_count, depth_maps, feature_type) + depth_feature = DepthFeature("depth_feature", feature_pipe.frame_rate, feature_pipe.frame_count, depth_maps, extraction_method) depth_feature.extract() return (depth_feature, feature_pipe) class ColorFeatureNode(FirstFeature): + @classmethod + def feature_type(cls) -> type[BaseFeature]: + return ColorFeature + @classmethod def INPUT_TYPES(cls): return { **super().INPUT_TYPES(), "required": { **super().INPUT_TYPES()["required"], - "feature_type": (["dominant_color", "color_variance", "saturation", "red_ratio", "green_ratio", "blue_ratio"],), } } - def create_feature(self, video_frames, frame_rate, feature_type): + def create_feature(self, video_frames, frame_rate, extraction_method): feature_pipe = FeaturePipe(frame_rate, video_frames) - color_feature = ColorFeature("color_feature", feature_pipe.frame_rate, feature_pipe.frame_count, video_frames, feature_type) + color_feature = ColorFeature("color_feature", feature_pipe.frame_rate, feature_pipe.frame_count, video_frames, extraction_method) color_feature.extract() return (color_feature, feature_pipe) class BrightnessFeatureNode(FirstFeature): @classmethod + def feature_type(cls) -> type[BaseFeature]: + return BrightnessFeature + + @classmethod def INPUT_TYPES(cls): return { **super().INPUT_TYPES(), "required": { **super().INPUT_TYPES()["required"], - "feature_type": (["mean_brightness", "brightness_variance", "dark_ratio", "mid_ratio", "bright_ratio"],), } } + + def create_feature(self, video_frames, frame_rate, extraction_method): + feature_pipe = FeaturePipe(frame_rate, video_frames) + brightness_feature = BrightnessFeature("brightness_feature", feature_pipe.frame_rate, feature_pipe.frame_count, video_frames, extraction_method) + brightness_feature.extract() + return (brightness_feature, feature_pipe) + +class MotionFeatureNode(FirstFeature): + @classmethod + def feature_type(cls) -> type[BaseFeature]: + return MotionFeature - def create_feature(self, video_frames, frame_rate, feature_type): - feature_pipe = FeaturePipe(frame_rate, video_frames) - brightness_feature = BrightnessFeature("brightness_feature", feature_pipe.frame_rate, feature_pipe.frame_count, video_frames, feature_type) - brightness_feature.extract() - return (brightness_feature, feature_pipe) - -class MotionFeatureNode(FirstFeature): @classmethod def INPUT_TYPES(cls): return { **super().INPUT_TYPES(), "required": { **super().INPUT_TYPES()["required"], - "feature_type": (["mean_motion", "max_motion", "motion_direction", "horizontal_motion", "vertical_motion", "motion_complexity","motion_speed"],), "flow_method": (["Farneback", "LucasKanade", "PyramidalLK"],), "flow_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 10.0, "step": 0.1}), "magnitude_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), @@ -158,7 +176,7 @@ class MotionFeatureNode(FirstFeature): RETURN_TYPES = ("FEATURE", "FEATURE_PIPE") FUNCTION = "create_feature" - def create_feature(self, video_frames, frame_rate, feature_type, flow_method, flow_threshold, magnitude_threshold): + def create_feature(self, video_frames, frame_rate, extraction_method, flow_method, flow_threshold, magnitude_threshold): feature_pipe = FeaturePipe(frame_rate, video_frames) num_frames = feature_pipe.frame_count @@ -172,7 +190,7 @@ class MotionFeatureNode(FirstFeature): feature_pipe.frame_rate, feature_pipe.frame_count, video_frames, - feature_type, + extraction_method, flow_method, flow_threshold, magnitude_threshold, @@ -186,6 +204,10 @@ class MotionFeatureNode(FirstFeature): class AreaFeatureNode(FirstFeature): + @classmethod + def feature_type(cls) -> type[BaseFeature]: + return AreaFeature + @classmethod def INPUT_TYPES(cls): return { @@ -193,7 +215,6 @@ class AreaFeatureNode(FirstFeature): "required": { **super().INPUT_TYPES()["required"], "masks": ("MASK",), - "feature_type": (["total_area", "largest_contour", "bounding_box"],), "threshold": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), } } @@ -201,9 +222,9 @@ class AreaFeatureNode(FirstFeature): RETURN_TYPES = ("FEATURE", "FEATURE_PIPE") FUNCTION = "create_feature" - def create_feature(self, masks, video_frames, frame_rate, feature_type, threshold): + def create_feature(self, masks, video_frames, frame_rate, extraction_method, threshold): feature_pipe = FeaturePipe(frame_rate, video_frames) area_feature = AreaFeature("area_feature", feature_pipe.frame_rate, feature_pipe.frame_count, - masks, feature_type=feature_type, threshold=threshold) + masks, feature_type=extraction_method, threshold=threshold) area_feature.extract() return (area_feature, feature_pipe) \ No newline at end of file diff --git a/nodes/flex/feature_extractors_audio.py b/nodes/flex/feature_extractors_audio.py new file mode 100644 index 0000000..82c57ac --- /dev/null +++ b/nodes/flex/feature_extractors_audio.py @@ -0,0 +1,208 @@ +from .feature_extractors import FeatureExtractorBase +from .features_audio import AudioFeature, PitchFeature, PitchRange, BaseFeature +from ... import RyanOnTheInside + +class AudioFeatureExtractor(FeatureExtractorBase): + @classmethod + def feature_type(cls) -> type[BaseFeature]: + return AudioFeature + + @classmethod + def INPUT_TYPES(cls): + return { + **super().INPUT_TYPES(), + "required": { + **super().INPUT_TYPES()["required"], + "audio": ("AUDIO",), + "feature_pipe": ("FEATURE_PIPE",), + } + } + + RETURN_TYPES = ("FEATURE", "FEATURE_PIPE") + FUNCTION = "extract_feature" + + + def extract_feature(self, audio, feature_pipe, extraction_method): + feature = AudioFeature(extraction_method, audio, feature_pipe.frame_count, feature_pipe.frame_rate, extraction_method) + feature.extract() + return (feature, feature_pipe) + +class PitchFeatureExtractor(FeatureExtractorBase): + @classmethod + def feature_type(cls) -> type[BaseFeature]: + return PitchFeature + + @classmethod + def INPUT_TYPES(cls): + return { + **super().INPUT_TYPES(), + "required": { + **super().INPUT_TYPES()["required"], + "audio": ("AUDIO",), + "feature_pipe": ("FEATURE_PIPE",), + "opt_crepe_model":(["none", "medium", "tiny", "small", "large", "full"], {"default": "medium"}) + }, + "optional": { + "opt_pitch_range_collections": ("PITCH_RANGE_COLLECTION",), + # "": ("CREPE_MODEL",), + }, + } + + RETURN_TYPES = ("FEATURE", "FEATURE_PIPE") + FUNCTION = "extract_feature" + + def extract_feature(self, audio, feature_pipe, extraction_method, opt_pitch_range_collections=None, opt_crepe_model=None): + if opt_pitch_range_collections is None: + opt_pitch_range_collections = [] + feature = PitchFeature( + feature_name=extraction_method, + audio=audio, + frame_count=feature_pipe.frame_count, + frame_rate=feature_pipe.frame_rate, + pitch_range_collections=opt_pitch_range_collections, + feature_type=extraction_method, + crepe_model=opt_crepe_model + ) + feature.extract() + return (feature, feature_pipe) + +class PitchAbstraction(RyanOnTheInside): + CATEGORY="RyanOnTheInside/FlexFeatures/Audio" + +class PitchRangeNode(PitchAbstraction): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "min_pitch": ("FLOAT", {"default": 80.0, "min": 20.0, "max": 2000.0, "step": 1.0}), + "max_pitch": ("FLOAT", {"default": 400.0, "min": 20.0, "max": 2000.0, "step": 1.0}), + }, + "optional": { + "previous_range_collection": ("PITCH_RANGE_COLLECTION",), + }, + } + + RETURN_TYPES = ("PITCH_RANGE_COLLECTION",) + FUNCTION = "create_pitch_range" + + CATEGORY = "RyanOnTheInside/FlexFeatures" + + def create_pitch_range(self, min_pitch, max_pitch, previous_range_collection=None): + pitch_range = PitchRange(min_pitch, max_pitch) + pitch_range_collection = { + "pitch_ranges": [pitch_range], + "chord_only": False, + } + if previous_range_collection is None: + collections = [pitch_range_collection] + else: + collections = previous_range_collection + [pitch_range_collection] + return (collections,) + +class PitchRangePresetNode(PitchAbstraction): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "preset": ( + [ + "Bass", + "Baritone", + "Tenor", + "Alto", + "Mezzo-soprano", + "Soprano", + "Contralto", + ], + ) + }, + "optional": { + "previous_range_collection": ("PITCH_RANGE_COLLECTION",), + }, + } + + RETURN_TYPES = ("PITCH_RANGE_COLLECTION",) + FUNCTION = "create_pitch_range_preset" + + CATEGORY = "RyanOnTheInside/FlexFeatures" + + def create_pitch_range_preset(self, preset, previous_range_collection=None): + presets = { + "Bass": (82.41, 196.00), # E2 - G3 + "Baritone": (98.00, 247.94), # G2 - B3 + "Tenor": (130.81, 349.23), # C3 - F4 + "Contralto": (130.81, 349.23), # C3 - F4 + "Alto": (174.61, 440.00), # F3 - A4 + "Mezzo-soprano": (196.00, 523.25), # G3 - C5 + "Soprano": (261.63, 1046.50), # C4 - C6 + } + + min_pitch, max_pitch = presets.get(preset, (20.0, 2000.0)) + pitch_range = PitchRange(min_pitch, max_pitch) + pitch_range_collection = { + "pitch_ranges": [pitch_range], + "chord_only": False, + } + if previous_range_collection is None: + collections = [pitch_range_collection] + else: + collections = previous_range_collection + [pitch_range_collection] + return (collections,) + +class PitchRangeByNoteNode(PitchAbstraction): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "chord_only": ("BOOLEAN", {"default": False}), + "pitch_tolerance_percent": ("FLOAT", {"default": 100.0, "min": 0.0, "max": 100.0, "step": 0.1}), + "notes": ("STRING", {"multiline": False}), + }, + "optional": { + "previous_range_collection": ("PITCH_RANGE_COLLECTION",), + }, + } + + RETURN_TYPES = ("PITCH_RANGE_COLLECTION",) + FUNCTION = "create_note_pitch_ranges" + + CATEGORY = "RyanOnTheInside/FlexFeatures" + + def create_note_pitch_ranges(self, chord_only, notes, pitch_tolerance_percent, previous_range_collection=None): + if not notes: + raise ValueError("At least one note must be selected.") + + # Parse the 'notes' string into a list of MIDI note numbers + selected_notes = [int(note.strip()) for note in notes.split(',') if note.strip().isdigit()] + + if not selected_notes: + raise ValueError("No valid notes found in the 'notes' field.") + + pitch_ranges = [] + for midi_note in selected_notes: + frequency = self._midi_to_frequency(midi_note) + tolerance = PitchFeature.calculate_tolerance(frequency, pitch_tolerance_percent) + min_pitch = frequency - tolerance + max_pitch = frequency + tolerance + pitch_range = PitchRange(min_pitch, max_pitch) + pitch_ranges.append(pitch_range) + + # Create a collection with the 'chord_only' attribute + pitch_range_collection = { + "pitch_ranges": pitch_ranges, + "chord_only": chord_only, + } + + # Combine with previous collections if provided + if previous_range_collection is None: + collections = [pitch_range_collection] + else: + collections = previous_range_collection + [pitch_range_collection] + + return (collections,) + + def _midi_to_frequency(self, midi_note): + import librosa + return librosa.midi_to_hz(midi_note) + + # The _calculate_tolerance method has been removed from here \ No newline at end of file diff --git a/nodes/flex/feature_extractors_midi.py b/nodes/flex/feature_extractors_midi.py new file mode 100644 index 0000000..b065939 --- /dev/null +++ b/nodes/flex/feature_extractors_midi.py @@ -0,0 +1,163 @@ +import mido +import os +from .feature_pipe import FeaturePipe +import folder_paths +from server import PromptServer +from aiohttp import web +import shutil +from .midi_feature import MIDIFeature +from .feature_extractors import FeatureExtractorBase + +class MIDILoadAndExtract(FeatureExtractorBase): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "midi_file": (folder_paths.get_filename_list("midi_files"),), + "track_selection": (["all"],), + "attribute": (MIDIFeature.get_attribute_names(), {"default": "Note On/Off"}), + "frame_rate": ("FLOAT", {"default": 30, "min": 0.1, "max": 120, "step": 0.1}), + "video_frames": ("IMAGE",), + "chord_only": ("BOOLEAN", {"default": False}), + "notes": ("STRING", {"default": ""}), + }, + } + + RETURN_TYPES = ("MIDI", "FEATURE", "FEATURE_PIPE") + FUNCTION = "process_midi" + CATEGORY = "RyanOnTheInside/Audio" + + def process_midi(self, midi_file, track_selection, notes, attribute, frame_rate, video_frames, chord_only=False): + try: + midi_path = folder_paths.get_full_path("midi_files", midi_file) + if not midi_path or not os.path.exists(midi_path): + raise FileNotFoundError(f"MIDI file not found: {midi_file}") + + midi_data = mido.MidiFile(midi_path) + + selected_notes = [int(n.strip()) for n in notes.split(',') if n.strip().isdigit()] + feature_pipe = FeaturePipe(frame_rate, video_frames) + + # Convert friendly attribute name to internal attribute name + internal_attribute = MIDIFeature.get_attribute_value(attribute) + + feature = MIDIFeature(f"midi_{internal_attribute}", midi_data, internal_attribute, + feature_pipe.frame_rate, feature_pipe.frame_count, + notes=selected_notes, chord_only=chord_only) + + feature.extract() + + return (midi_data, feature, feature_pipe) + + except Exception as e: + # error_msg = f"Error in MIDILoadAndExtract.process_midi: {type(e).__name__}: {str(e)}\n" + # error_msg += traceback.format_exc() + # print(error_msg) + raise RuntimeError(f"Error processing MIDI file: {type(e).__name__}: {str(e)}") + + @classmethod + def analyze_midi(cls, midi_path): + midi_data = mido.MidiFile(midi_path) + tracks = ["all"] + all_notes = set() + track_notes = {} + for i, track in enumerate(midi_data.tracks): + track_notes[str(i)] = set() + for msg in track: + if msg.type == 'note_on': + track_notes[str(i)].add(msg.note) + all_notes.add(msg.note) + if len(track_notes[str(i)]) == 0: + tracks.append(f"{i}: (Empty)") + else: + tracks.append(f"{i}: {track.name or f'Track {i}'}") + + return { + "tracks": tracks, + "all_notes": ",".join(map(str, sorted(set(all_notes)))), + "track_notes": {k: ",".join(map(str, sorted(v))) for k, v in track_notes.items()} + } + + @classmethod + def VALIDATE_INPUTS(cls, midi_file, track_selection, notes, attribute, frame_rate, video_frames): + if not folder_paths.exists_and_is_file(midi_file): + return "MIDI file not found: {}".format(midi_file) + + if notes != "all": + try: + note_list = [int(n.strip()) for n in notes.split(',') if n.strip()] + if not all(0 <= n <= 127 for n in note_list): + return "Invalid note value. All notes must be between 0 and 127." + except ValueError: + return "Invalid notes format. Please provide comma-separated integers or 'all'." + + return True + + +routes = PromptServer.instance.routes +@PromptServer.instance.routes.post('/get_track_notes') +async def get_track_notes(request): + data = await request.json() + midi_file = data.get('midi_file') + + if not midi_file: + return web.json_response({"error": "Missing required parameters"}, status=400) + + midi_path = folder_paths.get_full_path("midi_files", midi_file) + if not midi_path or not os.path.exists(midi_path): + return web.json_response({"error": "MIDI file not found"}, status=404) + + analysis = MIDILoadAndExtract.analyze_midi(midi_path) + return web.json_response(analysis) + +@routes.post('/upload_midi') +async def upload_midi(request): + data = await request.post() + midi_file = data['file'] + + if midi_file and midi_file.filename: + safe_filename = os.path.basename(midi_file.filename) + + midi_dir = folder_paths.get_folder_paths("midi_files")[0] + + if not os.path.exists(midi_dir): + os.makedirs(midi_dir, exist_ok=True) + + midi_path = os.path.join(midi_dir, safe_filename) + + with open(midi_path, 'wb') as f: + shutil.copyfileobj(midi_file.file, f) + + midi_files = folder_paths.get_filename_list("midi_files") + analysis = MIDILoadAndExtract.analyze_midi(midi_path) + + return web.json_response({ + "status": "success", + "uploaded_file": safe_filename, + "midi_files": midi_files, + "analysis": analysis + }) + else: + return web.json_response({"status": "error", "message": "No file uploaded"}, status=400) + +@PromptServer.instance.routes.post('/refresh_midi_data') +async def refresh_midi_data(request): + data = await request.json() + midi_file = data.get('midi_file') + track_selection = data.get('track_selection') + + if not midi_file: + return web.json_response({"error": "Missing required parameters"}, status=400) + + midi_path = folder_paths.get_full_path("midi_files", midi_file) + if not midi_path or not os.path.exists(midi_path): + return web.json_response({"error": "MIDI file not found"}, status=404) + + analysis = MIDILoadAndExtract.analyze_midi(midi_path) + + # Filter notes based on track selection + if track_selection != "all": + track_index = track_selection.split(':')[0] + analysis['all_notes'] = analysis['track_notes'].get(track_index, "") + + return web.json_response(analysis) \ No newline at end of file diff --git a/nodes/flex/feature_extractors_proximity.py b/nodes/flex/feature_extractors_proximity.py new file mode 100644 index 0000000..5f048ea --- /dev/null +++ b/nodes/flex/feature_extractors_proximity.py @@ -0,0 +1,186 @@ +from .features_proximity import Location, ProximityFeature +from .feature_extractors import FirstFeature +from ... import RyanOnTheInside +from .feature_pipe import FeaturePipe +import numpy as np +import cv2 + + +class ProximityFeatureNode(FirstFeature): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "video_frames": ("IMAGE",), + "frame_rate": ("FLOAT", {"default": 30.0, "min": 1.0, "max": 120.0, "step": 0.1}), + "anchor_locations": ("LOCATION",), + "query_locations": ("LOCATION",), + "normalization_method": (["frame", "minmax"],), + } + } + + RETURN_TYPES = ("FEATURE", "FEATURE_PIPE") + RETURN_NAMES = ("proximity_feature", "feature_pipe") + FUNCTION = "create_feature" + + def create_feature(self, video_frames, frame_rate, anchor_locations, query_locations, normalization_method): + feature_pipe = FeaturePipe(frame_rate, video_frames) + frame_dimensions = (video_frames.shape[2], video_frames.shape[1]) # width, height + + proximity_feature = ProximityFeature( + name="proximity_feature", + anchor_locations=anchor_locations, + query_locations=query_locations, + frame_rate=frame_rate, + frame_count=len(video_frames), + frame_dimensions=frame_dimensions, + normalization_method=normalization_method + ) + + # Ensure anchor and query points have the same number of columns + anchor_dim = anchor_locations[0].points.shape[1] + query_dim = query_locations[0].points.shape[1] + + if anchor_dim != query_dim: + if anchor_dim == 3: + for location in anchor_locations: + location.points = location.points[:, :2] + elif query_dim == 3: + for location in query_locations: + location.points = location.points[:, :2] + print("Depth ignored. Depth must be included for all points or no points.") + + # self.start_progress(len(video_frames), desc="Calculating proximity") + proximity_feature.extract() + # self.end_progress() + + return (proximity_feature, feature_pipe) + +class ProximityFeatureInput(RyanOnTheInside): + CATEGORY="RyanOnTheInside/Proximity" + +class LocationFromMask(ProximityFeatureInput): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "masks": ("MASK",), + "method": (["mask_boundary", "mask_center"],), + }, + "optional": { + "depth_maps": ("IMAGE",), + } + } + + RETURN_TYPES = ("LOCATION",) + FUNCTION = "generate_locations" + + def generate_locations(self, masks, method, depth_maps=None): + locations = [] + for i in range(masks.shape[0]): + mask = masks[i].cpu().numpy() + if np.sum(mask) == 0: # Check if the mask is empty + print(f"No mask found in frame {i}.") + locations.append(Location(np.array([-1]), np.array([-1]), np.array([-1]) if depth_maps is not None else None)) + continue + + if method == "mask_boundary": + contours, _ = cv2.findContours(mask.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + if contours: + boundary = np.vstack(contours).squeeze() + else: + boundary = np.array([[0, 0]]) # Fallback to a single point + elif method == "mask_center": + y, x = np.where(mask > 0.5) + if len(x) > 0 and len(y) > 0: + center_x, center_y = np.mean(x), np.mean(y) + boundary = np.array([[center_x, center_y]]) + else: + boundary = np.array([[mask.shape[1] / 2, mask.shape[0] / 2]]) + + if depth_maps is not None: + depth_map = depth_maps[i].cpu().numpy() + # Ensure boundary points are within the depth map dimensions + valid_points = (boundary[:, 0] < depth_map.shape[1]) & (boundary[:, 1] < depth_map.shape[0]) + boundary = boundary[valid_points] + + if boundary.size == 0: # Check if boundary is empty after filtering + print(f"No valid boundary points in frame {i}.") + locations.append(Location(np.array([-1]), np.array([-1]), np.array([-1]))) + continue + + # Extract depth values for valid boundary points + z = np.mean(depth_map[boundary[:, 1].astype(int), boundary[:, 0].astype(int)], axis=-1) + + location = Location(boundary[:, 0], boundary[:, 1], z) + else: + location = Location(boundary[:, 0], boundary[:, 1]) + + locations.append(location) + + return (locations,) + +class LocationFromPoint(ProximityFeatureInput): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "x": ("FLOAT", {"default": 0.0,"min":0.0,"step":0.01}), + "y": ("FLOAT", {"default": 0.0,"min":0.0,"step":0.01}), + "z": ("FLOAT", {"default": 0.0, "max":1.0, "min":0.0,"step":0.01}), + "batch_count": ("INT", {"default": 1, "min": 1}), + } + } + + RETURN_TYPES = ("LOCATION",) + FUNCTION = "generate_locations" + + def generate_locations(self, x, y, batch_count, z): + locations = [] + for _ in range(batch_count): + location = Location(np.array([x]), np.array([y]), np.array([z])) + locations.append(location) + + return (locations,) + +class LocationTransform(ProximityFeatureInput): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "locations": ("LOCATION",), + "feature": ("FEATURE",), + "transformation_type": (["translate", "scale"],), + "transformation_value": ("FLOAT", {"default": 1.0}), + } + } + + RETURN_TYPES = ("LOCATION",) + FUNCTION = "transform_locations" + + def transform_locations(self, locations, feature, transformation_type, transformation_value): + transformed_locations = [] + for frame_index, location in enumerate(locations): + feature_value = feature.get_value_at_frame(frame_index) + if transformation_type == "translate": + new_x = location.x + feature_value * transformation_value + new_y = location.y + feature_value * transformation_value + if location.z is not None: + new_z = location.z + feature_value * transformation_value + transformed_location = Location(new_x, new_y, new_z) + else: + transformed_location = Location(new_x, new_y) + elif transformation_type == "scale": + new_x = location.x * feature_value * transformation_value + new_y = location.y * feature_value * transformation_value + if location.z is not None: + new_z = location.z * feature_value * transformation_value + transformed_location = Location(new_x, new_y, new_z) + else: + transformed_location = Location(new_x, new_y) + transformed_locations.append(transformed_location) + + return (transformed_locations,) + + + diff --git a/nodes/flex/feature_modulation.py b/nodes/flex/feature_modulation.py index ab55b78..038757a 100644 --- a/nodes/flex/feature_modulation.py +++ b/nodes/flex/feature_modulation.py @@ -4,6 +4,7 @@ import matplotlib.pyplot as plt import torch from io import BytesIO from PIL import Image +import random class FeatureModulationBase(RyanOnTheInside): CATEGORY = "RyanOnTheInside/FlexFeatures/FeatureModulators" @@ -477,4 +478,51 @@ class PreviewFeature(FeatureModulationBase): else: processed_feature = feature - return (self.visualize(processed_feature),) \ No newline at end of file + return (self.visualize(processed_feature),) + +class FeatureTruncateOrExtend(FeatureModulationBase): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "feature": ("FEATURE",), + "target_feature_pipe": ("FEATURE_PIPE",), + "fill_method": (["zeros", "ones", "average", "random", "repeat"],), + **super().INPUT_TYPES()["required"], + } + } + + RETURN_TYPES = ("FEATURE", "IMAGE") + RETURN_NAMES = ("FEATURE", "FEATURE_VISUALIZATION") + FUNCTION = "truncate_or_extend" + + def truncate_or_extend(self, feature, target_feature_pipe, fill_method, invert_output): + source_values = [feature.get_value_at_frame(i) for i in range(feature.frame_count)] + target_length = target_feature_pipe.frame_count + + if len(source_values) > target_length: + # Truncate + adjusted_values = source_values[:target_length] + elif len(source_values) < target_length: + # Extend + adjusted_values = source_values.copy() + extension_length = target_length - len(source_values) + + if fill_method == "zeros": + adjusted_values.extend([0] * extension_length) + elif fill_method == "ones": + adjusted_values.extend([1] * extension_length) + elif fill_method == "average": + avg_value = np.mean(source_values) + adjusted_values.extend([avg_value] * extension_length) + elif fill_method == "random": + adjusted_values.extend([random.random() for _ in range(extension_length)]) + elif fill_method == "repeat": + while len(adjusted_values) < target_length: + adjusted_values.extend(source_values[:min(len(source_values), target_length - len(adjusted_values))]) + else: + # Same length, no adjustment needed + adjusted_values = source_values + + processed_feature = self.create_processed_feature(feature, adjusted_values, "TruncatedOrExtended", invert_output) + return (processed_feature, self.visualize(processed_feature)) \ No newline at end of file diff --git a/nodes/flex/features.py b/nodes/flex/features.py index 28eedab..1bb3155 100644 --- a/nodes/flex/features.py +++ b/nodes/flex/features.py @@ -1,6 +1,5 @@ from abc import ABC, abstractmethod import numpy as np -import librosa import torch import cv2 from ..masks.mask_utils import calculate_optical_flow @@ -19,6 +18,12 @@ class BaseFeature(ABC): def extract(self): pass + @classmethod + @abstractmethod + def get_extraction_methods(cls): + """Return a list of parameter names that can be modulated.""" + return [] + def get_value_at_frame(self, frame_index): if self.data is not None: return self.data[frame_index] @@ -26,6 +31,7 @@ class BaseFeature(ABC): return self.features[self.feature_name][frame_index] else: raise ValueError("No data or features available") + def normalize(self): if self.data is not None: @@ -53,6 +59,12 @@ class TimeFeature(BaseFeature): self.offset = offset super().__init__(name, "time", frame_rate, frame_count) + @classmethod + def get_extraction_methods(self): + return [ + "smooth", "accelerate", "pulse", "sawtooth","bounce" + ] + def extract(self): t = np.linspace(0, self.frame_count / self.frame_rate, self.frame_count) t = (t * self.speed + self.offset) % 1 @@ -72,66 +84,20 @@ class TimeFeature(BaseFeature): return self.normalize() -class AudioFeature(BaseFeature): - def __init__(self, name, audio, num_frames, frame_rate, feature_type='amplitude_envelope'): - self.audio = audio['waveform'].squeeze(0).mean(axis=0).cpu().numpy() - self.sample_rate = audio['sample_rate'] - self.num_frames = num_frames - self.feature_type = feature_type - super().__init__(name, "audio", frame_rate, num_frames) - self.frame_duration = 1 / self.frame_rate if self.frame_rate > 0 else len(self.audio) / (self.sample_rate * self.num_frames) - - def extract(self): - if self.feature_type == 'amplitude_envelope': - self.data = self._amplitude_envelope() - elif self.feature_type == 'rms_energy': - self.data = self._rms_energy() - elif self.feature_type == 'spectral_centroid': - self.data = self._spectral_centroid() - elif self.feature_type == 'onset_detection': - self.data = self._onset_detection() - elif self.feature_type == 'chroma_features': - self.data = self._chroma_features() - else: - raise ValueError("Unsupported feature type") - return self.normalize() - - def _get_audio_frame(self, frame_index): - start_time = frame_index * self.frame_duration - end_time = (frame_index + 1) * self.frame_duration - start_sample = int(start_time * self.sample_rate) - end_sample = int(end_time * self.sample_rate) - if start_sample >= len(self.audio): - return np.array([]) # Return empty array if we've run out of audio - return self.audio[start_sample:min(end_sample, len(self.audio))] - - def _amplitude_envelope(self): - def safe_max(frame): - return np.max(np.abs(frame)) if frame.size > 0 else 0 - return np.array([safe_max(self._get_audio_frame(i)) for i in range(self.num_frames)]) - - def _rms_energy(self): - return np.array([np.sqrt(np.mean(self._get_audio_frame(i)**2)) for i in range(self.num_frames)]) - - def _spectral_centroid(self): - return np.array([np.mean(librosa.feature.spectral_centroid(y=self._get_audio_frame(i), sr=self.sample_rate)[0]) for i in range(self.num_frames)]) - - def _onset_detection(self): - return np.array([np.mean(librosa.onset.onset_strength(y=self._get_audio_frame(i), sr=self.sample_rate)) for i in range(self.num_frames)]) - - def _chroma_features(self): - return np.array([np.mean(librosa.feature.chroma_stft(y=self._get_audio_frame(i), sr=self.sample_rate), axis=1) for i in range(self.num_frames)]) - class DepthFeature(BaseFeature): def __init__(self, name, frame_rate, frame_count, depth_maps, feature_name='mean_depth'): self.depth_maps = depth_maps self.features = None self.feature_name = feature_name - self.available_features = [ + self.available_features = self.get_extraction_methods() + super().__init__(name, "depth", frame_rate, frame_count) + + @classmethod + def get_extraction_methods(self): + return [ "mean_depth", "depth_variance", "depth_range", "gradient_magnitude", "foreground_ratio", "midground_ratio", "background_ratio" ] - super().__init__(name, "depth", frame_rate, frame_count) def extract(self): self.features = {self.feature_name: []} @@ -197,12 +163,16 @@ class ColorFeature(BaseFeature): raise ValueError(f"Expected images in BHWC format, but got shape {images.shape}") self.images = images self.feature_name = feature_name - self.available_features = [ + self.available_features = self.get_extraction_methods() + super().__init__(name, "color", frame_rate, frame_count) + + @classmethod + def get_extraction_methods(self): + return [ "dominant_color", "color_variance", "saturation", "red_ratio", "green_ratio", "blue_ratio" ] - super().__init__(name, "color", frame_rate, frame_count) - + def extract(self): self.features = {self.feature_name: []} @@ -275,12 +245,16 @@ class BrightnessFeature(BaseFeature): def __init__(self, name, frame_rate, frame_count, images, feature_name='mean_brightness'): self.images = images self.feature_name = feature_name - self.available_features = [ + self.available_features = self.get_extraction_methods() + super().__init__(name, "brightness", frame_rate, frame_count) + + @classmethod + def get_extraction_methods(self): + return [ "mean_brightness", "brightness_variance", "brightness_histogram", "dark_ratio", "mid_ratio", "bright_ratio" ] - super().__init__(name, "brightness", frame_rate, frame_count) - + def extract(self): self.features = {self.feature_name: []} @@ -342,14 +316,19 @@ class MotionFeature(BaseFeature): self.flow_method = flow_method self.flow_threshold = flow_threshold self.magnitude_threshold = magnitude_threshold - self.available_features = [ + self.available_features = self.get_extraction_methods() + self.progress_callback = progress_callback + super().__init__(name, "motion", frame_rate, frame_count) + + @classmethod + def get_extraction_methods(self): + return [ "mean_motion", "max_motion", "motion_direction", "horizontal_motion", "vertical_motion", "motion_complexity", "motion_speed" ] - self.progress_callback = progress_callback - super().__init__(name, "motion", frame_rate, frame_count) - + + def extract(self): print("Starting MotionFeature extraction") self.features = {self.feature_name: []} @@ -434,9 +413,13 @@ class AreaFeature(BaseFeature): self.masks = masks self.feature_type = feature_type self.threshold = threshold - self.available_features = ["total_area", "largest_contour", "bounding_box"] + self.available_features = self.get_extraction_methods() super().__init__(name, "area", frame_rate, frame_count) + @classmethod + def get_extraction_methods(self): + return ["total_area", "largest_contour", "bounding_box"] + def extract(self): self.data = [] @@ -489,4 +472,5 @@ class AreaFeature(BaseFeature): self.extract() # Re-extract the data with the new feature type else: raise ValueError(f"Invalid feature name. Available features are: {', '.join(self.available_features)}") + #TODO volume feature diff --git a/nodes/flex/features_audio.py b/nodes/flex/features_audio.py new file mode 100644 index 0000000..df7c9f8 --- /dev/null +++ b/nodes/flex/features_audio.py @@ -0,0 +1,383 @@ +from .features import BaseFeature +import librosa +import numpy as np +from scipy.signal import hilbert +from scipy import signal +import functools +from abc import ABC, abstractmethod +import numpy as np +import librosa +from scipy.signal import medfilt, hilbert +from scipy import signal +import functools + +class BaseAudioFeature(BaseFeature): + + @classmethod + @abstractmethod + def get_extraction_methods(cls): + """Return a list of parameter names that can be modulated.""" + return [] + + def __init__(self, name, audio, frame_count, frame_rate): + super().__init__(name, "audio", frame_rate, frame_count) + self.audio = audio + self.sample_rate = None + self.frame_duration = None + self.feature_name = None + self.available_features = [] + + def _prepare_audio(self): + self.sample_rate = self.audio['sample_rate'] + waveform = self.audio['waveform'] + + # Handle multi-dimensional tensors + if waveform.ndim > 1: + waveform = waveform.squeeze() + + # Handle multi-channel audio by averaging channels + if waveform.ndim > 1: + waveform = waveform.mean(axis=0) + + self.audio_array = waveform.cpu().numpy() + self.frame_duration = 1 / self.frame_rate + + def _get_audio_frame(self, frame_index): + start_time = frame_index * self.frame_duration + end_time = start_time + self.frame_duration + start_sample = int(start_time * self.sample_rate) + end_sample = int(end_time * self.sample_rate) + if start_sample >= len(self.audio_array): + return np.array([]) # Return empty array if we've run out of audio + return self.audio_array[start_sample:min(end_sample, len(self.audio_array))] + + def get_feature_sequence(self, feature_name=None): + if self.features is None: + self.extract() + if feature_name is None: + feature_name = self.feature_name + return self.features.get(feature_name, None) + + def set_active_feature(self, feature_name): + if feature_name in self.available_features: + self.feature_name = feature_name + else: + raise ValueError( + f"Invalid feature name. Available features are: {', '.join(self.available_features)}" + ) + +class AudioFeature(BaseAudioFeature): + + def __init__(self, feature_name, audio, frame_count, frame_rate, feature_type='amplitude_envelope'): + super().__init__(feature_name, audio, frame_count, frame_rate) + self.feature_type = feature_type + self.available_features = self.get_extraction_methods() + self.feature_name = feature_type + self._prepare_audio() + + @classmethod + def get_extraction_methods(cls): + """Return a list of parameter names that can be modulated.""" + return [ + 'amplitude_envelope', + 'rms_energy', + 'spectral_centroid', + 'onset_strength', + 'chroma_features' + ] + + def extract(self): + self.features = {self.feature_name: []} + for i in range(self.frame_count): + frame = self._get_audio_frame(i) + value = self._calculate_feature(frame) + self.features[self.feature_name].append(value) + self._normalize_features() + return self + + def _calculate_feature(self, frame): + if frame.size == 0: + return 0.0 + if self.feature_name == 'amplitude_envelope': + return np.max(np.abs(frame)) + elif self.feature_name == 'rms_energy': + return np.sqrt(np.mean(frame ** 2)) + elif self.feature_name == 'spectral_centroid': + centroid = librosa.feature.spectral_centroid(y=frame, sr=self.sample_rate) + return np.mean(centroid) + elif self.feature_name == 'onset_strength': + strength = librosa.onset.onset_strength(y=frame, sr=self.sample_rate) + return np.mean(strength) + elif self.feature_name == 'chroma_features': + chroma = librosa.feature.chroma_stft(y=frame, sr=self.sample_rate) + return np.mean(chroma) + else: + raise ValueError(f"Unsupported feature type: {self.feature_name}") + + def _normalize_features(self): + feature_array = np.array(self.features[self.feature_name], dtype=np.float32) + min_val = np.min(feature_array) + max_val = np.max(feature_array) + if max_val > min_val: + normalized = (feature_array - min_val) / (max_val - min_val) + else: + normalized = np.zeros_like(feature_array) + self.features[self.feature_name] = normalized.tolist() + + +class PitchFeature(BaseAudioFeature): + def __init__( + self, + feature_name, + audio, + frame_count, + frame_rate, + pitch_range_collections=None, + feature_type='frequency', + window_size=0, + vibrato_options=None, + crepe_model="none" + ): + super().__init__(feature_name, audio, frame_count, frame_rate) + self.available_features = self.get_extraction_methods() + self.feature_type = feature_type + self.pitch_range_collections = pitch_range_collections or [] + self.vibrato_options = vibrato_options + self.window_size = window_size + self.current_frame = 0 + self.crepe_model = crepe_model + self.feature_name = feature_name + self._prepare_audio() + + # Parameters for pitch detection + self.frame_length = 2048 + self.hop_length = 512 + self.fmin = 20 # Lower frequency limit + self.fmax = 4000 # Upper frequency limit + + @staticmethod + def calculate_tolerance(frequency, tolerance_percent): + semitone_above = frequency * 2**(1/12) + frequency_difference = semitone_above - frequency + tolerance = (frequency_difference / 2) * (tolerance_percent / 100.0) + return tolerance + + @classmethod + def get_extraction_methods(cls): + """Return a list of parameter names that can be modulated.""" + return [ + 'frequency', + 'semitone', + 'pitch_direction', + 'vibrato_signal', + 'vibrato_strength', + ] + + @classmethod + def pitch_to_note(cls, pitch): + if pitch == 0: + return "N/A" + return librosa.hz_to_note(pitch) + + def _calculate_pitch_sequence(self): + # Try to use CREPE model if available + if self.crepe_model != "none": + try: + import crepe + import tensorflow + time_steps, frequencies, confidence, activation = crepe.predict( + self.audio_array, + self.sample_rate, + model_capacity=self.crepe_model, + viterbi=True, + step_size=int(1000 * self.hop_length / self.sample_rate) # Step size in ms + ) + # time_steps, frequencies, confidences, _ = crepe.predict( + # self.audio_array, + # self.sample_rate, + # model_capacity=self.crepe_model, + # viterbi=True, + # step_size=int(1000 * self.hop_length / self.sample_rate) # Step size in ms + # ) + pitches = frequencies + confidences = confidence + except ImportError as e: + print(f"Please 'pip install crepe tensorflow' to use any CREPE model :)") + pitches, confidences = self._fallback_pitch_estimation() + except Exception as e: + print(f"Error using CREPE model: {e}. Falling back to librosa's pitch estimation.") + pitches, confidences = self._fallback_pitch_estimation() + else: + # Fallback to librosa if CREPE model is not provided + pitches, confidences = self._fallback_pitch_estimation() + + # Interpolate to match frame count + frame_times = np.linspace(0, len(self.audio_array) / self.sample_rate, num=self.frame_count) + if len(pitches) == 0: + pitches = np.zeros(len(frame_times)) + confidences = np.zeros(len(frame_times)) + pitch_times = frame_times + else: + pitch_times = librosa.frames_to_time(np.arange(len(pitches)), sr=self.sample_rate, hop_length=self.hop_length) + pitches = np.nan_to_num(pitches) + confidences = np.nan_to_num(confidences) + pitches = np.interp(frame_times, pitch_times, pitches) + confidences = np.interp(frame_times, pitch_times, confidences) + + # Handle potential NaNs + pitches = np.nan_to_num(pitches) + confidences = np.nan_to_num(confidences) + + self.features = {} + self.features[self.feature_name + '_pitch'] = pitches.tolist() + self.features[self.feature_name + '_confidence'] = confidences.tolist() + + def _fallback_pitch_estimation(self): + pitches, magnitudes = librosa.piptrack( + y=self.audio_array, + sr=self.sample_rate, + fmin=self.fmin, + fmax=self.fmax, + hop_length=self.hop_length, + threshold=0.1 + ) + pitches = pitches.max(axis=0) + confidences = magnitudes.max(axis=0) + return pitches, confidences + + def extract(self): + self.features = {} + self._calculate_pitch_sequence() + + if self.feature_type == 'frequency': + self._extract_frequency() + elif self.feature_type == 'semitone': + self._extract_semitone() + elif self.feature_type == 'pitch_direction': + self._extract_pitch_direction() + elif self.feature_type == 'vibrato_signal': + self._extract_vibrato_signal() + elif self.feature_type == 'vibrato_strength': + self._extract_vibrato_strength() + else: + raise ValueError(f"Unsupported feature type: {self.feature_type}") + + self._normalize_features() + return self + + def _extract_frequency(self): + pitches = np.array(self.features[self.feature_name + '_pitch']) + valid_pitches = pitches[pitches > 0] + + if len(valid_pitches) == 0: + feature_values = np.zeros_like(pitches) + else: + feature_values = pitches.copy() + self.features[self.feature_name + '_original'] = feature_values.tolist() + self.features[self.feature_name] = feature_values.tolist() + + def _extract_semitone(self): + pitches = np.array(self.features[self.feature_name + '_pitch']) + midi_notes = librosa.hz_to_midi(pitches) + midi_notes = np.round(midi_notes) + semitone_freqs = librosa.midi_to_hz(midi_notes) + + semitone_freqs[pitches == 0] = 0.0 + self.features[self.feature_name + '_original'] = semitone_freqs.tolist() + self.features[self.feature_name] = semitone_freqs.tolist() + + def _extract_pitch_direction(self): + pitches = np.array(self.features[self.feature_name + '_pitch']) + pitches[pitches == 0] = np.nan # Replace zeros with NaN for diff calculation + pitch_diff = np.diff(pitches, prepend=pitches[0]) + + direction = np.sign(pitch_diff) + direction = np.nan_to_num(direction) # Replace NaNs back to zero + + self.features[self.feature_name + '_original'] = direction.tolist() + self.features[self.feature_name] = direction.tolist() + + def _extract_vibrato_signal(self): + pitches = np.array(self.features[self.feature_name + '_pitch']) + pitches[pitches == 0] = np.nan # Ignore zero pitches + + # Interpolate to fill NaNs for Hilbert transform + valid_indices = np.where(~np.isnan(pitches))[0] + if len(valid_indices) == 0: + vibrato_signal = np.zeros_like(pitches) + else: + pitches = np.interp(np.arange(len(pitches)), valid_indices, pitches[valid_indices]) + + analytic_signal = hilbert(pitches) + instantaneous_phase = np.unwrap(np.angle(analytic_signal)) + instantaneous_frequency = np.diff(instantaneous_phase) + + vibrato_signal = np.concatenate(([0], instantaneous_frequency)) + + self.features[self.feature_name + '_original'] = vibrato_signal.tolist() + self.features[self.feature_name] = vibrato_signal.tolist() + + def _extract_vibrato_strength(self): + pitches = np.array(self.features[self.feature_name + '_pitch']) + pitches[pitches == 0] = np.nan # Ignore zero pitches + + window_size_frames = int(0.1 * self.sample_rate / self.hop_length) # 100ms window + if window_size_frames < 1: + window_size_frames = 1 + + # Compute standard deviation over the window + vibrato_strength = [] + for i in range(len(pitches)): + start = max(0, i - window_size_frames // 2) + end = min(len(pitches), i + window_size_frames // 2) + window_pitches = pitches[start:end] + window_pitches = window_pitches[~np.isnan(window_pitches)] + if len(window_pitches) > 0: + std = np.std(window_pitches) + else: + std = 0.0 + vibrato_strength.append(std) + + vibrato_strength = np.array(vibrato_strength) + self.features[self.feature_name + '_original'] = vibrato_strength.tolist() + self.features[self.feature_name] = vibrato_strength.tolist() + + def _normalize_features(self): + feature_array = np.array(self.features[self.feature_name], dtype=np.float32) + finite_mask = np.isfinite(feature_array) + if not np.any(finite_mask): + normalized = np.zeros_like(feature_array) + else: + min_val = np.min(feature_array[finite_mask]) + max_val = np.max(feature_array[finite_mask]) + if max_val > min_val: + normalized = np.zeros_like(feature_array) + normalized[finite_mask] = (feature_array[finite_mask] - min_val) / (max_val - min_val) + else: + normalized = np.zeros_like(feature_array) + self.features[self.feature_name] = normalized.tolist() + + def get_pitch_feature(self, frame_index): + if self.features is None: + self.extract() + + original_value = self.features[self.feature_name + '_original'][frame_index] + normalized_value = self.features[self.feature_name][frame_index] + actual_pitch = self.features[self.feature_name + '_pitch'][frame_index] + # Provide smoothed pitch if available + smoothed_pitch = self.features.get(self.feature_name + '_smoothed', [0.0]*self.frame_count)[frame_index] + + return { + 'original': original_value, + 'normalized': normalized_value, + 'actual_pitch': actual_pitch, + 'smoothed_pitch': smoothed_pitch + } + +class PitchRange: + def __init__(self, min_pitch, max_pitch): + self.min_pitch = min_pitch + self.max_pitch = max_pitch + + def contains(self, pitch): + return self.min_pitch <= pitch <= self.max_pitch \ No newline at end of file diff --git a/nodes/flex/features_proximity.py b/nodes/flex/features_proximity.py new file mode 100644 index 0000000..65cbd3d --- /dev/null +++ b/nodes/flex/features_proximity.py @@ -0,0 +1,95 @@ +from .features import BaseFeature +import torch +import numpy as np + +import numpy as np +from .features import BaseFeature + +class ProximityFeature(BaseFeature): + def __init__(self, name, anchor_locations, query_locations, frame_rate, frame_count, frame_dimensions, normalization_method='frame'): + super().__init__(name, "proximity", frame_rate, frame_count) + self.anchor_locations = anchor_locations + self.query_locations = query_locations + self.frame_diagonal = np.sqrt(frame_dimensions[0]**2 + frame_dimensions[1]**2) + self.proximity_values = None + self.normalization_method = normalization_method + + def extract(self): + from scipy.spatial.distance import cdist + + proximities = [] + for anchor, query in zip(self.anchor_locations, self.query_locations): + if len(anchor) == 0 or len(query) == 0: + proximities.append(self.frame_diagonal if self.normalization_method == 'frame' else float('inf')) + else: + # Ensure points are treated as floats + anchor_points = anchor.points.astype(float) + query_points = query.points.astype(float) + distances = cdist(anchor_points, query_points) + min_distance = np.min(distances) + proximities.append(min_distance) + + proximities = np.array(proximities, dtype=float) + + if self.normalization_method == 'frame': + self.proximity_values = 1 - np.clip(proximities / self.frame_diagonal, 0, 1) + elif self.normalization_method == 'minmax': + min_proximity = np.min(proximities) + max_proximity = np.max(proximities) + if max_proximity > min_proximity: + self.proximity_values = 1 - (proximities - min_proximity) / (max_proximity - min_proximity) + else: + self.proximity_values = np.ones_like(proximities) + else: + raise ValueError(f"Unknown normalization method: {self.normalization_method}") + + self.smooth_proximities() + return self + + def smooth_proximities(self, window_size=5): + kernel = np.ones(window_size) / window_size + self.proximity_values = np.convolve(self.proximity_values, kernel, mode='same') + + def get_value_at_frame(self, frame_index): + return self.proximity_values[frame_index] + +class Location: + def __init__(self, x, y, z=None): + x = np.asarray(x, dtype=float).reshape(-1) + y = np.asarray(y, dtype=float).reshape(-1) + + if z is not None: + z = np.asarray(z, dtype=float).reshape(-1) + if abs(len(z) - len(x)) <= 1: + z = z[:len(x)] if len(z) > len(x) else np.pad(z, (0, len(x) - len(z)), 'constant', constant_values=0.0) + elif len(z) != len(x): + raise ValueError(f"Mismatch in dimensions: x,y have length {len(x)}, but z has length {len(z)}") + self.points = np.column_stack((x, y, z)) + else: + self.points = np.column_stack((x, y)) + + + def __len__(self): + return len(self.points) + + def __getitem__(self, idx): + return self.points[idx] + + @property + def x(self): + return self.points[:, 0] + + @property + def y(self): + return self.points[:, 1] + + @property + def z(self): + return self.points[:, 2] if self.points.shape[1] > 2 else None + + @classmethod + def from_tensor(cls, tensor): + return cls(tensor[:, 0].astype(float), tensor[:, 1].astype(float), tensor[:, 2].astype(float) if tensor.shape[1] > 2 else None) + + def to_tensor(self): + return torch.from_numpy(self.points).float() \ No newline at end of file diff --git a/nodes/flex/visualizers.py b/nodes/flex/visualizers.py new file mode 100644 index 0000000..2d1e2df --- /dev/null +++ b/nodes/flex/visualizers.py @@ -0,0 +1,197 @@ +from ... import RyanOnTheInside +import cv2 +import numpy as np +import torch +from scipy.spatial.distance import cdist +import math + +class EffectVisualizer(RyanOnTheInside): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "video_frames": ("IMAGE",), + "feature": ("FEATURE",), + "text_color": ("STRING", {"default": "(255,255,255)"}), + "font_scale": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 2.0, "step": 0.1}), + } + } + + RETURN_TYPES = ("IMAGE",) + FUNCTION = "visualize" + CATEGORY = "RyanOnTheInside/FlexFeatures/EffectVisualizers" + + def visualize(self, video_frames, feature, text_color, font_scale): + text_color = self.parse_color(text_color) + output_frames = [] + padding = 10 # Padding from the edges + + for frame_index in range(len(video_frames)): + frame = video_frames[frame_index].cpu().numpy() + frame = (frame * 255).astype(np.uint8) + frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) + + # Display feature name and value on the frame + feature_value = feature.get_value_at_frame(frame_index) + text = f"{feature.name}: {feature_value:.2f}" + text_size = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, font_scale, 2)[0] + text_x = padding + text_y = padding + text_size[1] + + cv2.putText(frame, text, (text_x, text_y), cv2.FONT_HERSHEY_SIMPLEX, font_scale, text_color, 2) + + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + output_frames.append(frame) + + output_tensor = torch.from_numpy(np.stack(output_frames)).float() / 255.0 + return (output_tensor,) + + def parse_color(self, color_string): + return tuple(map(int, color_string.strip("()").split(","))) + + +class ProximityVisualizer(EffectVisualizer): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + **super().INPUT_TYPES()["required"], + "anchor_locations": ("LOCATION",), + "query_locations": ("LOCATION",), + "anchor_color": ("STRING", {"default": "(255,0,0)"}), + "query_color": ("STRING", {"default": "(0,255,0)"}), + "line_color": ("STRING", {"default": "(0,0,255)"}), + } + } + + RETURN_TYPES = ("IMAGE",) + FUNCTION = "visualize_proximity" + + def visualize_proximity(self, video_frames, anchor_locations, query_locations, feature, + anchor_color, query_color, line_color, text_color, font_scale): + anchor_color = self.parse_color(anchor_color) + query_color = self.parse_color(query_color) + line_color = self.parse_color(line_color) + text_color = self.parse_color(text_color) + + output_frames = [] + height, width = video_frames.shape[1:3] # Extract height and width from video_frames + + # Calculate the frame diagonal from the video frames + frame_diagonal = np.sqrt(width**2 + height**2) + scale_factor = frame_diagonal / feature.frame_diagonal + + for frame_index in range(len(video_frames)): + frame = video_frames[frame_index].cpu().numpy() + frame = (frame * 255).astype(np.uint8) + frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) + + anchor = anchor_locations[frame_index] + query = query_locations[frame_index] + + # Draw all anchor and query points (using only x and y coordinates) + for point in anchor: + scaled_point = (int(point[0] * scale_factor), int(point[1] * scale_factor)) + cv2.circle(frame, scaled_point, 2, anchor_color, -1) + for point in query: + scaled_point = (int(point[0] * scale_factor), int(point[1] * scale_factor)) + cv2.circle(frame, scaled_point, 2, query_color, -1) + + # Find the closest pair of points + if len(anchor) > 0 and len(query) > 0: + distances = cdist(anchor.points[:, :2], query.points[:, :2]) # Use only x and y for distance calculation + min_idx = np.unravel_index(distances.argmin(), distances.shape) + closest_anchor = anchor[min_idx[0]] + closest_query = query[min_idx[1]] + + # Draw line between closest points (using only x and y coordinates) + cv2.line(frame, + (int(closest_anchor[0] * scale_factor), int(closest_anchor[1] * scale_factor)), + (int(closest_query[0] * scale_factor), int(closest_query[1] * scale_factor)), + line_color, 2) + + # Display coordinates of closest points + anchor_text = f"Anchor: ({closest_anchor[0] * scale_factor:.2f}, {closest_anchor[1] * scale_factor:.2f}" + query_text = f"Query: ({closest_query[0] * scale_factor:.2f}, {closest_query[1] * scale_factor:.2f}" + + if closest_anchor.shape[0] > 2 and closest_query.shape[0] > 2: + anchor_text += f", {closest_anchor[2]:.2f})" + query_text += f", {closest_query[2]:.2f})" + else: + anchor_text += ") [2D]" + query_text += ") [2D]" + + cv2.putText(frame, anchor_text, (10, 60), cv2.FONT_HERSHEY_SIMPLEX, font_scale, text_color, 1) + cv2.putText(frame, query_text, (10, 90), cv2.FONT_HERSHEY_SIMPLEX, font_scale, text_color, 1) + + # Add proximity value to the frame + proximity_value = feature.get_value_at_frame(frame_index) + cv2.putText(frame, f"Proximity: {proximity_value:.2f}", (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 1, text_color, 2) + + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + output_frames.append(frame) + + output_tensor = torch.from_numpy(np.stack(output_frames)).float() / 255.0 + return (output_tensor,) + + + +class PitchVisualizer(EffectVisualizer): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "video_frames": ("IMAGE",), + "feature": ("FEATURE",), + "text_color": ("STRING", {"default": "(255,255,255)"}), + "font_scale": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 2.0, "step": 0.1}), + } + } + + RETURN_TYPES = ("IMAGE",) + FUNCTION = "visualize_pitch" + + def visualize_pitch(self, video_frames, feature, text_color, font_scale): + text_color = self.parse_color(text_color) + output_frames = [] + padding = 10 # Padding from the edges + + # Ensure video_frames is BHWC + if video_frames.shape[-1] != 3: + video_frames = video_frames.permute(0, 2, 3, 1) + + for frame_index in range(video_frames.shape[0]): + frame = video_frames[frame_index].cpu().numpy() + frame = (frame * 255).astype(np.uint8) + frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) + + # Get pitch feature data for the current frame + pitch_data = feature.get_pitch_feature(frame_index) + + # Convert pitch to approximate note + note = feature.pitch_to_note(pitch_data['actual_pitch']) + smoothed_note = feature.pitch_to_note(pitch_data['smoothed_pitch']) + + # Display feature values on the frame + texts = [ + f"Original: {pitch_data['original']:.2f}", + f"Normalized: {pitch_data['normalized']:.2f}", + f"Actual Pitch: {pitch_data['actual_pitch']:.2f} Hz", + f"Note: {note}", + f"Smoothed Pitch: {pitch_data['smoothed_pitch']:.2f} Hz", + f"Smoothed Note: {smoothed_note}" + ] + + for i, text in enumerate(texts): + text_size = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, font_scale, 2)[0] + text_x = padding + text_y = padding + (i + 1) * (text_size[1] + 10) # Add some vertical spacing + + cv2.putText(frame, text, (text_x, text_y), cv2.FONT_HERSHEY_SIMPLEX, font_scale, text_color, 2) + + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + output_frames.append(frame) + + output_tensor = torch.from_numpy(np.stack(output_frames)).float() / 255.0 + return (output_tensor,) diff --git a/nodes/masks/flex_masks_normal.py b/nodes/masks/flex_masks_normal.py new file mode 100644 index 0000000..9fb2ada --- /dev/null +++ b/nodes/masks/flex_masks_normal.py @@ -0,0 +1,102 @@ +import numpy as np +import torch +import cv2 +from .mask_base import FlexMaskBase + +class FlexMaskNormalBase(FlexMaskBase): + @classmethod + def INPUT_TYPES(cls): + return { + **super().INPUT_TYPES(), + "required": { + **super().INPUT_TYPES()["required"], + "normal_map": ("IMAGE",), + } + } + + def normalize_vector(self, vector): + return vector / np.linalg.norm(vector) + + def normal_map_to_array(self, normal_map: torch.Tensor) -> np.ndarray: + # Convert normal map from [0, 1] to [-1, 1] range + normal_array = normal_map.cpu().numpy() * 2 - 1 + return normal_array + + def main_function(self, masks, feature, feature_pipe, strength, feature_threshold, invert, subtract_original, grow_with_blur, normal_map, **kwargs): + # This method should be overridden by child classes + raise NotImplementedError("Subclasses must implement main_function") + +class FlexMaskNormalLighting(FlexMaskNormalBase): + @classmethod + def INPUT_TYPES(cls): + return { + **super().INPUT_TYPES(), + "required": { + **super().INPUT_TYPES()["required"], + "light_direction_x": ("FLOAT", {"default": 0.5, "min": -1.0, "max": 1.0, "step": 0.01}), + "light_direction_y": ("FLOAT", {"default": -0.5, "min": -1.0, "max": 1.0, "step": 0.01}), + "light_direction_z": ("FLOAT", {"default": 0.7, "min": -1.0, "max": 1.0, "step": 0.01}), + "shadow_threshold": ("FLOAT", {"default": 0.2, "min": 0.0, "max": 1.0, "step": 0.01}), + "feature_param": (["none", "direction", "threshold", "both"],), + "feature_mode": (["rotate", "intensity"],), + } + } + + CATEGORY = "RyanOnTheInside/FlexMasks" + + def process_mask(self, mask: np.ndarray, feature_value: float, strength: float, + normal_map: torch.Tensor, light_direction_x: float, light_direction_y: float, + light_direction_z: float, shadow_threshold: float, feature_param: str, + feature_mode: str, **kwargs) -> np.ndarray: + + normal_array = self.normal_map_to_array(normal_map) + + # Normalize light direction + light_direction = self.normalize_vector(np.array([light_direction_x, light_direction_y, light_direction_z])) + + # Apply feature modulation + if feature_param != "none": + if feature_mode == "rotate": + # Rotate light direction based on feature value + rotation_angle = 2 * np.pi * feature_value * strength + rotation_matrix = np.array([ + [np.cos(rotation_angle), -np.sin(rotation_angle), 0], + [np.sin(rotation_angle), np.cos(rotation_angle), 0], + [0, 0, 1] + ]) + light_direction = np.dot(rotation_matrix, light_direction) + elif feature_mode == "intensity": + # Modify shadow threshold based on feature value + shadow_threshold += (feature_value - 0.5) * strength + shadow_threshold = np.clip(shadow_threshold, 0.0, 1.0) + + # Calculate dot product between normal vectors and light direction + dot_product = np.einsum('bhwc,c->bhw', normal_array, light_direction) + + # Create lighting mask + lighting_mask = (dot_product > shadow_threshold).astype(np.float32) + + # Combine with input mask + combined_mask = mask * lighting_mask + + return combined_mask + + def main_function(self, masks, feature, feature_pipe, strength, feature_threshold, invert, subtract_original, grow_with_blur, normal_map, light_direction_x, light_direction_y, light_direction_z, shadow_threshold, feature_param, feature_mode, **kwargs): + processed_masks = [] + for i in range(masks.shape[0]): # Iterate over all frames + mask = masks[i].cpu().numpy() + feature_value = feature.get_value_at_frame(i) + + if feature_value >= feature_threshold: + processed_mask = self.process_mask( + mask, feature_value, strength, normal_map[i], + light_direction_x, light_direction_y, light_direction_z, + shadow_threshold, feature_param, feature_mode + ) + else: + processed_mask = mask + + processed_masks.append(processed_mask) + + processed_masks = np.stack(processed_masks) + return self.apply_mask_operation(torch.from_numpy(processed_masks), masks, strength, invert, subtract_original, grow_with_blur) \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index ad3cc4e..2b4eab6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui_ryanonyheinside" description = "Custom nodes introducing particle simulations, optical flow, audio manipulation & reactivity, and temporal masks" -version = "1.6.0" +version = "1.8.0" license = {file = "LICENSE"} dependencies = ["opencv-python==4.10.0","scipy","torchaudio", "pillow","librosa","pymunk==6.8.1","matplotlib","openunmix","mido"] diff --git a/readme.md b/readme.md index 96979e7..0eae930 100644 --- a/readme.md +++ b/readme.md @@ -33,6 +33,7 @@ *Examples showcasing various effects using particle emitters, vortices, and other node features* ##### 🆕 Recent Updates +-9/15/24 - **alot** Depth From Shape, Audio Pitch Feature, Pitch Range filters, new MIDI keybord for Pitch Range specification, Image from audio, mask from audio, Improved depth chamber, wave propagation, emanating rings, and a lot more - 9/8/24 - **Area Feature**: Adds area as a driving reactivity feature! - 9/7/24 - **Proximity Feature**: Adds proximity as a driving reactivity feature! Allows for the distince of objects from one another to control other nodes. - 9/5/24 - **FlexImageParallax**: Add illusory depth. diff --git a/web/js/midi_extensions.js b/web/js/midi_extensions.js index 4628a55..d1a352a 100644 --- a/web/js/midi_extensions.js +++ b/web/js/midi_extensions.js @@ -1,5 +1,4 @@ import { app } from "../../scripts/app.js"; -import { api } from "../../scripts/api.js"; import { drawPiano, handlePianoMouseDown } from "./piano.js"; app.registerExtension({ @@ -11,7 +10,6 @@ app.registerExtension({ async beforeRegisterNodeDef(nodeType, nodeData, app) { if (nodeData.name === "MIDILoadAndExtract") { - console.log("MIDI Extension: Modifying MIDILoadAndExtract node"); // Add the disableWidget method to the nodeType prototype nodeType.prototype.disableWidget = function(widget) { @@ -41,7 +39,6 @@ app.registerExtension({ .then(response => response.json()) .then(data => { if (data.success) { - console.log("MIDI file uploaded successfully"); // Update the midi_file widget with the new file path const midiFileWidget = this.widgets.find(w => w.name === "midi_file"); if (midiFileWidget) { @@ -67,7 +64,6 @@ app.registerExtension({ }; nodeType.prototype.onNodeCreated = function() { - console.log("MIDI Extension: onNodeCreated called for MIDILoadAndExtract"); this.properties = this.properties || {}; this.properties.selectedNotes = []; // Initialize with an empty array this.pianoScroll = 0; @@ -95,7 +91,6 @@ app.registerExtension({ const midiFileWidget = this.widgets.find(w => w.name === "midi_file"); if (midiFileWidget) { midiFileWidget.callback = (value) => { - console.log('MIDI file changed to:', value); this.updateTrackSelection(); this.saveState(); // Save state when MIDI file changes }; @@ -127,28 +122,23 @@ app.registerExtension({ // Instead of calling updateTrackSelection here, we'll set up a callback // that will be called when the graph is available this.onGraphAvailable = () => { - console.log('Graph available, updating track selection'); this.updateTrackSelection(); }; }; nodeType.prototype.loadSavedState = function() { - console.log(`loadSavedState called for node ${this.id}`); const savedState = localStorage.getItem(`MIDILoadAndExtract_${this.id}`); if (savedState) { this.loadedState = JSON.parse(savedState); - console.log('Loaded saved state:', this.loadedState); const midiFileWidget = this.widgets.find(w => w.name === "midi_file"); if (midiFileWidget && this.loadedState.midiFile) { midiFileWidget.value = this.loadedState.midiFile; - console.log(`Set midiFile to: ${midiFileWidget.value}`); } } }; nodeType.prototype.updateTrackSelection = async function() { - console.log(`updateTrackSelection called, isInitialLoad: ${this.isInitialLoad}`); if (this.isUpdatingTrackSelection) { console.log('Track selection update already in progress, skipping'); return; @@ -157,7 +147,6 @@ app.registerExtension({ try { const midiFile = this.getCurrentMidiFile(); - console.log(`Current MIDI file: ${midiFile}`); if (!midiFile) { console.warn('No MIDI file selected'); @@ -176,7 +165,6 @@ app.registerExtension({ return; } - console.log('Received track data:', data); const trackSelectionWidget = this.widgets.find(w => w.name === "track_selection"); if (trackSelectionWidget) { @@ -186,7 +174,6 @@ app.registerExtension({ // Update available notes this.availableNotes = new Set(data.all_notes.split(',').map(Number)); - console.log(`Updated available notes: ${Array.from(this.availableNotes)}`); // Apply loaded state if this is the initial load if (this.isInitialLoad && this.loadedState) { @@ -195,7 +182,6 @@ app.registerExtension({ } this.setDirtyCanvas(true, true); - console.log('Track selection update completed'); } catch (error) { console.error('Error updating track selection:', error); this.availableNotes = new Set(); // Reset to empty Set in case of error @@ -205,7 +191,6 @@ app.registerExtension({ }; nodeType.prototype.refreshMIDIData = async function() { - console.log("Refreshing MIDI data"); const midiFile = this.getCurrentMidiFile(); const trackSelection = this.widgets.find(w => w.name === "track_selection").value; @@ -227,7 +212,6 @@ app.registerExtension({ return; } - console.log('Received refreshed MIDI data:', data); // Update available tracks const trackSelectionWidget = this.widgets.find(w => w.name === "track_selection"); @@ -237,7 +221,6 @@ app.registerExtension({ // Update available notes this.availableNotes = new Set(data.all_notes.split(',').map(Number)); - console.log(`Updated available notes: ${Array.from(this.availableNotes)}`); // Filter out selected notes that are no longer available if (this.properties.selectedNotes) { @@ -256,13 +239,11 @@ app.registerExtension({ nodeType.prototype.applyLoadedState = function() { if (this.loadedState) { - console.log('Applying loaded state:', this.loadedState); const trackSelectionWidget = this.widgets.find(w => w.name === "track_selection"); if (trackSelectionWidget && this.loadedState.trackSelection) { if (trackSelectionWidget.options.values.includes(this.loadedState.trackSelection)) { trackSelectionWidget.value = this.loadedState.trackSelection; - console.log(`Set track selection to loaded value: ${trackSelectionWidget.value}`); } else { console.log(`Loaded track selection ${this.loadedState.trackSelection} is no longer valid`); } @@ -270,7 +251,6 @@ app.registerExtension({ if (this.loadedState.selectedNotes) { this.properties.selectedNotes = this.loadedState.selectedNotes.filter(note => this.availableNotes.has(note)); - console.log(`Applied selected notes: ${this.properties.selectedNotes}`); } this.syncNotesWidget(); @@ -285,7 +265,6 @@ app.registerExtension({ trackSelection: this.widgets.find(w => w.name === "track_selection")?.value, selectedNotes: this.properties.selectedNotes }; - console.log("Saving state:", state); localStorage.setItem(`MIDILoadAndExtract_${this.id}`, JSON.stringify(state)); }; diff --git a/web/js/pitch_extension.js b/web/js/pitch_extension.js new file mode 100644 index 0000000..60acfe6 --- /dev/null +++ b/web/js/pitch_extension.js @@ -0,0 +1,108 @@ +import { app } from "../../scripts/app.js"; +import { drawPiano, handlePianoMouseDown } from "./piano.js"; + +app.registerExtension({ + name: "org.ryanontheinside.pitch_range_by_note", + + async beforeRegisterNodeDef(nodeType, nodeData, app) { + if (nodeData.name === "PitchRangeByNoteNode") { + + nodeType.prototype.onNodeCreated = function() { + this.properties = this.properties || {}; + this.properties.selectedNotes = []; // Initialize with an empty array + this.pianoScroll = 0; + this.scrollLeftIndicator = null; + this.scrollRightIndicator = null; + this.availableNotes = new Set(Array.from({length: 128}, (_, i) => i)); // All MIDI notes + + const notesWidget = this.widgets.find(widget => widget.name === "notes"); + if (notesWidget) { + notesWidget.hidden = true; + notesWidget.disabled = true; // Disable the widget immediately + } + + // Increase the node's height and width + const extraHeight = 80; // Piano height + margins + const pianoWidth = 1020; // Total width of the piano + this.size[1] += extraHeight; + this.size[0] = Math.max(this.size[0], pianoWidth); // Set width to at least piano width plus margins + + // Ensure the node recomputes its size + this.setDirtyCanvas(true, true); + + // Sync the notes widget initially + this.syncNotesWidget(); + }; + + nodeType.prototype.syncNotesWidget = function() { + const notesWidget = this.widgets.find(widget => widget.name === "notes"); + if (notesWidget) { + const newValue = this.properties.selectedNotes.sort((a, b) => a - b).join(", "); + if (notesWidget.value !== newValue) { + notesWidget.value = newValue; + // Trigger a change event on the widget + if (notesWidget.callback) { + notesWidget.callback(newValue); + } + } + } + // Ensure the node updates its internal state + this.onPropertyChanged('selectedNotes'); + }; + + nodeType.prototype.onPropertyChanged = function(property) { + if (property === 'selectedNotes') { + // Trigger any necessary updates or recomputation + this.setDirtyCanvas(true, true); + } + }; + + // Save the original method + nodeType.prototype._originalOnDrawForeground = nodeType.prototype.onDrawForeground; + + nodeType.prototype.onDrawForeground = function(ctx) { + // Call the original onDrawForeground method if it exists + if (typeof nodeType.prototype._originalOnDrawForeground === 'function') { + nodeType.prototype._originalOnDrawForeground.call(this, ctx); + } + + if (this.flags.collapsed) return; + + // Draw widgets that are not hidden + for (let w of this.widgets) { + if (w.hidden) { + continue; // Skip drawing hidden widgets + } + if (w.draw) { + w.draw(ctx, this, this.pos); + } + } + + drawPiano(ctx, this); + }; + + // Save the original onMouseDown method + nodeType.prototype._originalOnMouseDown = nodeType.prototype.onMouseDown; + + nodeType.prototype.onMouseDown = function(event, pos, graphcanvas) { + // Call the original onMouseDown method if it exists + if (typeof nodeType.prototype._originalOnMouseDown === 'function') { + nodeType.prototype._originalOnMouseDown.call(this, event, pos, graphcanvas); + } + + const [x, y] = pos; + return handlePianoMouseDown(this, x, y); + }; + + // Disable the notes widget + const onConnectionsChange = nodeType.prototype.onConnectionsChange; + nodeType.prototype.onConnectionsChange = function (type, index, connected, link_info) { + onConnectionsChange?.apply(this, arguments); + const notesWidget = this.widgets.find((w) => w.name === "notes"); + if (notesWidget) { + notesWidget.disabled = true; + } + }; + } + }, +}); \ No newline at end of file