Depth from shape, Pitch Feature, Pitch Range specification, etc

__init__.py

@@ -1,4 +1,4 @@
-from .node_configs import CombinedMeta
+from .node_configs.node_configs import CombinedMeta
 from collections import OrderedDict
 
 #allows for central management and inheritance of class variables for help documentation
@@ -51,16 +51,17 @@ from .nodes.masks.temporal_masks import (
 from .nodes.audio.audio_nodes import (
     AudioSeparator, 
     DownloadOpenUnmixModel,
+    # DownloadCREPEModel,
     AudioFeatureVisualizer,
     FrequencyFilterCustom,
     FrequencyFilterPreset,
     AudioFilter,
+    EmptyMaskFromAudio,
+    EmptyImageFromAudio,
     
 )
 
 from .nodes.flex.feature_extractors import(
-    
-    AudioFeatureExtractor,
     TimeFeatureNode,
     DepthFeatureNode,
     ColorFeatureNode,
@@ -69,18 +70,30 @@ from .nodes.flex.feature_extractors import(
     AreaFeatureNode,
 )
 
-from .nodes.flex.midi_feature_extractor import(
+from .nodes.flex.feature_extractors_audio import(
+    AudioFeatureExtractor,
+    PitchRangeNode,
+    PitchRangePresetNode,
+    PitchRangeByNoteNode,
+    PitchFeatureExtractor,
+)
+
+from .nodes.flex.feature_extractors_midi import(
     MIDILoadAndExtract,
 )
 
-from .nodes.flex.proximity_feature_extractor import(
+from .nodes.flex.feature_extractors_proximity import(
     LocationFromMask,
     ProximityFeatureNode,
-    ProximityVisualizer,
     LocationFromPoint,
     LocationTransform,
 )
 
+from .nodes.flex.visualizers import(
+    ProximityVisualizer,
+    EffectVisualizer,
+    PitchVisualizer,
+)
 
 
 
@@ -103,11 +116,6 @@ from .nodes.masks.particle_system_masks import (
     ParticleSpeedModulation,
     )
 
-# from .nodes.masks.opacity_masks import(
-#     DepthBasedMaskOpacity,
-#     FlexDepthBasedMaskOpacity,
-# )
-
 from .nodes.masks.mask_utility_nodes import (
     _mfc, 
     TextMaskNode, 
@@ -146,6 +154,11 @@ from .nodes.masks.flex_masks import (
 
 )
 
+from .nodes.masks.flex_masks_normal import (
+    FlexMaskNormalLighting,
+
+)
+
 from .nodes.images.flex_images import (
     FlexImageEdgeDetect,
     FlexImagePosterize,
@@ -175,6 +188,7 @@ from .nodes.flex.feature_modulation import (
     PreviewFeature,
     FeatureMath,
     FeatureRebase,
+    FeatureTruncateOrExtend,
 )
 
 
@@ -232,18 +246,26 @@ NODE_CLASS_MAPPINGS = {
     "FlexMaskEmanatingRings":       FlexMaskEmanatingRings,
     "FlexMaskRandomShapes":         FlexMaskRandomShapes,
     "FlexMaskDepthChamber":         FlexMaskDepthChamber,
+    #"FlexMaskNormalLighting":       FlexMaskNormalLighting,
     # "FlexMaskDepthChamberRelative": FlexMaskDepthChamberRelative,
 
     #audio  
     "AudioSeparator":               AudioSeparator,
     "DownloadOpenUnmixModel":       DownloadOpenUnmixModel,
+    # "DownloadCREPEModel":           DownloadCREPEModel,
     "AudioFeatureVisualizer":       AudioFeatureVisualizer,
     "FrequencyFilterCustom":        FrequencyFilterCustom,
     "FrequencyFilterPreset":        FrequencyFilterPreset,
     "AudioFilter":                  AudioFilter,
+    "EmptyMaskFromAudio":           EmptyMaskFromAudio,
+    "EmptyImageFromAudio":          EmptyImageFromAudio,
 
     #features   
     "AudioFeatureExtractor":        AudioFeatureExtractor,
+    "PitchFeatureExtractor":        PitchFeatureExtractor,
+    "PitchRange":                   PitchRangeNode,
+    "PitchRangePreset":             PitchRangePresetNode,
+    "PitchRangeByNoteNode":         PitchRangeByNoteNode,
     "MIDILoadAndExtract":           MIDILoadAndExtract,
     "TimeFeatureNode":              TimeFeatureNode,
     "DepthFeatureNode":             DepthFeatureNode,
@@ -252,7 +274,6 @@ NODE_CLASS_MAPPINGS = {
     "MotionFeatureNode":            MotionFeatureNode,
     "LocationFromMask":             LocationFromMask,
     "ProximityFeatureNode":         ProximityFeatureNode,
-    "ProximityVisualizer":          ProximityVisualizer,
     "LocationFromPoint":            LocationFromPoint,
     "LocationTransform":            LocationTransform,
     "AreaFeatureNode":              AreaFeatureNode,
@@ -270,6 +291,8 @@ NODE_CLASS_MAPPINGS = {
     "FeatureMath":                  FeatureMath,
     "PreviewFeature":               PreviewFeature,
     "FeatureRebase":                FeatureRebase,
+    "FeatureTruncateOrExtend":      FeatureTruncateOrExtend,
+
     #images
     'FlexImageEdgeDetect':          FlexImageEdgeDetect,
     "FlexImagePosterize":           FlexImagePosterize,
@@ -285,6 +308,10 @@ NODE_CLASS_MAPPINGS = {
     # "FlexDepthBasedMaskOpacity":  FlexDepthBasedMaskOpacity,
     # "DepthBasedMaskOpacity":      DepthBasedMaskOpacity,
 
+    #visulizers
+    "ProximityVisualizer":          ProximityVisualizer,
+    "EffectVisualizer":             EffectVisualizer,
+    "PitchVisualizer":              PitchVisualizer,
 
     #garb   
     "DyeImage":                     DyeImage,
@@ -328,18 +355,17 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "ParticleSizeModulation": "Particle Size Modulation",
     "ParticleSpeedModulation":"Particle Speed Modulation",
 
-    "AudioMaskMorph":"Audio Mask Morph",
-    "AudioMaskWarp": "Audio Mask Warp",
-    "AudioMaskTransform":"Audio Mask Transform",
-    "AudioMaskMath": "Audio Mask Math",
+  
     "AudioSeparator": "Audio Separator",
 
     "AudioFeatureVisualizer": "Audio Feature Visualizer" ,
     "Frequency Filter Custom": "Frequency Filter Custom",
     "Frequency Filter Preset": "Frequency Filter Preset",
     "AudioFilter": "Audio Filter",
-    
+  
+
     "MIDILoadAndExtract":   "MIDI Load & Feature Extract",
+    "PitchRangeByNoteNode": "Pitch Range By Note",
     "AudioFeatureExtractor": "Audio Feature & Extractor",
     "TimeFeatureNode":          "Time Feature",
     "DepthFeatureNode":"Depth Feature",

nodes/audio/audio_nodes.py

@@ -14,6 +14,11 @@ from ... import RyanOnTheInside
 
 class AudioNodeBase(RyanOnTheInside):
     CATEGORY= "RyanOnTheInside/Audio"
+    @staticmethod
+    def create_empty_tensor(audio, frame_rate, height, width, channels):
+        audio_duration = audio['waveform'].shape[-1] / audio['sample_rate']
+        num_frames = int(audio_duration * frame_rate)
+        return torch.zeros((num_frames, height, width, channels), dtype=torch.float32)
 
 class DownloadOpenUnmixModel(AudioNodeBase):
     @classmethod
@@ -167,6 +172,44 @@ class AudioFilter(AudioNodeBase):
 
         return signal.lfilter(b, a, audio)
 
+# class DownloadCREPEModel(AudioNodeBase):
+#     @classmethod
+#     def INPUT_TYPES(cls):
+#         return {
+#             "required": {
+#                 "model_name": (["tiny", "small", "medium", "large", "full"], {"default": "medium"}),
+#             }
+#         }
+
+#     RETURN_TYPES = ("CREPE_MODEL",)
+#     FUNCTION = "download_and_load_model"
+#     CATEGORY = "RyanOnTheInside/Audio"
+
+#     def download_and_load_model(self, model_name):
+#         try:
+#             import crepe
+#         except ImportError:
+#             raise ImportError("""To use this node please 
+#                               pip install crepe tensorflow
+#                               :)
+#                               """)
+
+#         download_path = os.path.join(folder_paths.models_dir, "crepe")
+#         os.makedirs(download_path, exist_ok=True)
+
+#         model_file = f"crepe_{model_name}.json"
+#         model_path = os.path.join(download_path, model_file)
+
+#         if not os.path.exists(model_path):
+#             print(f"Downloading CREPE {model_name} model...")
+#             model = crepe.core.build_and_load_model(model_name)
+#             print(f"Model downloaded and loaded.")
+#         else:
+#             print(f"Loading model from: {model_path}")
+#             model = crepe.core.build_and_load_model(model_name)
+
+#         return (model,)
+
 class FrequencyFilterPreset(AudioNodeBase):
     @classmethod
     def INPUT_TYPES(cls):
@@ -318,4 +361,42 @@ class AudioFeatureVisualizer(AudioNodeBase):
             raise ValueError(f"Unsupported visualization type: {visualization_type}")
 
 
-        return (mask,)
+        return (mask,)   
+
+class EmptyImageFromAudio(AudioNodeBase):
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "audio": ("AUDIO",),
+                "frame_rate": ("FLOAT", {"default": 30, "min": 0.1, "max": 120, "step": 0.1}),
+                "height": ("INT", {"default": 512, "min": 16, "max": 4096, "step": 1}),
+                "width": ("INT", {"default": 512, "min": 16, "max": 4096, "step": 1}),
+            }
+        }
+
+    RETURN_TYPES = ("IMAGE",)
+    FUNCTION = "create_empty_image"
+
+    def create_empty_image(self, audio, frame_rate, height, width):
+        empty_image = self.create_empty_tensor(audio, frame_rate, height, width, channels=3)
+        return (empty_image,)
+
+class EmptyMaskFromAudio(AudioNodeBase):
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "audio": ("AUDIO",),
+                "frame_rate": ("FLOAT", {"default": 30, "min": 0.1, "max": 120, "step": 0.1}),
+                "height": ("INT", {"default": 512, "min": 16, "max": 4096, "step": 1}),
+                "width": ("INT", {"default": 512, "min": 16, "max": 4096, "step": 1}),
+            }
+        }
+
+    RETURN_TYPES = ("MASK",)
+    FUNCTION = "create_empty_mask"
+
+    def create_empty_mask(self, audio, frame_rate, height, width):
+        empty_mask = self.create_empty_tensor(audio, frame_rate, height, width, channels=1)
+        return (empty_mask,)

nodes/flex/feature_externals.py

@@ -2,7 +2,7 @@ import torch
 import torch.nn.functional as F
 import numpy  as np
 from ... import RyanOnTheInside
-from ... import RyanOnTheInside
+import cv2
 
 class FlexExternalModulator(RyanOnTheInside):
     CATEGORY = "RyanOnTheInside/FlexExternalMod"
@@ -42,11 +42,6 @@ class FeatureToWeightsStrategy(FlexExternalModulator):
         return (weights_strategy,)
 
 
-
-import torch
-import torch.nn.functional as F
-import cv2
-
 class DepthShapeModifier(FlexExternalModulator):
     @classmethod
     def INPUT_TYPES(cls):

nodes/flex/feature_extractors.py

@@ -1,11 +1,26 @@
 from .feature_pipe import FeaturePipe
 from ... import RyanOnTheInside
-from .features import AudioFeature, TimeFeature, DepthFeature, ColorFeature, BrightnessFeature, MotionFeature, AreaFeature
-from .proximity_feature  import  Location
+from .features import TimeFeature, DepthFeature, ColorFeature, BrightnessFeature, MotionFeature, AreaFeature, BaseFeature
+from abc import ABC, abstractmethod
 from tqdm import tqdm
 from comfy.utils import ProgressBar
+import typing
+
+class FeatureExtractorBase(RyanOnTheInside, ABC):
+    @classmethod
+    @abstractmethod
+    def feature_type(cls) -> type[BaseFeature]:
+        pass
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        feature_class = cls.feature_type()
+        return {            
+            "required": {
+                "extraction_method": (feature_class.get_extraction_methods(), {"default": feature_class.get_extraction_methods()[0]}),
+            }
+        }
 
-class FeatureExtractorBase(RyanOnTheInside):
     def __init__(self):
         self.progress_bar = None
         self.tqdm_bar = None
@@ -34,31 +49,19 @@ class FeatureExtractorBase(RyanOnTheInside):
         self.total_steps = 0
     CATEGORY="RyanOnTheInside/FlexFeatures"
 
-class AudioFeatureExtractor(FeatureExtractorBase):
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "audio": ("AUDIO",),
-                "feature_pipe": ("FEATURE_PIPE",),
-                "feature_type": (["amplitude_envelope",  "spectral_centroid", "onset_detection", "chroma_features"],),
-            }
-        }
-
-    RETURN_TYPES = ("FEATURE", "FEATURE_PIPE")
-    FUNCTION = "extract_feature"
-    
-
-    def extract_feature(self, audio, feature_pipe, feature_type):
-        feature = AudioFeature(feature_type, audio, feature_pipe.frame_count, feature_pipe.frame_rate, feature_type)
-        feature.extract()
-        return (feature, feature_pipe)
-
+#    
 class FirstFeature(FeatureExtractorBase):
+
+    @classmethod
+    def feature_type(cls) -> type[BaseFeature]:
+        pass
+
     @classmethod
     def INPUT_TYPES(cls):
-        return {
+        return {            
+            **super().INPUT_TYPES(),
             "required": {
+                **super().INPUT_TYPES()["required"],
                 "video_frames": ("IMAGE",),
                 "frame_rate": ("FLOAT", {"default": 30.0, "min": 1.0, "max": 120.0, "step": 0.1}),
             }
@@ -68,13 +71,16 @@ class FirstFeature(FeatureExtractorBase):
     FUNCTION = "create_feature"
    
 class TimeFeatureNode(FirstFeature):
+    @classmethod
+    def feature_type(cls) -> type[BaseFeature]:
+        return TimeFeature
+
     @classmethod
     def INPUT_TYPES(cls):
         return {
             **super().INPUT_TYPES(),
             "required": {
                 **super().INPUT_TYPES()["required"],
-                "effect_type": (["smooth", "accelerate", "pulse", "sawtooth", "bounce"],),
                 "speed": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 10.0, "step": 0.1}),
                 "offset": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
             }
@@ -82,14 +88,18 @@ class TimeFeatureNode(FirstFeature):
 
    
 
-    def create_feature(self, effect_type, speed, offset, video_frames, frame_rate):
+    def create_feature(self, extraction_method, speed, offset, video_frames, frame_rate):
         feature_pipe = FeaturePipe(frame_rate, video_frames)
         time_feature = TimeFeature("time_effect", feature_pipe.frame_rate, feature_pipe.frame_count, 
-                                   effect_type=effect_type, speed=speed, offset=offset)
+                                   effect_type=extraction_method, speed=speed, offset=offset)
         time_feature.extract()
         return (time_feature, feature_pipe)
     
 class DepthFeatureNode(FirstFeature):
+    @classmethod
+    def feature_type(cls) -> type[BaseFeature]:
+        return DepthFeature
+
     @classmethod
     def INPUT_TYPES(cls):
         return {
@@ -97,58 +107,66 @@ class DepthFeatureNode(FirstFeature):
             "required": {
                 **super().INPUT_TYPES()["required"],
                 "depth_maps": ("IMAGE",),
-                "feature_type": (["mean_depth", "depth_variance", "depth_range", "gradient_magnitude", "foreground_ratio", "midground_ratio", "background_ratio"],),
             }
         }
 
-    def create_feature(self, depth_maps, frame_rate, video_frames, feature_type):
+    def create_feature(self, depth_maps, frame_rate, video_frames, extraction_method):
         feature_pipe = FeaturePipe(frame_rate, video_frames)
-        depth_feature = DepthFeature("depth_feature", feature_pipe.frame_rate, feature_pipe.frame_count, depth_maps, feature_type)
+        depth_feature = DepthFeature("depth_feature", feature_pipe.frame_rate, feature_pipe.frame_count, depth_maps, extraction_method)
         depth_feature.extract()
         return (depth_feature, feature_pipe)
 
 class ColorFeatureNode(FirstFeature):
+    @classmethod
+    def feature_type(cls) -> type[BaseFeature]:
+        return ColorFeature
+
     @classmethod
     def INPUT_TYPES(cls):
         return {
             **super().INPUT_TYPES(),
             "required": {
                 **super().INPUT_TYPES()["required"],
-                "feature_type": (["dominant_color", "color_variance", "saturation", "red_ratio", "green_ratio", "blue_ratio"],),
             }
         }
 
-    def create_feature(self, video_frames, frame_rate, feature_type):
+    def create_feature(self, video_frames, frame_rate, extraction_method):
         feature_pipe = FeaturePipe(frame_rate, video_frames)
-        color_feature = ColorFeature("color_feature", feature_pipe.frame_rate, feature_pipe.frame_count, video_frames, feature_type)
+        color_feature = ColorFeature("color_feature", feature_pipe.frame_rate, feature_pipe.frame_count, video_frames, extraction_method)
         color_feature.extract()
         return (color_feature, feature_pipe)
 
 class BrightnessFeatureNode(FirstFeature):
     @classmethod
+    def feature_type(cls) -> type[BaseFeature]:
+        return BrightnessFeature
+
+    @classmethod
     def INPUT_TYPES(cls):
         return {
             **super().INPUT_TYPES(),
             "required": {
                 **super().INPUT_TYPES()["required"],
-                "feature_type": (["mean_brightness", "brightness_variance", "dark_ratio", "mid_ratio", "bright_ratio"],),
             }
         }
+
+    def create_feature(self, video_frames, frame_rate, extraction_method):
+        feature_pipe = FeaturePipe(frame_rate, video_frames)
+        brightness_feature = BrightnessFeature("brightness_feature", feature_pipe.frame_rate, feature_pipe.frame_count, video_frames, extraction_method)
+        brightness_feature.extract()
+        return (brightness_feature, feature_pipe)
+    
+class MotionFeatureNode(FirstFeature):
+    @classmethod
+    def feature_type(cls) -> type[BaseFeature]:
+        return MotionFeature
 
-    def create_feature(self, video_frames, frame_rate, feature_type):
-        feature_pipe = FeaturePipe(frame_rate, video_frames)
-        brightness_feature = BrightnessFeature("brightness_feature", feature_pipe.frame_rate, feature_pipe.frame_count, video_frames, feature_type)
-        brightness_feature.extract()
-        return (brightness_feature, feature_pipe)
-    
-class MotionFeatureNode(FirstFeature):
     @classmethod
     def INPUT_TYPES(cls):
         return {
             **super().INPUT_TYPES(),
             "required": {
                 **super().INPUT_TYPES()["required"],
-                "feature_type": (["mean_motion", "max_motion", "motion_direction", "horizontal_motion", "vertical_motion", "motion_complexity","motion_speed"],),
                 "flow_method": (["Farneback", "LucasKanade", "PyramidalLK"],),
                 "flow_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 10.0, "step": 0.1}),
                 "magnitude_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
@@ -158,7 +176,7 @@ class MotionFeatureNode(FirstFeature):
     RETURN_TYPES = ("FEATURE", "FEATURE_PIPE")
     FUNCTION = "create_feature"
 
-    def create_feature(self, video_frames, frame_rate, feature_type, flow_method, flow_threshold, magnitude_threshold):
+    def create_feature(self, video_frames, frame_rate, extraction_method, flow_method, flow_threshold, magnitude_threshold):
         feature_pipe = FeaturePipe(frame_rate, video_frames)
         num_frames = feature_pipe.frame_count
 
@@ -172,7 +190,7 @@ class MotionFeatureNode(FirstFeature):
             feature_pipe.frame_rate,
             feature_pipe.frame_count,
             video_frames,
-            feature_type,
+            extraction_method,
             flow_method,
             flow_threshold,
             magnitude_threshold,
@@ -186,6 +204,10 @@ class MotionFeatureNode(FirstFeature):
     
 
 class AreaFeatureNode(FirstFeature):
+    @classmethod
+    def feature_type(cls) -> type[BaseFeature]:
+        return AreaFeature
+
     @classmethod
     def INPUT_TYPES(cls):
         return {
@@ -193,7 +215,6 @@ class AreaFeatureNode(FirstFeature):
             "required": {
                 **super().INPUT_TYPES()["required"],
                 "masks": ("MASK",),
-                "feature_type": (["total_area", "largest_contour", "bounding_box"],),
                 "threshold": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}),
             }
         }
@@ -201,9 +222,9 @@ class AreaFeatureNode(FirstFeature):
     RETURN_TYPES = ("FEATURE", "FEATURE_PIPE")
     FUNCTION = "create_feature"
 
-    def create_feature(self, masks, video_frames, frame_rate, feature_type, threshold):
+    def create_feature(self, masks, video_frames, frame_rate, extraction_method, threshold):
         feature_pipe = FeaturePipe(frame_rate, video_frames)
         area_feature = AreaFeature("area_feature", feature_pipe.frame_rate, feature_pipe.frame_count, 
-                                   masks, feature_type=feature_type, threshold=threshold)
+                                   masks, feature_type=extraction_method, threshold=threshold)
         area_feature.extract()
         return (area_feature, feature_pipe)

nodes/flex/feature_extractors_audio.py

@@ -0,0 +1,208 @@
+from .feature_extractors import FeatureExtractorBase
+from .features_audio import AudioFeature, PitchFeature, PitchRange, BaseFeature
+from ... import RyanOnTheInside
+
+class AudioFeatureExtractor(FeatureExtractorBase):
+    @classmethod
+    def feature_type(cls) -> type[BaseFeature]:
+        return AudioFeature
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            **super().INPUT_TYPES(),
+            "required": {
+                **super().INPUT_TYPES()["required"],
+                "audio": ("AUDIO",),
+                "feature_pipe": ("FEATURE_PIPE",),
+            }
+        }
+
+    RETURN_TYPES = ("FEATURE", "FEATURE_PIPE")
+    FUNCTION = "extract_feature"
+    
+
+    def extract_feature(self, audio, feature_pipe, extraction_method):
+        feature = AudioFeature(extraction_method, audio, feature_pipe.frame_count, feature_pipe.frame_rate, extraction_method)
+        feature.extract()
+        return (feature, feature_pipe)
+
+class PitchFeatureExtractor(FeatureExtractorBase):
+    @classmethod
+    def feature_type(cls) -> type[BaseFeature]:
+        return  PitchFeature
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {            
+            **super().INPUT_TYPES(),
+            "required": {
+                **super().INPUT_TYPES()["required"],
+                "audio": ("AUDIO",),
+                "feature_pipe": ("FEATURE_PIPE",),
+                "opt_crepe_model":(["none", "medium", "tiny", "small", "large", "full"], {"default": "medium"})
+            },
+            "optional": {
+                "opt_pitch_range_collections": ("PITCH_RANGE_COLLECTION",),
+                # "": ("CREPE_MODEL",),
+            },
+        }
+
+    RETURN_TYPES = ("FEATURE", "FEATURE_PIPE")
+    FUNCTION = "extract_feature"
+
+    def extract_feature(self, audio, feature_pipe, extraction_method, opt_pitch_range_collections=None, opt_crepe_model=None):
+        if opt_pitch_range_collections is None:
+            opt_pitch_range_collections = []
+        feature = PitchFeature(
+            feature_name=extraction_method,
+            audio=audio,
+            frame_count=feature_pipe.frame_count,
+            frame_rate=feature_pipe.frame_rate,
+            pitch_range_collections=opt_pitch_range_collections,
+            feature_type=extraction_method,
+            crepe_model=opt_crepe_model
+        )
+        feature.extract()
+        return (feature, feature_pipe)
+
+class PitchAbstraction(RyanOnTheInside):
+    CATEGORY="RyanOnTheInside/FlexFeatures/Audio"
+
+class PitchRangeNode(PitchAbstraction):
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "min_pitch": ("FLOAT", {"default": 80.0, "min": 20.0, "max": 2000.0, "step": 1.0}),
+                "max_pitch": ("FLOAT", {"default": 400.0, "min": 20.0, "max": 2000.0, "step": 1.0}),
+            },
+            "optional": {
+                "previous_range_collection": ("PITCH_RANGE_COLLECTION",),
+            },
+        }
+
+    RETURN_TYPES = ("PITCH_RANGE_COLLECTION",)
+    FUNCTION = "create_pitch_range"
+
+    CATEGORY = "RyanOnTheInside/FlexFeatures"
+
+    def create_pitch_range(self, min_pitch, max_pitch, previous_range_collection=None):
+        pitch_range = PitchRange(min_pitch, max_pitch)
+        pitch_range_collection = {
+            "pitch_ranges": [pitch_range],
+            "chord_only": False,
+        }
+        if previous_range_collection is None:
+            collections = [pitch_range_collection]
+        else:
+            collections = previous_range_collection + [pitch_range_collection]
+        return (collections,)
+    
+class PitchRangePresetNode(PitchAbstraction):
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "preset": (
+                    [
+                        "Bass",
+                        "Baritone",
+                        "Tenor",
+                        "Alto",
+                        "Mezzo-soprano",
+                        "Soprano",
+                        "Contralto",
+                    ],
+                )
+            },
+            "optional": {
+                "previous_range_collection": ("PITCH_RANGE_COLLECTION",),
+            },
+        }
+
+    RETURN_TYPES = ("PITCH_RANGE_COLLECTION",)
+    FUNCTION = "create_pitch_range_preset"
+
+    CATEGORY = "RyanOnTheInside/FlexFeatures"
+
+    def create_pitch_range_preset(self, preset, previous_range_collection=None):
+        presets = {
+            "Bass": (82.41, 196.00),            # E2 - G3
+            "Baritone": (98.00, 247.94),        # G2 - B3
+            "Tenor": (130.81, 349.23),          # C3 - F4
+            "Contralto": (130.81, 349.23),      # C3 - F4
+            "Alto": (174.61, 440.00),           # F3 - A4
+            "Mezzo-soprano": (196.00, 523.25),  # G3 - C5
+            "Soprano": (261.63, 1046.50),       # C4 - C6
+        }
+
+        min_pitch, max_pitch = presets.get(preset, (20.0, 2000.0))
+        pitch_range = PitchRange(min_pitch, max_pitch)
+        pitch_range_collection = {
+            "pitch_ranges": [pitch_range],
+            "chord_only": False,
+        }
+        if previous_range_collection is None:
+            collections = [pitch_range_collection]
+        else:
+            collections = previous_range_collection + [pitch_range_collection]
+        return (collections,)
+    
+class PitchRangeByNoteNode(PitchAbstraction):
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "chord_only": ("BOOLEAN", {"default": False}),
+                "pitch_tolerance_percent": ("FLOAT", {"default": 100.0, "min": 0.0, "max": 100.0, "step": 0.1}),
+                "notes": ("STRING", {"multiline": False}),
+            },
+            "optional": {
+                "previous_range_collection": ("PITCH_RANGE_COLLECTION",),
+            },
+        }
+
+    RETURN_TYPES = ("PITCH_RANGE_COLLECTION",)
+    FUNCTION = "create_note_pitch_ranges"
+
+    CATEGORY = "RyanOnTheInside/FlexFeatures"
+
+    def create_note_pitch_ranges(self, chord_only, notes, pitch_tolerance_percent, previous_range_collection=None):
+        if not notes:
+            raise ValueError("At least one note must be selected.")
+
+        # Parse the 'notes' string into a list of MIDI note numbers
+        selected_notes = [int(note.strip()) for note in notes.split(',') if note.strip().isdigit()]
+
+        if not selected_notes:
+            raise ValueError("No valid notes found in the 'notes' field.")
+
+        pitch_ranges = []
+        for midi_note in selected_notes:
+            frequency = self._midi_to_frequency(midi_note)
+            tolerance = PitchFeature.calculate_tolerance(frequency, pitch_tolerance_percent)
+            min_pitch = frequency - tolerance
+            max_pitch = frequency + tolerance
+            pitch_range = PitchRange(min_pitch, max_pitch)
+            pitch_ranges.append(pitch_range)
+
+        # Create a collection with the 'chord_only' attribute
+        pitch_range_collection = {
+            "pitch_ranges": pitch_ranges,
+            "chord_only": chord_only,
+        }
+
+        # Combine with previous collections if provided
+        if previous_range_collection is None:
+            collections = [pitch_range_collection]
+        else:
+            collections = previous_range_collection + [pitch_range_collection]
+
+        return (collections,)
+
+    def _midi_to_frequency(self, midi_note):
+        import librosa
+        return librosa.midi_to_hz(midi_note)
+
+    # The _calculate_tolerance method has been removed from here

nodes/flex/feature_extractors_midi.py

@@ -0,0 +1,163 @@
+import mido
+import os
+from .feature_pipe import FeaturePipe
+import folder_paths
+from server import PromptServer
+from aiohttp import web
+import shutil
+from .midi_feature import MIDIFeature
+from .feature_extractors import FeatureExtractorBase
+
+class MIDILoadAndExtract(FeatureExtractorBase):
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "midi_file": (folder_paths.get_filename_list("midi_files"),),
+                "track_selection": (["all"],),
+                "attribute": (MIDIFeature.get_attribute_names(), {"default": "Note On/Off"}),
+                "frame_rate": ("FLOAT", {"default": 30, "min": 0.1, "max": 120, "step": 0.1}),
+                "video_frames": ("IMAGE",),
+                "chord_only": ("BOOLEAN", {"default": False}),
+                "notes":  ("STRING", {"default": ""}),
+            },
+        }
+
+    RETURN_TYPES = ("MIDI", "FEATURE", "FEATURE_PIPE")
+    FUNCTION = "process_midi"
+    CATEGORY = "RyanOnTheInside/Audio"
+
+    def process_midi(self, midi_file, track_selection, notes, attribute, frame_rate, video_frames, chord_only=False):
+        try:
+            midi_path = folder_paths.get_full_path("midi_files", midi_file)
+            if not midi_path or not os.path.exists(midi_path):
+                raise FileNotFoundError(f"MIDI file not found: {midi_file}")
+
+            midi_data = mido.MidiFile(midi_path)
+            
+            selected_notes = [int(n.strip()) for n in notes.split(',') if n.strip().isdigit()]
+            feature_pipe = FeaturePipe(frame_rate, video_frames)
+            
+            # Convert friendly attribute name to internal attribute name
+            internal_attribute = MIDIFeature.get_attribute_value(attribute)
+            
+            feature = MIDIFeature(f"midi_{internal_attribute}", midi_data, internal_attribute, 
+                                  feature_pipe.frame_rate, feature_pipe.frame_count, 
+                                  notes=selected_notes, chord_only=chord_only)
+            
+            feature.extract()
+
+            return (midi_data, feature, feature_pipe)
+
+        except Exception as e:
+            # error_msg = f"Error in MIDILoadAndExtract.process_midi: {type(e).__name__}: {str(e)}\n"
+            # error_msg += traceback.format_exc()
+            # print(error_msg)
+            raise RuntimeError(f"Error processing MIDI file: {type(e).__name__}: {str(e)}")
+
+    @classmethod
+    def analyze_midi(cls, midi_path):
+        midi_data = mido.MidiFile(midi_path)
+        tracks = ["all"]
+        all_notes = set()
+        track_notes = {}
+        for i, track in enumerate(midi_data.tracks):
+            track_notes[str(i)] = set()
+            for msg in track:
+                if msg.type == 'note_on':
+                    track_notes[str(i)].add(msg.note)
+                    all_notes.add(msg.note)
+            if len(track_notes[str(i)]) == 0:
+                tracks.append(f"{i}: (Empty)")
+            else:
+                tracks.append(f"{i}: {track.name or f'Track {i}'}")
+        
+        return {
+            "tracks": tracks,
+            "all_notes": ",".join(map(str, sorted(set(all_notes)))),
+            "track_notes": {k: ",".join(map(str, sorted(v))) for k, v in track_notes.items()}
+        }
+    
+    @classmethod
+    def VALIDATE_INPUTS(cls, midi_file, track_selection, notes, attribute, frame_rate, video_frames):
+        if not folder_paths.exists_and_is_file(midi_file):
+            return "MIDI file not found: {}".format(midi_file)
+        
+        if notes != "all":
+            try:
+                note_list = [int(n.strip()) for n in notes.split(',') if n.strip()]
+                if not all(0 <= n <= 127 for n in note_list):
+                    return "Invalid note value. All notes must be between 0 and 127."
+            except ValueError:
+                return "Invalid notes format. Please provide comma-separated integers or 'all'."
+        
+        return True
+
+
+routes = PromptServer.instance.routes
+@PromptServer.instance.routes.post('/get_track_notes')
+async def get_track_notes(request):
+    data = await request.json()
+    midi_file = data.get('midi_file')
+
+    if not midi_file:
+        return web.json_response({"error": "Missing required parameters"}, status=400)
+
+    midi_path = folder_paths.get_full_path("midi_files", midi_file)
+    if not midi_path or not os.path.exists(midi_path):
+        return web.json_response({"error": "MIDI file not found"}, status=404)
+
+    analysis = MIDILoadAndExtract.analyze_midi(midi_path)
+    return web.json_response(analysis)
+
+@routes.post('/upload_midi')
+async def upload_midi(request):
+    data = await request.post()
+    midi_file = data['file']
+    
+    if midi_file and midi_file.filename:
+        safe_filename = os.path.basename(midi_file.filename)
+        
+        midi_dir = folder_paths.get_folder_paths("midi_files")[0]
+        
+        if not os.path.exists(midi_dir):
+            os.makedirs(midi_dir, exist_ok=True)
+        
+        midi_path = os.path.join(midi_dir, safe_filename)
+
+        with open(midi_path, 'wb') as f:
+            shutil.copyfileobj(midi_file.file, f)
+
+        midi_files = folder_paths.get_filename_list("midi_files")
+        analysis = MIDILoadAndExtract.analyze_midi(midi_path)
+
+        return web.json_response({
+            "status": "success",
+            "uploaded_file": safe_filename,
+            "midi_files": midi_files,
+            "analysis": analysis
+        })
+    else:
+        return web.json_response({"status": "error", "message": "No file uploaded"}, status=400)
+    
+@PromptServer.instance.routes.post('/refresh_midi_data')
+async def refresh_midi_data(request):
+    data = await request.json()
+    midi_file = data.get('midi_file')
+    track_selection = data.get('track_selection')
+
+    if not midi_file:
+        return web.json_response({"error": "Missing required parameters"}, status=400)
+
+    midi_path = folder_paths.get_full_path("midi_files", midi_file)
+    if not midi_path or not os.path.exists(midi_path):
+        return web.json_response({"error": "MIDI file not found"}, status=404)
+
+    analysis = MIDILoadAndExtract.analyze_midi(midi_path)
+    
+    # Filter notes based on track selection
+    if track_selection != "all":
+        track_index = track_selection.split(':')[0]
+        analysis['all_notes'] = analysis['track_notes'].get(track_index, "")
+
+    return web.json_response(analysis)

nodes/flex/feature_extractors_proximity.py

@@ -0,0 +1,186 @@
+from .features_proximity import Location,   ProximityFeature
+from .feature_extractors import FirstFeature
+from ... import RyanOnTheInside
+from .feature_pipe import FeaturePipe
+import numpy as np
+import cv2
+
+
+class ProximityFeatureNode(FirstFeature):
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "video_frames": ("IMAGE",),
+                "frame_rate": ("FLOAT", {"default": 30.0, "min": 1.0, "max": 120.0, "step": 0.1}),
+                "anchor_locations": ("LOCATION",),
+                "query_locations": ("LOCATION",),
+                "normalization_method": (["frame", "minmax"],),
+            }
+        }
+
+    RETURN_TYPES = ("FEATURE", "FEATURE_PIPE")
+    RETURN_NAMES = ("proximity_feature", "feature_pipe")
+    FUNCTION = "create_feature"
+
+    def create_feature(self, video_frames, frame_rate, anchor_locations, query_locations, normalization_method):
+        feature_pipe = FeaturePipe(frame_rate, video_frames)
+        frame_dimensions = (video_frames.shape[2], video_frames.shape[1])  # width, height
+        
+        proximity_feature = ProximityFeature(
+            name="proximity_feature",
+            anchor_locations=anchor_locations,
+            query_locations=query_locations,
+            frame_rate=frame_rate,
+            frame_count=len(video_frames),
+            frame_dimensions=frame_dimensions,
+            normalization_method=normalization_method
+        )
+
+        # Ensure anchor and query points have the same number of columns
+        anchor_dim = anchor_locations[0].points.shape[1]
+        query_dim = query_locations[0].points.shape[1]
+
+        if anchor_dim != query_dim:
+            if anchor_dim == 3:
+                for location in anchor_locations:
+                    location.points = location.points[:, :2]
+            elif query_dim == 3:
+                for location in query_locations:
+                    location.points = location.points[:, :2]
+            print("Depth ignored. Depth must be included for all points or no points.")
+
+        # self.start_progress(len(video_frames), desc="Calculating proximity")
+        proximity_feature.extract()
+        # self.end_progress()
+
+        return (proximity_feature, feature_pipe)
+
+class ProximityFeatureInput(RyanOnTheInside):
+    CATEGORY="RyanOnTheInside/Proximity"
+
+class LocationFromMask(ProximityFeatureInput):
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "masks": ("MASK",),
+                "method": (["mask_boundary", "mask_center"],),
+            },
+            "optional": {
+                "depth_maps": ("IMAGE",),
+            }
+        }
+
+    RETURN_TYPES = ("LOCATION",)
+    FUNCTION = "generate_locations"
+
+    def generate_locations(self, masks, method, depth_maps=None):
+        locations = []
+        for i in range(masks.shape[0]):
+            mask = masks[i].cpu().numpy()
+            if np.sum(mask) == 0:  # Check if the mask is empty
+                print(f"No mask found in frame {i}.")
+                locations.append(Location(np.array([-1]), np.array([-1]), np.array([-1]) if depth_maps is not None else None))
+                continue
+
+            if method == "mask_boundary":
+                contours, _ = cv2.findContours(mask.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+                if contours:
+                    boundary = np.vstack(contours).squeeze()
+                else:
+                    boundary = np.array([[0, 0]])  # Fallback to a single point
+            elif method == "mask_center":
+                y, x = np.where(mask > 0.5)
+                if len(x) > 0 and len(y) > 0:
+                    center_x, center_y = np.mean(x), np.mean(y)
+                    boundary = np.array([[center_x, center_y]])
+                else:
+                    boundary = np.array([[mask.shape[1] / 2, mask.shape[0] / 2]])
+
+            if depth_maps is not None:
+                depth_map = depth_maps[i].cpu().numpy()
+                # Ensure boundary points are within the depth map dimensions
+                valid_points = (boundary[:, 0] < depth_map.shape[1]) & (boundary[:, 1] < depth_map.shape[0])
+                boundary = boundary[valid_points]
+                
+                if boundary.size == 0:  # Check if boundary is empty after filtering
+                    print(f"No valid boundary points in frame {i}.")
+                    locations.append(Location(np.array([-1]), np.array([-1]), np.array([-1])))
+                    continue
+
+                # Extract depth values for valid boundary points
+                z = np.mean(depth_map[boundary[:, 1].astype(int), boundary[:, 0].astype(int)], axis=-1)
+                
+                location = Location(boundary[:, 0], boundary[:, 1], z)
+            else:
+                location = Location(boundary[:, 0], boundary[:, 1])
+            
+            locations.append(location)
+
+        return (locations,)
+
+class LocationFromPoint(ProximityFeatureInput):
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "x": ("FLOAT", {"default": 0.0,"min":0.0,"step":0.01}),
+                "y": ("FLOAT", {"default": 0.0,"min":0.0,"step":0.01}),
+                "z": ("FLOAT", {"default": 0.0,  "max":1.0, "min":0.0,"step":0.01}),
+                "batch_count": ("INT", {"default": 1, "min": 1}),
+            }
+        }
+
+    RETURN_TYPES = ("LOCATION",)
+    FUNCTION = "generate_locations"
+
+    def generate_locations(self, x, y, batch_count, z):
+        locations = []
+        for _ in range(batch_count):
+            location = Location(np.array([x]), np.array([y]), np.array([z]))
+            locations.append(location)
+
+        return (locations,)
+
+class LocationTransform(ProximityFeatureInput):
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "locations": ("LOCATION",),
+                "feature": ("FEATURE",),
+                "transformation_type": (["translate", "scale"],),
+                "transformation_value": ("FLOAT", {"default": 1.0}),
+            }
+        }
+
+    RETURN_TYPES = ("LOCATION",)
+    FUNCTION = "transform_locations"
+
+    def transform_locations(self, locations, feature, transformation_type, transformation_value):
+        transformed_locations = []
+        for frame_index, location in enumerate(locations):
+            feature_value = feature.get_value_at_frame(frame_index)
+            if transformation_type == "translate":
+                new_x = location.x + feature_value * transformation_value
+                new_y = location.y + feature_value * transformation_value
+                if location.z is not None:
+                    new_z = location.z + feature_value * transformation_value
+                    transformed_location = Location(new_x, new_y, new_z)
+                else:
+                    transformed_location = Location(new_x, new_y)
+            elif transformation_type == "scale":
+                new_x = location.x * feature_value * transformation_value
+                new_y = location.y * feature_value * transformation_value
+                if location.z is not None:
+                    new_z = location.z * feature_value * transformation_value
+                    transformed_location = Location(new_x, new_y, new_z)
+                else:
+                    transformed_location = Location(new_x, new_y)
+            transformed_locations.append(transformed_location)
+
+        return (transformed_locations,)
+
+
+

nodes/flex/feature_modulation.py

@@ -4,6 +4,7 @@ import matplotlib.pyplot as plt
 import torch
 from io import BytesIO
 from PIL import Image
+import random
 
 class FeatureModulationBase(RyanOnTheInside):
     CATEGORY = "RyanOnTheInside/FlexFeatures/FeatureModulators"
@@ -477,4 +478,51 @@ class PreviewFeature(FeatureModulationBase):
         else:
             processed_feature = feature
         
-        return (self.visualize(processed_feature),)
+        return (self.visualize(processed_feature),)
+
+class FeatureTruncateOrExtend(FeatureModulationBase):
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "feature": ("FEATURE",),
+                "target_feature_pipe": ("FEATURE_PIPE",),
+                "fill_method": (["zeros", "ones", "average", "random", "repeat"],),
+                **super().INPUT_TYPES()["required"],
+            }
+        }
+
+    RETURN_TYPES = ("FEATURE", "IMAGE")
+    RETURN_NAMES = ("FEATURE", "FEATURE_VISUALIZATION")
+    FUNCTION = "truncate_or_extend"
+
+    def truncate_or_extend(self, feature, target_feature_pipe, fill_method, invert_output):
+        source_values = [feature.get_value_at_frame(i) for i in range(feature.frame_count)]
+        target_length = target_feature_pipe.frame_count
+
+        if len(source_values) > target_length:
+            # Truncate
+            adjusted_values = source_values[:target_length]
+        elif len(source_values) < target_length:
+            # Extend
+            adjusted_values = source_values.copy()
+            extension_length = target_length - len(source_values)
+            
+            if fill_method == "zeros":
+                adjusted_values.extend([0] * extension_length)
+            elif fill_method == "ones":
+                adjusted_values.extend([1] * extension_length)
+            elif fill_method == "average":
+                avg_value = np.mean(source_values)
+                adjusted_values.extend([avg_value] * extension_length)
+            elif fill_method == "random":
+                adjusted_values.extend([random.random() for _ in range(extension_length)])
+            elif fill_method == "repeat":
+                while len(adjusted_values) < target_length:
+                    adjusted_values.extend(source_values[:min(len(source_values), target_length - len(adjusted_values))])
+        else:
+            # Same length, no adjustment needed
+            adjusted_values = source_values
+
+        processed_feature = self.create_processed_feature(feature, adjusted_values, "TruncatedOrExtended", invert_output)
+        return (processed_feature, self.visualize(processed_feature))

nodes/flex/features.py

@@ -1,6 +1,5 @@
 from abc import ABC, abstractmethod
 import numpy as np
-import librosa
 import torch
 import cv2
 from ..masks.mask_utils import calculate_optical_flow
@@ -19,6 +18,12 @@ class BaseFeature(ABC):
     def extract(self):
         pass
 
+    @classmethod
+    @abstractmethod
+    def get_extraction_methods(cls):
+        """Return a list of parameter names that can be modulated."""
+        return []
+    
     def get_value_at_frame(self, frame_index):
         if self.data is not None:
             return self.data[frame_index]
@@ -26,6 +31,7 @@ class BaseFeature(ABC):
             return self.features[self.feature_name][frame_index]
         else:
             raise ValueError("No data or features available")
+    
 
     def normalize(self):
         if self.data is not None:
@@ -53,6 +59,12 @@ class TimeFeature(BaseFeature):
         self.offset = offset
         super().__init__(name, "time", frame_rate, frame_count)
 
+    @classmethod
+    def get_extraction_methods(self):
+        return [
+            "smooth", "accelerate", "pulse", "sawtooth","bounce"
+        ]
+
     def extract(self):
         t = np.linspace(0, self.frame_count / self.frame_rate, self.frame_count)
         t = (t * self.speed + self.offset) % 1
@@ -72,66 +84,20 @@ class TimeFeature(BaseFeature):
         
         return self.normalize()
 
-class AudioFeature(BaseFeature):
-    def __init__(self, name, audio, num_frames, frame_rate, feature_type='amplitude_envelope'):
-        self.audio = audio['waveform'].squeeze(0).mean(axis=0).cpu().numpy()
-        self.sample_rate = audio['sample_rate']
-        self.num_frames = num_frames
-        self.feature_type = feature_type
-        super().__init__(name, "audio", frame_rate, num_frames)
-        self.frame_duration = 1 / self.frame_rate if self.frame_rate > 0 else len(self.audio) / (self.sample_rate * self.num_frames)
-
-    def extract(self):
-        if self.feature_type == 'amplitude_envelope':
-            self.data = self._amplitude_envelope()
-        elif self.feature_type == 'rms_energy':
-            self.data = self._rms_energy()
-        elif self.feature_type == 'spectral_centroid':
-            self.data = self._spectral_centroid()
-        elif self.feature_type == 'onset_detection':
-            self.data = self._onset_detection()
-        elif self.feature_type == 'chroma_features':
-            self.data = self._chroma_features()
-        else:
-            raise ValueError("Unsupported feature type")
-        return self.normalize()
-
-    def _get_audio_frame(self, frame_index):
-        start_time = frame_index * self.frame_duration
-        end_time = (frame_index + 1) * self.frame_duration
-        start_sample = int(start_time * self.sample_rate)
-        end_sample = int(end_time * self.sample_rate)
-        if start_sample >= len(self.audio):
-            return np.array([])  # Return empty array if we've run out of audio
-        return self.audio[start_sample:min(end_sample, len(self.audio))]
-
-    def _amplitude_envelope(self):
-        def safe_max(frame):
-            return np.max(np.abs(frame)) if frame.size > 0 else 0
-        return np.array([safe_max(self._get_audio_frame(i)) for i in range(self.num_frames)])
-
-    def _rms_energy(self):
-        return np.array([np.sqrt(np.mean(self._get_audio_frame(i)**2)) for i in range(self.num_frames)])
-
-    def _spectral_centroid(self):
-        return np.array([np.mean(librosa.feature.spectral_centroid(y=self._get_audio_frame(i), sr=self.sample_rate)[0]) for i in range(self.num_frames)])
-
-    def _onset_detection(self):
-        return np.array([np.mean(librosa.onset.onset_strength(y=self._get_audio_frame(i), sr=self.sample_rate)) for i in range(self.num_frames)])
-
-    def _chroma_features(self):
-        return np.array([np.mean(librosa.feature.chroma_stft(y=self._get_audio_frame(i), sr=self.sample_rate), axis=1) for i in range(self.num_frames)])
-
 class DepthFeature(BaseFeature):
     def __init__(self, name, frame_rate, frame_count, depth_maps, feature_name='mean_depth'):
         self.depth_maps = depth_maps
         self.features = None
         self.feature_name = feature_name
-        self.available_features = [
+        self.available_features = self.get_extraction_methods()
+        super().__init__(name, "depth", frame_rate, frame_count)
+
+    @classmethod
+    def get_extraction_methods(self):
+        return [
             "mean_depth", "depth_variance", "depth_range", "gradient_magnitude",
             "foreground_ratio", "midground_ratio", "background_ratio"
         ]
-        super().__init__(name, "depth", frame_rate, frame_count)
 
     def extract(self):
         self.features = {self.feature_name: []}
@@ -197,12 +163,16 @@ class ColorFeature(BaseFeature):
             raise ValueError(f"Expected images in BHWC format, but got shape {images.shape}")
         self.images = images
         self.feature_name = feature_name
-        self.available_features = [
+        self.available_features = self.get_extraction_methods()
+        super().__init__(name, "color", frame_rate, frame_count)
+
+    @classmethod
+    def get_extraction_methods(self):
+        return [
             "dominant_color", "color_variance", "saturation",
             "red_ratio", "green_ratio", "blue_ratio"
         ]
-        super().__init__(name, "color", frame_rate, frame_count)
-
+    
     def extract(self):
         self.features = {self.feature_name: []}
         
@@ -275,12 +245,16 @@ class BrightnessFeature(BaseFeature):
     def __init__(self, name, frame_rate, frame_count, images, feature_name='mean_brightness'):
         self.images = images
         self.feature_name = feature_name
-        self.available_features = [
+        self.available_features = self.get_extraction_methods()
+        super().__init__(name, "brightness", frame_rate, frame_count)
+
+    @classmethod
+    def get_extraction_methods(self):
+        return [
             "mean_brightness", "brightness_variance", "brightness_histogram",
             "dark_ratio", "mid_ratio", "bright_ratio"
         ]
-        super().__init__(name, "brightness", frame_rate, frame_count)
-
+    
     def extract(self):
         self.features = {self.feature_name: []}
         
@@ -342,14 +316,19 @@ class MotionFeature(BaseFeature):
         self.flow_method = flow_method
         self.flow_threshold = flow_threshold
         self.magnitude_threshold = magnitude_threshold
-        self.available_features = [
+        self.available_features = self.get_extraction_methods()
+        self.progress_callback = progress_callback
+        super().__init__(name, "motion", frame_rate, frame_count)
+
+    @classmethod
+    def get_extraction_methods(self):
+        return [
             "mean_motion", "max_motion", "motion_direction",
             "horizontal_motion", "vertical_motion", "motion_complexity",
             "motion_speed"
         ]
-        self.progress_callback = progress_callback
-        super().__init__(name, "motion", frame_rate, frame_count)
-
+    
+    
     def extract(self):
         print("Starting MotionFeature extraction")
         self.features = {self.feature_name: []}
@@ -434,9 +413,13 @@ class AreaFeature(BaseFeature):
         self.masks = masks
         self.feature_type = feature_type
         self.threshold = threshold
-        self.available_features = ["total_area", "largest_contour", "bounding_box"]
+        self.available_features = self.get_extraction_methods()
         super().__init__(name, "area", frame_rate, frame_count)
 
+    @classmethod
+    def get_extraction_methods(self):
+        return ["total_area", "largest_contour", "bounding_box"]
+
     def extract(self):
         self.data = []
         
@@ -489,4 +472,5 @@ class AreaFeature(BaseFeature):
             self.extract()  # Re-extract the data with the new feature type
         else:
             raise ValueError(f"Invalid feature name. Available features are: {', '.join(self.available_features)}")
+
 #TODO volume feature

nodes/flex/features_audio.py

@@ -0,0 +1,383 @@
+from .features import BaseFeature
+import librosa
+import numpy as np
+from scipy.signal import hilbert
+from scipy import signal
+import functools
+from abc  import ABC, abstractmethod
+import numpy as np
+import librosa
+from scipy.signal import medfilt, hilbert
+from scipy import signal
+import functools
+
+class BaseAudioFeature(BaseFeature):
+
+    @classmethod
+    @abstractmethod
+    def get_extraction_methods(cls):
+        """Return a list of parameter names that can be modulated."""
+        return []
+    
+    def __init__(self, name, audio, frame_count, frame_rate):
+        super().__init__(name, "audio", frame_rate, frame_count)
+        self.audio = audio
+        self.sample_rate = None
+        self.frame_duration = None
+        self.feature_name = None
+        self.available_features = []
+
+    def _prepare_audio(self):
+        self.sample_rate = self.audio['sample_rate']
+        waveform = self.audio['waveform']
+
+        # Handle multi-dimensional tensors
+        if waveform.ndim > 1:
+            waveform = waveform.squeeze()
+
+        # Handle multi-channel audio by averaging channels
+        if waveform.ndim > 1:
+            waveform = waveform.mean(axis=0)
+
+        self.audio_array = waveform.cpu().numpy()
+        self.frame_duration = 1 / self.frame_rate
+
+    def _get_audio_frame(self, frame_index):
+        start_time = frame_index * self.frame_duration
+        end_time = start_time + self.frame_duration
+        start_sample = int(start_time * self.sample_rate)
+        end_sample = int(end_time * self.sample_rate)
+        if start_sample >= len(self.audio_array):
+            return np.array([])  # Return empty array if we've run out of audio
+        return self.audio_array[start_sample:min(end_sample, len(self.audio_array))]
+
+    def get_feature_sequence(self, feature_name=None):
+        if self.features is None:
+            self.extract()
+        if feature_name is None:
+            feature_name = self.feature_name
+        return self.features.get(feature_name, None)
+
+    def set_active_feature(self, feature_name):
+        if feature_name in self.available_features:
+            self.feature_name = feature_name
+        else:
+            raise ValueError(
+                f"Invalid feature name. Available features are: {', '.join(self.available_features)}"
+            )
+   
+class AudioFeature(BaseAudioFeature):
+
+    def __init__(self, feature_name, audio, frame_count, frame_rate, feature_type='amplitude_envelope'):
+        super().__init__(feature_name, audio, frame_count, frame_rate)
+        self.feature_type = feature_type
+        self.available_features = self.get_extraction_methods()
+        self.feature_name = feature_type
+        self._prepare_audio()
+
+    @classmethod
+    def get_extraction_methods(cls):
+        """Return a list of parameter names that can be modulated."""
+        return [
+            'amplitude_envelope',
+            'rms_energy',
+            'spectral_centroid',
+            'onset_strength',
+            'chroma_features'
+        ]
+    
+    def extract(self):
+        self.features = {self.feature_name: []}
+        for i in range(self.frame_count):
+            frame = self._get_audio_frame(i)
+            value = self._calculate_feature(frame)
+            self.features[self.feature_name].append(value)
+        self._normalize_features()
+        return self
+
+    def _calculate_feature(self, frame):
+        if frame.size == 0:
+            return 0.0
+        if self.feature_name == 'amplitude_envelope':
+            return np.max(np.abs(frame))
+        elif self.feature_name == 'rms_energy':
+            return np.sqrt(np.mean(frame ** 2))
+        elif self.feature_name == 'spectral_centroid':
+            centroid = librosa.feature.spectral_centroid(y=frame, sr=self.sample_rate)
+            return np.mean(centroid)
+        elif self.feature_name == 'onset_strength':
+            strength = librosa.onset.onset_strength(y=frame, sr=self.sample_rate)
+            return np.mean(strength)
+        elif self.feature_name == 'chroma_features':
+            chroma = librosa.feature.chroma_stft(y=frame, sr=self.sample_rate)
+            return np.mean(chroma)
+        else:
+            raise ValueError(f"Unsupported feature type: {self.feature_name}")
+
+    def _normalize_features(self):
+        feature_array = np.array(self.features[self.feature_name], dtype=np.float32)
+        min_val = np.min(feature_array)
+        max_val = np.max(feature_array)
+        if max_val > min_val:
+            normalized = (feature_array - min_val) / (max_val - min_val)
+        else:
+            normalized = np.zeros_like(feature_array)
+        self.features[self.feature_name] = normalized.tolist()
+
+
+class PitchFeature(BaseAudioFeature):
+    def __init__(
+        self,
+        feature_name,
+        audio,
+        frame_count,
+        frame_rate,
+        pitch_range_collections=None,
+        feature_type='frequency',
+        window_size=0,
+        vibrato_options=None,
+        crepe_model="none"
+    ):
+        super().__init__(feature_name, audio, frame_count, frame_rate)
+        self.available_features = self.get_extraction_methods()
+        self.feature_type = feature_type
+        self.pitch_range_collections = pitch_range_collections or []
+        self.vibrato_options = vibrato_options
+        self.window_size = window_size
+        self.current_frame = 0
+        self.crepe_model = crepe_model
+        self.feature_name = feature_name
+        self._prepare_audio()
+        
+        # Parameters for pitch detection
+        self.frame_length = 2048
+        self.hop_length = 512
+        self.fmin = 20  # Lower frequency limit
+        self.fmax = 4000  # Upper frequency limit
+
+    @staticmethod
+    def calculate_tolerance(frequency, tolerance_percent):
+        semitone_above = frequency * 2**(1/12)
+        frequency_difference = semitone_above - frequency
+        tolerance = (frequency_difference / 2) * (tolerance_percent / 100.0)
+        return tolerance
+    
+    @classmethod
+    def get_extraction_methods(cls):
+        """Return a list of parameter names that can be modulated."""
+        return [
+            'frequency',
+            'semitone',
+            'pitch_direction',
+            'vibrato_signal',
+            'vibrato_strength',
+        ]
+    
+    @classmethod
+    def pitch_to_note(cls, pitch):
+        if pitch == 0:
+            return "N/A"
+        return librosa.hz_to_note(pitch)
+
+    def _calculate_pitch_sequence(self):
+        # Try to use CREPE model if available
+        if self.crepe_model != "none":
+            try:
+                import crepe
+                import tensorflow
+                time_steps, frequencies, confidence, activation = crepe.predict(
+                    self.audio_array,
+                    self.sample_rate,
+                    model_capacity=self.crepe_model,
+                    viterbi=True,
+                    step_size=int(1000 * self.hop_length / self.sample_rate)  # Step size in ms
+                )            
+                # time_steps, frequencies, confidences, _ = crepe.predict(
+                #     self.audio_array,
+                #     self.sample_rate,
+                #     model_capacity=self.crepe_model,
+                #     viterbi=True,
+                #     step_size=int(1000 * self.hop_length / self.sample_rate)  # Step size in ms
+                # )
+                pitches = frequencies
+                confidences = confidence
+            except ImportError as e:
+                print(f"Please 'pip install crepe tensorflow' to use any CREPE model :)")
+                pitches, confidences = self._fallback_pitch_estimation()
+            except Exception as e:
+                print(f"Error using CREPE model: {e}. Falling back to librosa's pitch estimation.")
+                pitches, confidences = self._fallback_pitch_estimation()
+        else:
+            # Fallback to librosa if CREPE model is not provided
+            pitches, confidences = self._fallback_pitch_estimation()
+
+        # Interpolate to match frame count
+        frame_times = np.linspace(0, len(self.audio_array) / self.sample_rate, num=self.frame_count)
+        if len(pitches) == 0:
+            pitches = np.zeros(len(frame_times))
+            confidences = np.zeros(len(frame_times))
+            pitch_times = frame_times
+        else:
+            pitch_times = librosa.frames_to_time(np.arange(len(pitches)), sr=self.sample_rate, hop_length=self.hop_length)
+            pitches = np.nan_to_num(pitches)
+            confidences = np.nan_to_num(confidences)
+            pitches = np.interp(frame_times, pitch_times, pitches)
+            confidences = np.interp(frame_times, pitch_times, confidences)
+
+        # Handle potential NaNs
+        pitches = np.nan_to_num(pitches)
+        confidences = np.nan_to_num(confidences)
+
+        self.features = {}
+        self.features[self.feature_name + '_pitch'] = pitches.tolist()
+        self.features[self.feature_name + '_confidence'] = confidences.tolist()
+
+    def _fallback_pitch_estimation(self):
+        pitches, magnitudes = librosa.piptrack(
+            y=self.audio_array,
+            sr=self.sample_rate,
+            fmin=self.fmin,
+            fmax=self.fmax,
+            hop_length=self.hop_length,
+            threshold=0.1
+        )
+        pitches = pitches.max(axis=0)
+        confidences = magnitudes.max(axis=0)
+        return pitches, confidences
+
+    def extract(self):
+        self.features = {}
+        self._calculate_pitch_sequence()
+
+        if self.feature_type == 'frequency':
+            self._extract_frequency()
+        elif self.feature_type == 'semitone':
+            self._extract_semitone()
+        elif self.feature_type == 'pitch_direction':
+            self._extract_pitch_direction()
+        elif self.feature_type == 'vibrato_signal':
+            self._extract_vibrato_signal()
+        elif self.feature_type == 'vibrato_strength':
+            self._extract_vibrato_strength()
+        else:
+            raise ValueError(f"Unsupported feature type: {self.feature_type}")
+
+        self._normalize_features()
+        return self
+
+    def _extract_frequency(self):
+        pitches = np.array(self.features[self.feature_name + '_pitch'])
+        valid_pitches = pitches[pitches > 0]
+
+        if len(valid_pitches) == 0:
+            feature_values = np.zeros_like(pitches)
+        else:
+            feature_values = pitches.copy()
+        self.features[self.feature_name + '_original'] = feature_values.tolist()
+        self.features[self.feature_name] = feature_values.tolist()
+
+    def _extract_semitone(self):
+        pitches = np.array(self.features[self.feature_name + '_pitch'])
+        midi_notes = librosa.hz_to_midi(pitches)
+        midi_notes = np.round(midi_notes)
+        semitone_freqs = librosa.midi_to_hz(midi_notes)
+
+        semitone_freqs[pitches == 0] = 0.0
+        self.features[self.feature_name + '_original'] = semitone_freqs.tolist()
+        self.features[self.feature_name] = semitone_freqs.tolist()
+
+    def _extract_pitch_direction(self):
+        pitches = np.array(self.features[self.feature_name + '_pitch'])
+        pitches[pitches == 0] = np.nan  # Replace zeros with NaN for diff calculation
+        pitch_diff = np.diff(pitches, prepend=pitches[0])
+
+        direction = np.sign(pitch_diff)
+        direction = np.nan_to_num(direction)  # Replace NaNs back to zero
+
+        self.features[self.feature_name + '_original'] = direction.tolist()
+        self.features[self.feature_name] = direction.tolist()
+
+    def _extract_vibrato_signal(self):
+        pitches = np.array(self.features[self.feature_name + '_pitch'])
+        pitches[pitches == 0] = np.nan  # Ignore zero pitches
+
+        # Interpolate to fill NaNs for Hilbert transform
+        valid_indices = np.where(~np.isnan(pitches))[0]
+        if len(valid_indices) == 0:
+            vibrato_signal = np.zeros_like(pitches)
+        else:
+            pitches = np.interp(np.arange(len(pitches)), valid_indices, pitches[valid_indices])
+
+            analytic_signal = hilbert(pitches)
+            instantaneous_phase = np.unwrap(np.angle(analytic_signal))
+            instantaneous_frequency = np.diff(instantaneous_phase)
+
+            vibrato_signal = np.concatenate(([0], instantaneous_frequency))
+
+        self.features[self.feature_name + '_original'] = vibrato_signal.tolist()
+        self.features[self.feature_name] = vibrato_signal.tolist()
+
+    def _extract_vibrato_strength(self):
+        pitches = np.array(self.features[self.feature_name + '_pitch'])
+        pitches[pitches == 0] = np.nan  # Ignore zero pitches
+
+        window_size_frames = int(0.1 * self.sample_rate / self.hop_length)  # 100ms window
+        if window_size_frames < 1:
+            window_size_frames = 1
+
+        # Compute standard deviation over the window
+        vibrato_strength = []
+        for i in range(len(pitches)):
+            start = max(0, i - window_size_frames // 2)
+            end = min(len(pitches), i + window_size_frames // 2)
+            window_pitches = pitches[start:end]
+            window_pitches = window_pitches[~np.isnan(window_pitches)]
+            if len(window_pitches) > 0:
+                std = np.std(window_pitches)
+            else:
+                std = 0.0
+            vibrato_strength.append(std)
+
+        vibrato_strength = np.array(vibrato_strength)
+        self.features[self.feature_name + '_original'] = vibrato_strength.tolist()
+        self.features[self.feature_name] = vibrato_strength.tolist()
+
+    def _normalize_features(self):
+        feature_array = np.array(self.features[self.feature_name], dtype=np.float32)
+        finite_mask = np.isfinite(feature_array)
+        if not np.any(finite_mask):
+            normalized = np.zeros_like(feature_array)
+        else:
+            min_val = np.min(feature_array[finite_mask])
+            max_val = np.max(feature_array[finite_mask])
+            if max_val > min_val:
+                normalized = np.zeros_like(feature_array)
+                normalized[finite_mask] = (feature_array[finite_mask] - min_val) / (max_val - min_val)
+            else:
+                normalized = np.zeros_like(feature_array)
+        self.features[self.feature_name] = normalized.tolist()
+
+    def get_pitch_feature(self, frame_index):
+        if self.features is None:
+            self.extract()
+
+        original_value = self.features[self.feature_name + '_original'][frame_index]
+        normalized_value = self.features[self.feature_name][frame_index]
+        actual_pitch = self.features[self.feature_name + '_pitch'][frame_index]
+        # Provide smoothed pitch if available
+        smoothed_pitch = self.features.get(self.feature_name + '_smoothed', [0.0]*self.frame_count)[frame_index]
+
+        return {
+            'original': original_value,
+            'normalized': normalized_value,
+            'actual_pitch': actual_pitch,
+            'smoothed_pitch': smoothed_pitch
+        }
+        
+class PitchRange:
+    def __init__(self, min_pitch, max_pitch):
+        self.min_pitch = min_pitch
+        self.max_pitch = max_pitch
+
+    def contains(self, pitch):
+        return self.min_pitch <= pitch <= self.max_pitch

nodes/flex/features_proximity.py

@@ -0,0 +1,95 @@
+from .features import  BaseFeature
+import torch
+import numpy as np
+
+import numpy as np
+from .features import BaseFeature
+
+class ProximityFeature(BaseFeature):
+    def __init__(self, name, anchor_locations, query_locations, frame_rate, frame_count, frame_dimensions, normalization_method='frame'):
+        super().__init__(name, "proximity", frame_rate, frame_count)
+        self.anchor_locations = anchor_locations
+        self.query_locations = query_locations
+        self.frame_diagonal = np.sqrt(frame_dimensions[0]**2 + frame_dimensions[1]**2)
+        self.proximity_values = None
+        self.normalization_method = normalization_method
+
+    def extract(self):
+        from scipy.spatial.distance import cdist
+        
+        proximities = []
+        for anchor, query in zip(self.anchor_locations, self.query_locations):
+            if len(anchor) == 0 or len(query) == 0:
+                proximities.append(self.frame_diagonal if self.normalization_method == 'frame' else float('inf'))
+            else:
+                # Ensure points are treated as floats
+                anchor_points = anchor.points.astype(float)
+                query_points = query.points.astype(float)
+                distances = cdist(anchor_points, query_points)
+                min_distance = np.min(distances)
+                proximities.append(min_distance)
+        
+        proximities = np.array(proximities, dtype=float)
+        
+        if self.normalization_method == 'frame':
+            self.proximity_values = 1 - np.clip(proximities / self.frame_diagonal, 0, 1)
+        elif self.normalization_method == 'minmax':
+            min_proximity = np.min(proximities)
+            max_proximity = np.max(proximities)
+            if max_proximity > min_proximity:
+                self.proximity_values = 1 - (proximities - min_proximity) / (max_proximity - min_proximity)
+            else:
+                self.proximity_values = np.ones_like(proximities)
+        else:
+            raise ValueError(f"Unknown normalization method: {self.normalization_method}")
+        
+        self.smooth_proximities()
+        return self
+
+    def smooth_proximities(self, window_size=5):
+        kernel = np.ones(window_size) / window_size
+        self.proximity_values = np.convolve(self.proximity_values, kernel, mode='same')
+
+    def get_value_at_frame(self, frame_index):
+        return self.proximity_values[frame_index]
+
+class Location:
+    def __init__(self, x, y, z=None):
+        x = np.asarray(x, dtype=float).reshape(-1)
+        y = np.asarray(y, dtype=float).reshape(-1)
+        
+        if z is not None:
+            z = np.asarray(z, dtype=float).reshape(-1)
+            if abs(len(z) - len(x)) <= 1:
+                z = z[:len(x)] if len(z) > len(x) else np.pad(z, (0, len(x) - len(z)), 'constant', constant_values=0.0)
+            elif len(z) != len(x):
+                raise ValueError(f"Mismatch in dimensions: x,y have length {len(x)}, but z has length {len(z)}")
+            self.points = np.column_stack((x, y, z))
+        else:
+            self.points = np.column_stack((x, y))
+        
+
+    def __len__(self):
+        return len(self.points)
+
+    def __getitem__(self, idx):
+        return self.points[idx]
+
+    @property
+    def x(self):
+        return self.points[:, 0]
+
+    @property
+    def y(self):
+        return self.points[:, 1]
+
+    @property
+    def z(self):
+        return self.points[:, 2] if self.points.shape[1] > 2 else None
+
+    @classmethod
+    def from_tensor(cls, tensor):
+        return cls(tensor[:, 0].astype(float), tensor[:, 1].astype(float), tensor[:, 2].astype(float) if tensor.shape[1] > 2 else None)
+
+    def to_tensor(self):
+        return torch.from_numpy(self.points).float()

nodes/flex/visualizers.py

@@ -0,0 +1,197 @@
+from ... import RyanOnTheInside 
+import cv2
+import numpy as np
+import  torch
+from scipy.spatial.distance import cdist
+import math
+
+class EffectVisualizer(RyanOnTheInside):
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "video_frames": ("IMAGE",),
+                "feature": ("FEATURE",),
+                "text_color": ("STRING", {"default": "(255,255,255)"}),
+                "font_scale": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 2.0, "step": 0.1}),
+            }
+        }
+
+    RETURN_TYPES = ("IMAGE",)
+    FUNCTION = "visualize"
+    CATEGORY = "RyanOnTheInside/FlexFeatures/EffectVisualizers"
+
+    def visualize(self, video_frames, feature, text_color, font_scale):
+        text_color = self.parse_color(text_color)
+        output_frames = []
+        padding = 10  # Padding from the edges
+
+        for frame_index in range(len(video_frames)):
+            frame = video_frames[frame_index].cpu().numpy()
+            frame = (frame * 255).astype(np.uint8)
+            frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
+
+            # Display feature name and value on the frame
+            feature_value = feature.get_value_at_frame(frame_index)
+            text = f"{feature.name}: {feature_value:.2f}"
+            text_size = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, font_scale, 2)[0]
+            text_x = padding
+            text_y = padding + text_size[1]
+
+            cv2.putText(frame, text, (text_x, text_y), cv2.FONT_HERSHEY_SIMPLEX, font_scale, text_color, 2)
+
+            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            output_frames.append(frame)
+
+        output_tensor = torch.from_numpy(np.stack(output_frames)).float() / 255.0
+        return (output_tensor,)
+
+    def parse_color(self, color_string):
+        return tuple(map(int, color_string.strip("()").split(",")))
+
+
+class ProximityVisualizer(EffectVisualizer):
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                **super().INPUT_TYPES()["required"],
+                "anchor_locations": ("LOCATION",),
+                "query_locations": ("LOCATION",),
+                "anchor_color": ("STRING", {"default": "(255,0,0)"}),
+                "query_color": ("STRING", {"default": "(0,255,0)"}),
+                "line_color": ("STRING", {"default": "(0,0,255)"}),
+            }
+        }
+
+    RETURN_TYPES = ("IMAGE",)
+    FUNCTION = "visualize_proximity"
+
+    def visualize_proximity(self, video_frames, anchor_locations, query_locations, feature, 
+                            anchor_color, query_color, line_color, text_color, font_scale):
+        anchor_color = self.parse_color(anchor_color)
+        query_color = self.parse_color(query_color)
+        line_color = self.parse_color(line_color)
+        text_color = self.parse_color(text_color)
+
+        output_frames = []
+        height, width = video_frames.shape[1:3]  # Extract height and width from video_frames
+
+        # Calculate the frame diagonal from the video frames
+        frame_diagonal = np.sqrt(width**2 + height**2)
+        scale_factor = frame_diagonal / feature.frame_diagonal
+
+        for frame_index in range(len(video_frames)):
+            frame = video_frames[frame_index].cpu().numpy()
+            frame = (frame * 255).astype(np.uint8)
+            frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
+
+            anchor = anchor_locations[frame_index]
+            query = query_locations[frame_index]
+
+            # Draw all anchor and query points (using only x and y coordinates)
+            for point in anchor:
+                scaled_point = (int(point[0] * scale_factor), int(point[1] * scale_factor))
+                cv2.circle(frame, scaled_point, 2, anchor_color, -1)
+            for point in query:
+                scaled_point = (int(point[0] * scale_factor), int(point[1] * scale_factor))
+                cv2.circle(frame, scaled_point, 2, query_color, -1)
+
+            # Find the closest pair of points
+            if len(anchor) > 0 and len(query) > 0:
+                distances = cdist(anchor.points[:, :2], query.points[:, :2])  # Use only x and y for distance calculation
+                min_idx = np.unravel_index(distances.argmin(), distances.shape)
+                closest_anchor = anchor[min_idx[0]]
+                closest_query = query[min_idx[1]]
+
+                # Draw line between closest points (using only x and y coordinates)
+                cv2.line(frame, 
+                         (int(closest_anchor[0] * scale_factor), int(closest_anchor[1] * scale_factor)), 
+                         (int(closest_query[0] * scale_factor), int(closest_query[1] * scale_factor)), 
+                         line_color, 2)
+
+                # Display coordinates of closest points
+                anchor_text = f"Anchor: ({closest_anchor[0] * scale_factor:.2f}, {closest_anchor[1] * scale_factor:.2f}"
+                query_text = f"Query: ({closest_query[0] * scale_factor:.2f}, {closest_query[1] * scale_factor:.2f}"
+                
+                if closest_anchor.shape[0] > 2 and closest_query.shape[0] > 2:
+                    anchor_text += f", {closest_anchor[2]:.2f})"
+                    query_text += f", {closest_query[2]:.2f})"
+                else:
+                    anchor_text += ") [2D]"
+                    query_text += ") [2D]"
+
+                cv2.putText(frame, anchor_text, (10, 60), cv2.FONT_HERSHEY_SIMPLEX, font_scale, text_color, 1)
+                cv2.putText(frame, query_text, (10, 90), cv2.FONT_HERSHEY_SIMPLEX, font_scale, text_color, 1)
+
+            # Add proximity value to the frame
+            proximity_value = feature.get_value_at_frame(frame_index)
+            cv2.putText(frame, f"Proximity: {proximity_value:.2f}", (10, 30),
+                        cv2.FONT_HERSHEY_SIMPLEX, 1, text_color, 2)
+
+            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            output_frames.append(frame)
+
+        output_tensor = torch.from_numpy(np.stack(output_frames)).float() / 255.0
+        return (output_tensor,)
+
+
+
+class PitchVisualizer(EffectVisualizer):
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "video_frames": ("IMAGE",),
+                "feature": ("FEATURE",),
+                "text_color": ("STRING", {"default": "(255,255,255)"}),
+                "font_scale": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 2.0, "step": 0.1}),
+            }
+        }
+
+    RETURN_TYPES = ("IMAGE",)
+    FUNCTION = "visualize_pitch"
+
+    def visualize_pitch(self, video_frames, feature, text_color, font_scale):
+        text_color = self.parse_color(text_color)
+        output_frames = []
+        padding = 10  # Padding from the edges
+
+        # Ensure video_frames is BHWC
+        if video_frames.shape[-1] != 3:
+            video_frames = video_frames.permute(0, 2, 3, 1)
+
+        for frame_index in range(video_frames.shape[0]):
+            frame = video_frames[frame_index].cpu().numpy()
+            frame = (frame * 255).astype(np.uint8)
+            frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
+
+            # Get pitch feature data for the current frame
+            pitch_data = feature.get_pitch_feature(frame_index)
+
+            # Convert pitch to approximate note
+            note = feature.pitch_to_note(pitch_data['actual_pitch'])
+            smoothed_note = feature.pitch_to_note(pitch_data['smoothed_pitch'])
+
+            # Display feature values on the frame
+            texts = [
+                f"Original: {pitch_data['original']:.2f}",
+                f"Normalized: {pitch_data['normalized']:.2f}",
+                f"Actual Pitch: {pitch_data['actual_pitch']:.2f} Hz",
+                f"Note: {note}",
+                f"Smoothed Pitch: {pitch_data['smoothed_pitch']:.2f} Hz",
+                f"Smoothed Note: {smoothed_note}"
+            ]
+
+            for i, text in enumerate(texts):
+                text_size = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, font_scale, 2)[0]
+                text_x = padding
+                text_y = padding + (i + 1) * (text_size[1] + 10)  # Add some vertical spacing
+
+                cv2.putText(frame, text, (text_x, text_y), cv2.FONT_HERSHEY_SIMPLEX, font_scale, text_color, 2)
+
+            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            output_frames.append(frame)
+
+        output_tensor = torch.from_numpy(np.stack(output_frames)).float() / 255.0
+        return (output_tensor,)

nodes/masks/flex_masks_normal.py

@@ -0,0 +1,102 @@
+import numpy as np
+import torch
+import cv2
+from .mask_base import FlexMaskBase
+
+class FlexMaskNormalBase(FlexMaskBase):
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            **super().INPUT_TYPES(),
+            "required": {
+                **super().INPUT_TYPES()["required"],
+                "normal_map": ("IMAGE",),
+            }
+        }
+
+    def normalize_vector(self, vector):
+        return vector / np.linalg.norm(vector)
+
+    def normal_map_to_array(self, normal_map: torch.Tensor) -> np.ndarray:
+        # Convert normal map from [0, 1] to [-1, 1] range
+        normal_array = normal_map.cpu().numpy() * 2 - 1
+        return normal_array
+
+    def main_function(self, masks, feature, feature_pipe, strength, feature_threshold, invert, subtract_original, grow_with_blur, normal_map, **kwargs):
+        # This method should be overridden by child classes
+        raise NotImplementedError("Subclasses must implement main_function")
+
+class FlexMaskNormalLighting(FlexMaskNormalBase):
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            **super().INPUT_TYPES(),
+            "required": {
+                **super().INPUT_TYPES()["required"],
+                "light_direction_x": ("FLOAT", {"default": 0.5, "min": -1.0, "max": 1.0, "step": 0.01}),
+                "light_direction_y": ("FLOAT", {"default": -0.5, "min": -1.0, "max": 1.0, "step": 0.01}),
+                "light_direction_z": ("FLOAT", {"default": 0.7, "min": -1.0, "max": 1.0, "step": 0.01}),
+                "shadow_threshold": ("FLOAT", {"default": 0.2, "min": 0.0, "max": 1.0, "step": 0.01}),
+                "feature_param": (["none", "direction", "threshold", "both"],),
+                "feature_mode": (["rotate", "intensity"],),
+            }
+        }
+
+    CATEGORY = "RyanOnTheInside/FlexMasks"
+
+    def process_mask(self, mask: np.ndarray, feature_value: float, strength: float, 
+                     normal_map: torch.Tensor, light_direction_x: float, light_direction_y: float, 
+                     light_direction_z: float, shadow_threshold: float, feature_param: str, 
+                     feature_mode: str, **kwargs) -> np.ndarray:
+        
+        normal_array = self.normal_map_to_array(normal_map)
+        
+        # Normalize light direction
+        light_direction = self.normalize_vector(np.array([light_direction_x, light_direction_y, light_direction_z]))
+        
+        # Apply feature modulation
+        if feature_param != "none":
+            if feature_mode == "rotate":
+                # Rotate light direction based on feature value
+                rotation_angle = 2 * np.pi * feature_value * strength
+                rotation_matrix = np.array([
+                    [np.cos(rotation_angle), -np.sin(rotation_angle), 0],
+                    [np.sin(rotation_angle), np.cos(rotation_angle), 0],
+                    [0, 0, 1]
+                ])
+                light_direction = np.dot(rotation_matrix, light_direction)
+            elif feature_mode == "intensity":
+                # Modify shadow threshold based on feature value
+                shadow_threshold += (feature_value - 0.5) * strength
+                shadow_threshold = np.clip(shadow_threshold, 0.0, 1.0)
+        
+        # Calculate dot product between normal vectors and light direction
+        dot_product = np.einsum('bhwc,c->bhw', normal_array, light_direction)
+        
+        # Create lighting mask
+        lighting_mask = (dot_product > shadow_threshold).astype(np.float32)
+        
+        # Combine with input mask
+        combined_mask = mask * lighting_mask
+        
+        return combined_mask
+
+    def main_function(self, masks, feature, feature_pipe, strength, feature_threshold, invert, subtract_original, grow_with_blur, normal_map, light_direction_x, light_direction_y, light_direction_z, shadow_threshold, feature_param, feature_mode, **kwargs):
+        processed_masks = []
+        for i in range(masks.shape[0]):  # Iterate over all frames
+            mask = masks[i].cpu().numpy()
+            feature_value = feature.get_value_at_frame(i)
+            
+            if feature_value >= feature_threshold:
+                processed_mask = self.process_mask(
+                    mask, feature_value, strength, normal_map[i], 
+                    light_direction_x, light_direction_y, light_direction_z, 
+                    shadow_threshold, feature_param, feature_mode
+                )
+            else:
+                processed_mask = mask
+            
+            processed_masks.append(processed_mask)
+        
+        processed_masks = np.stack(processed_masks)
+        return self.apply_mask_operation(torch.from_numpy(processed_masks), masks, strength, invert, subtract_original, grow_with_blur)

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "comfyui_ryanonyheinside"
 description = "Custom nodes introducing particle simulations, optical flow, audio manipulation & reactivity, and temporal masks"
-version = "1.6.0"
+version = "1.8.0"
 license = {file = "LICENSE"}
 dependencies = ["opencv-python==4.10.0","scipy","torchaudio", "pillow","librosa","pymunk==6.8.1","matplotlib","openunmix","mido"]
 

readme.md

@@ -33,6 +33,7 @@
 *Examples showcasing various effects using particle emitters, vortices, and other node features*
 
 ##### 🆕 Recent Updates
+-9/15/24 - **alot** Depth From Shape, Audio Pitch Feature, Pitch Range  filters, new MIDI keybord for Pitch Range specification, Image from audio, mask from audio, Improved depth chamber, wave propagation, emanating rings,   and a lot more 
 - 9/8/24 - **Area Feature**: Adds area  as a  driving reactivity feature!
 - 9/7/24 - **Proximity Feature**: Adds proximity as a driving reactivity feature! Allows for the distince of objects from one another to control other nodes.
 - 9/5/24 - **FlexImageParallax**: Add illusory depth.

web/js/midi_extensions.js

@@ -1,5 +1,4 @@
 import { app } from "../../scripts/app.js";
-import { api } from "../../scripts/api.js";
 import { drawPiano, handlePianoMouseDown } from "./piano.js";
 
 app.registerExtension({
@@ -11,7 +10,6 @@ app.registerExtension({
 
     async beforeRegisterNodeDef(nodeType, nodeData, app) {
         if (nodeData.name === "MIDILoadAndExtract") {
-            console.log("MIDI Extension: Modifying MIDILoadAndExtract node");
 
             // Add the disableWidget method to the nodeType prototype
             nodeType.prototype.disableWidget = function(widget) {
@@ -41,7 +39,6 @@ app.registerExtension({
                         .then(response => response.json())
                         .then(data => {
                             if (data.success) {
-                                console.log("MIDI file uploaded successfully");
                                 // Update the midi_file widget with the new file path
                                 const midiFileWidget = this.widgets.find(w => w.name === "midi_file");
                                 if (midiFileWidget) {
@@ -67,7 +64,6 @@ app.registerExtension({
             };
             
             nodeType.prototype.onNodeCreated = function() {
-                console.log("MIDI Extension: onNodeCreated called for MIDILoadAndExtract");
                 this.properties = this.properties || {};
                 this.properties.selectedNotes = []; // Initialize with an empty array
                 this.pianoScroll = 0;
@@ -95,7 +91,6 @@ app.registerExtension({
                 const midiFileWidget = this.widgets.find(w => w.name === "midi_file");
                 if (midiFileWidget) {
                     midiFileWidget.callback = (value) => {
-                        console.log('MIDI file changed to:', value);
                         this.updateTrackSelection();
                         this.saveState(); // Save state when MIDI file changes
                     };
@@ -127,28 +122,23 @@ app.registerExtension({
                 // Instead of calling updateTrackSelection here, we'll set up a callback
                 // that will be called when the graph is available
                 this.onGraphAvailable = () => {
-                    console.log('Graph available, updating track selection');
                     this.updateTrackSelection();
                 };
             };
 
             nodeType.prototype.loadSavedState = function() {
-                console.log(`loadSavedState called for node ${this.id}`);
                 const savedState = localStorage.getItem(`MIDILoadAndExtract_${this.id}`);
                 if (savedState) {
                     this.loadedState = JSON.parse(savedState);
-                    console.log('Loaded saved state:', this.loadedState);
                     
                     const midiFileWidget = this.widgets.find(w => w.name === "midi_file");
                     if (midiFileWidget && this.loadedState.midiFile) {
                         midiFileWidget.value = this.loadedState.midiFile;
-                        console.log(`Set midiFile to: ${midiFileWidget.value}`);
                     }
                 }
             };
 
             nodeType.prototype.updateTrackSelection = async function() {
-                console.log(`updateTrackSelection called, isInitialLoad: ${this.isInitialLoad}`);
                 if (this.isUpdatingTrackSelection) {
                     console.log('Track selection update already in progress, skipping');
                     return;
@@ -157,7 +147,6 @@ app.registerExtension({
 
                 try {
                     const midiFile = this.getCurrentMidiFile();
-                    console.log(`Current MIDI file: ${midiFile}`);
 
                     if (!midiFile) {
                         console.warn('No MIDI file selected');
@@ -176,7 +165,6 @@ app.registerExtension({
                         return;
                     }
                     
-                    console.log('Received track data:', data);
 
                     const trackSelectionWidget = this.widgets.find(w => w.name === "track_selection");
                     if (trackSelectionWidget) {
@@ -186,7 +174,6 @@ app.registerExtension({
 
                     // Update available notes
                     this.availableNotes = new Set(data.all_notes.split(',').map(Number));
-                    console.log(`Updated available notes: ${Array.from(this.availableNotes)}`);
                     
                     // Apply loaded state if this is the initial load
                     if (this.isInitialLoad && this.loadedState) {
@@ -195,7 +182,6 @@ app.registerExtension({
                     }
 
                     this.setDirtyCanvas(true, true);
-                    console.log('Track selection update completed');
                 } catch (error) {
                     console.error('Error updating track selection:', error);
                     this.availableNotes = new Set(); // Reset to empty Set in case of error
@@ -205,7 +191,6 @@ app.registerExtension({
             };
 
             nodeType.prototype.refreshMIDIData = async function() {
-                console.log("Refreshing MIDI data");
                 const midiFile = this.getCurrentMidiFile();
                 const trackSelection = this.widgets.find(w => w.name === "track_selection").value;
 
@@ -227,7 +212,6 @@ app.registerExtension({
                         return;
                     }
                     
-                    console.log('Received refreshed MIDI data:', data);
 
                     // Update available tracks
                     const trackSelectionWidget = this.widgets.find(w => w.name === "track_selection");
@@ -237,7 +221,6 @@ app.registerExtension({
 
                     // Update available notes
                     this.availableNotes = new Set(data.all_notes.split(',').map(Number));
-                    console.log(`Updated available notes: ${Array.from(this.availableNotes)}`);
                     
                     // Filter out selected notes that are no longer available
                     if (this.properties.selectedNotes) {
@@ -256,13 +239,11 @@ app.registerExtension({
 
             nodeType.prototype.applyLoadedState = function() {
                 if (this.loadedState) {
-                    console.log('Applying loaded state:', this.loadedState);
                     
                     const trackSelectionWidget = this.widgets.find(w => w.name === "track_selection");
                     if (trackSelectionWidget && this.loadedState.trackSelection) {
                         if (trackSelectionWidget.options.values.includes(this.loadedState.trackSelection)) {
                             trackSelectionWidget.value = this.loadedState.trackSelection;
-                            console.log(`Set track selection to loaded value: ${trackSelectionWidget.value}`);
                         } else {
                             console.log(`Loaded track selection ${this.loadedState.trackSelection} is no longer valid`);
                         }
@@ -270,7 +251,6 @@ app.registerExtension({
                     
                     if (this.loadedState.selectedNotes) {
                         this.properties.selectedNotes = this.loadedState.selectedNotes.filter(note => this.availableNotes.has(note));
-                        console.log(`Applied selected notes: ${this.properties.selectedNotes}`);
                     }
                     
                     this.syncNotesWidget();
@@ -285,7 +265,6 @@ app.registerExtension({
                     trackSelection: this.widgets.find(w => w.name === "track_selection")?.value,
                     selectedNotes: this.properties.selectedNotes
                 };
-                console.log("Saving state:", state);
                 localStorage.setItem(`MIDILoadAndExtract_${this.id}`, JSON.stringify(state));
             };
 

web/js/pitch_extension.js

@@ -0,0 +1,108 @@
+import { app } from "../../scripts/app.js";
+import { drawPiano, handlePianoMouseDown } from "./piano.js";
+
+app.registerExtension({
+    name: "org.ryanontheinside.pitch_range_by_note",
+
+    async beforeRegisterNodeDef(nodeType, nodeData, app) {
+        if (nodeData.name === "PitchRangeByNoteNode") {
+
+            nodeType.prototype.onNodeCreated = function() {
+                this.properties = this.properties || {};
+                this.properties.selectedNotes = []; // Initialize with an empty array
+                this.pianoScroll = 0;
+                this.scrollLeftIndicator = null;
+                this.scrollRightIndicator = null;
+                this.availableNotes = new Set(Array.from({length: 128}, (_, i) => i)); // All MIDI notes
+
+                const notesWidget = this.widgets.find(widget => widget.name === "notes");
+                if (notesWidget) {
+                    notesWidget.hidden = true;
+                    notesWidget.disabled = true; // Disable the widget immediately
+                }
+
+                // Increase the node's height and width
+                const extraHeight = 80; // Piano height + margins
+                const pianoWidth = 1020; // Total width of the piano
+                this.size[1] += extraHeight;
+                this.size[0] = Math.max(this.size[0], pianoWidth); // Set width to at least piano width plus margins
+
+                // Ensure the node recomputes its size
+                this.setDirtyCanvas(true, true);
+
+                // Sync the notes widget initially
+                this.syncNotesWidget();
+            };
+
+            nodeType.prototype.syncNotesWidget = function() {
+                const notesWidget = this.widgets.find(widget => widget.name === "notes");
+                if (notesWidget) {
+                    const newValue = this.properties.selectedNotes.sort((a, b) => a - b).join(", ");
+                    if (notesWidget.value !== newValue) {
+                        notesWidget.value = newValue;
+                        // Trigger a change event on the widget
+                        if (notesWidget.callback) {
+                            notesWidget.callback(newValue);
+                        }
+                    }
+                }
+                // Ensure the node updates its internal state
+                this.onPropertyChanged('selectedNotes');
+            };
+
+            nodeType.prototype.onPropertyChanged = function(property) {
+                if (property === 'selectedNotes') {
+                    // Trigger any necessary updates or recomputation
+                    this.setDirtyCanvas(true, true);
+                }
+            };
+
+            // Save the original method
+            nodeType.prototype._originalOnDrawForeground = nodeType.prototype.onDrawForeground;
+
+            nodeType.prototype.onDrawForeground = function(ctx) {
+                // Call the original onDrawForeground method if it exists
+                if (typeof nodeType.prototype._originalOnDrawForeground === 'function') {
+                    nodeType.prototype._originalOnDrawForeground.call(this, ctx);
+                }
+
+                if (this.flags.collapsed) return;
+
+                // Draw widgets that are not hidden
+                for (let w of this.widgets) {
+                    if (w.hidden) {
+                        continue;  // Skip drawing hidden widgets
+                    }
+                    if (w.draw) {
+                        w.draw(ctx, this, this.pos);
+                    }
+                }
+
+                drawPiano(ctx, this);
+            };
+
+            // Save the original onMouseDown method
+            nodeType.prototype._originalOnMouseDown = nodeType.prototype.onMouseDown;
+
+            nodeType.prototype.onMouseDown = function(event, pos, graphcanvas) {
+                // Call the original onMouseDown method if it exists
+                if (typeof nodeType.prototype._originalOnMouseDown === 'function') {
+                    nodeType.prototype._originalOnMouseDown.call(this, event, pos, graphcanvas);
+                }
+
+                const [x, y] = pos;
+                return handlePianoMouseDown(this, x, y);
+            };
+
+            // Disable the notes widget
+            const onConnectionsChange = nodeType.prototype.onConnectionsChange;
+            nodeType.prototype.onConnectionsChange = function (type, index, connected, link_info) {
+                onConnectionsChange?.apply(this, arguments);
+                const notesWidget = this.widgets.find((w) => w.name === "notes");
+                if (notesWidget) {
+                    notesWidget.disabled = true;
+                }
+            };
+        }
+    },
+});