diff --git a/__init__.py b/__init__.py index 147b72d..f16fd7b 100644 --- a/__init__.py +++ b/__init__.py @@ -101,6 +101,7 @@ from .nodes.flex.feature_extractors import( AreaFeatureNode, ManualFeatureNode, ManualFeatureFromPipe, + ManualFeatureDraw, ) from .nodes.flex.feature_extractors_audio import( @@ -111,6 +112,7 @@ from .nodes.flex.feature_extractors_audio import( PitchRangeByNoteNode, PitchFeatureExtractor, RhythmFeatureExtractor, + ) from .nodes.flex.feature_extractors_midi import( @@ -401,6 +403,7 @@ NODE_CLASS_MAPPINGS = { "AudioFeatureExtractorFirst": AudioFeatureExtractorFirst, "PitchFeatureExtractor": PitchFeatureExtractor, "RhythmFeatureExtractor": RhythmFeatureExtractor, + "PitchRange": PitchRangeNode, "PitchRangePreset": PitchRangePresetNode, "PitchRangeByNoteNode": PitchRangeByNoteNode, @@ -408,6 +411,7 @@ NODE_CLASS_MAPPINGS = { "TimeFeatureNode": TimeFeatureNode, "ManualFeatureNode": ManualFeatureNode, "ManualFeatureFromPipe": ManualFeatureFromPipe, + "ManualFeatureDraw": ManualFeatureDraw, "DepthFeatureNode": DepthFeatureNode, "ColorFeatureNode": ColorFeatureNode, "BrightnessFeatureNode": BrightnessFeatureNode, diff --git a/nodes/depth/depth_base.py b/nodes/depth/depth_base.py index fa8b4ee..49b4886 100644 --- a/nodes/depth/depth_base.py +++ b/nodes/depth/depth_base.py @@ -46,18 +46,12 @@ class FlexDepthBase(RyanOnTheInside, FlexBase): feature_param, feature_mode, opt_feature=None, - opt_feature_pipe=None, **kwargs ): - if (opt_feature is None) != (opt_feature_pipe is None): - raise ValueError( - "Both opt_feature and opt_feature_pipe must be provided together, or neither should be provided." - ) - num_frames = depth_maps.shape[0] depth_maps_np = depth_maps.cpu().numpy() - if opt_feature is None and opt_feature_pipe is None: + if opt_feature is None: self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}") result = [] @@ -74,7 +68,7 @@ class FlexDepthBase(RyanOnTheInside, FlexBase): result.append(processed_depth_map) self.update_progress() else: - num_frames = opt_feature_pipe.frame_count + num_frames = opt_feature.frame_count self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}") result = [] diff --git a/nodes/flex/feature_extractors.py b/nodes/flex/feature_extractors.py index ebd98ac..683622f 100644 --- a/nodes/flex/feature_extractors.py +++ b/nodes/flex/feature_extractors.py @@ -21,6 +21,10 @@ class FeatureExtractorBase(RyanOnTheInside, ABC): return { "required": { "extraction_method": (feature_class.get_extraction_methods(), {"default": feature_class.get_extraction_methods()[0]}), + "frame_rate": ("FLOAT", {"default": 30.0, "min": 1.0, "max": 120.0, "step": 0.1}), + "frame_count": ("INT", {"default": 30, "min": 1}), + "width": ("INT", {"default": 512, "min": 64, "max": 4096, "step": 64}), + "height": ("INT", {"default": 512, "min": 64, "max": 4096, "step": 64}), } } @@ -50,6 +54,7 @@ class FeatureExtractorBase(RyanOnTheInside, ABC): self.tqdm_bar = None self.current_progress = 0 self.total_steps = 0 + CATEGORY="RyanOnTheInside/FlexFeatures" class ManualFeatureNode(FeatureExtractorBase): @@ -62,32 +67,29 @@ class ManualFeatureNode(FeatureExtractorBase): return { **super().INPUT_TYPES(), "required": { - "frame_rate": ("FLOAT", {"default": 30.0, "min": 1.0, "max": 120.0, "step": 0.1}), - "frame_count": ("INT", {"default": 30, "min": 1}), + **super().INPUT_TYPES()["required"], "frame_numbers": ("STRING", {"default": "0,10,20"}), "values": ("STRING", {"default": "0.0,0.5,1.0"}), "last_value": ("FLOAT", {"default": 1.0}), - "width": ("INT", {"default": 1920, "min": 1}), - "height": ("INT", {"default": 1080, "min": 1}), "interpolation_method": (["none", "linear", "ease_in", "ease_out"], {"default": "none"}), } } - RETURN_TYPES = ("FEATURE", "FEATURE_PIPE") + RETURN_TYPES = ("FEATURE",) FUNCTION = "create_feature" - def _apply_interpolation(self, feature_pipe, frame_numbers, values, last_value, interpolation_method): + def _apply_interpolation(self, frame_count, frame_rate, frame_numbers, values, last_value, interpolation_method, height=None, width=None): """Shared interpolation logic""" - manual_feature = ManualFeature("manual_feature", feature_pipe.frame_rate, feature_pipe.frame_count, - frame_numbers[0], frame_numbers[-1], values[0], values[-1], - method=interpolation_method) + manual_feature = ManualFeature("manual_feature", frame_rate, frame_count, + frame_numbers[0], frame_numbers[-1], values[0], values[-1], + method=interpolation_method, height=height, width=width) - manual_feature.data = np.zeros(feature_pipe.frame_count, dtype=np.float32) + manual_feature.data = np.zeros(frame_count, dtype=np.float32) interpolation_kind = 'linear' if len(frame_numbers) < 3 else 'quadratic' if interpolation_method == 'none': for frame, value in zip(frame_numbers, values): - if 0 <= frame < feature_pipe.frame_count: + if 0 <= frame < frame_count: manual_feature.data[frame] = value else: raise ValueError(f"Frame number {frame} is out of bounds.") @@ -100,7 +102,7 @@ class ManualFeatureNode(FeatureExtractorBase): reversed_values = [values[-1] - (v - values[0]) for v in values] f = interp1d(frame_numbers, reversed_values, kind=interpolation_kind, fill_value="extrapolate") - x = np.arange(feature_pipe.frame_count) + x = np.arange(frame_count) manual_feature.data = f(x) if interpolation_method == 'ease_out': @@ -108,7 +110,7 @@ class ManualFeatureNode(FeatureExtractorBase): return manual_feature - def create_feature(self, frame_rate, frame_count, frame_numbers, values, last_value, width, height, interpolation_method): + def create_feature(self, frame_rate, frame_count, frame_numbers, values, last_value, width, height, interpolation_method, extraction_method): # Parse inputs frame_numbers = list(map(int, frame_numbers.split(','))) values = list(map(float, values.split(','))) @@ -120,14 +122,10 @@ class ManualFeatureNode(FeatureExtractorBase): if max(frame_numbers) >= frame_count: raise ValueError(f"Frame numbers must be less than frame_count ({frame_count})") - # Create feature pipe with specified frame_count - video_frames = torch.zeros((frame_count, height, width, 3), dtype=torch.float32) - feature_pipe = FeaturePipe(frame_rate, video_frames) - # Apply interpolation - manual_feature = self._apply_interpolation(feature_pipe, frame_numbers, values, last_value, interpolation_method) + manual_feature = self._apply_interpolation(frame_count, frame_rate, frame_numbers, values, last_value, interpolation_method, height, width) - return (manual_feature, feature_pipe) + return (manual_feature,) class ManualFeatureFromPipe(ManualFeatureNode): @classmethod @@ -157,31 +155,12 @@ class ManualFeatureFromPipe(ManualFeatureNode): values.append(last_value) # Apply interpolation using parent class method - manual_feature = self._apply_interpolation(feature_pipe, frame_numbers, values, last_value, interpolation_method) + manual_feature = self._apply_interpolation(feature_pipe.frame_count, feature_pipe.frame_rate, frame_numbers, values, last_value, interpolation_method) return (manual_feature, feature_pipe) -class FirstFeature(FeatureExtractorBase): - - @classmethod - def feature_type(cls) -> type[BaseFeature]: - pass - - @classmethod - def INPUT_TYPES(cls): - return { - **super().INPUT_TYPES(), - "required": { - **super().INPUT_TYPES()["required"], - "video_frames": ("IMAGE",), - "frame_rate": ("FLOAT", {"default": 30.0, "min": 1.0, "max": 120.0, "step": 0.1}), - } - } - - RETURN_TYPES = ("FEATURE", "FEATURE_PIPE") - FUNCTION = "create_feature" -class TimeFeatureNode(FirstFeature): +class TimeFeatureNode(FeatureExtractorBase): @classmethod def feature_type(cls) -> type[BaseFeature]: return TimeFeature @@ -197,16 +176,16 @@ class TimeFeatureNode(FirstFeature): } } - + RETURN_TYPES = ("FEATURE",) + FUNCTION = "create_feature" - def create_feature(self, extraction_method, speed, offset, video_frames, frame_rate): - feature_pipe = FeaturePipe(frame_rate, video_frames) - time_feature = TimeFeature("time_effect", feature_pipe.frame_rate, feature_pipe.frame_count, - effect_type=extraction_method, speed=speed, offset=offset) + def create_feature(self, extraction_method, speed, offset, frame_rate, frame_count, width, height): + time_feature = TimeFeature("time_effect", frame_rate, frame_count, width, height, + effect_type=extraction_method, speed=speed, offset=offset) time_feature.extract() - return (time_feature, feature_pipe) + return (time_feature,) -class DepthFeatureNode(FirstFeature): +class DepthFeatureNode(FeatureExtractorBase): @classmethod def feature_type(cls) -> type[BaseFeature]: return DepthFeature @@ -221,13 +200,15 @@ class DepthFeatureNode(FirstFeature): } } - def create_feature(self, depth_maps, frame_rate, video_frames, extraction_method): - feature_pipe = FeaturePipe(frame_rate, video_frames) - depth_feature = DepthFeature("depth_feature", feature_pipe.frame_rate, feature_pipe.frame_count, depth_maps, extraction_method) - depth_feature.extract() - return (depth_feature, feature_pipe) + RETURN_TYPES = ("FEATURE",) + FUNCTION = "create_feature" -class ColorFeatureNode(FirstFeature): + def create_feature(self, depth_maps, frame_rate, frame_count, width, height, extraction_method): + depth_feature = DepthFeature("depth_feature", frame_rate, frame_count, depth_maps, extraction_method, width=width, height=height) + depth_feature.extract() + return (depth_feature,) + +class ColorFeatureNode(FeatureExtractorBase): @classmethod def feature_type(cls) -> type[BaseFeature]: return ColorFeature @@ -238,16 +219,19 @@ class ColorFeatureNode(FirstFeature): **super().INPUT_TYPES(), "required": { **super().INPUT_TYPES()["required"], + "images": ("IMAGE",), } } - def create_feature(self, video_frames, frame_rate, extraction_method): - feature_pipe = FeaturePipe(frame_rate, video_frames) - color_feature = ColorFeature("color_feature", feature_pipe.frame_rate, feature_pipe.frame_count, video_frames, extraction_method) - color_feature.extract() - return (color_feature, feature_pipe) + RETURN_TYPES = ("FEATURE",) + FUNCTION = "create_feature" -class BrightnessFeatureNode(FirstFeature): + def create_feature(self, images, frame_rate, frame_count, width, height, extraction_method): + color_feature = ColorFeature("color_feature", frame_rate, frame_count, images, extraction_method, width=width, height=height) + color_feature.extract() + return (color_feature,) + +class BrightnessFeatureNode(FeatureExtractorBase): @classmethod def feature_type(cls) -> type[BaseFeature]: return BrightnessFeature @@ -258,16 +242,19 @@ class BrightnessFeatureNode(FirstFeature): **super().INPUT_TYPES(), "required": { **super().INPUT_TYPES()["required"], + "images": ("IMAGE",), } } - def create_feature(self, video_frames, frame_rate, extraction_method): - feature_pipe = FeaturePipe(frame_rate, video_frames) - brightness_feature = BrightnessFeature("brightness_feature", feature_pipe.frame_rate, feature_pipe.frame_count, video_frames, extraction_method) + RETURN_TYPES = ("FEATURE",) + FUNCTION = "create_feature" + + def create_feature(self, images, frame_rate, frame_count, width, height, extraction_method): + brightness_feature = BrightnessFeature("brightness_feature", frame_rate, frame_count, images, extraction_method, width=width, height=height) brightness_feature.extract() - return (brightness_feature, feature_pipe) + return (brightness_feature,) -class MotionFeatureNode(FirstFeature): +class MotionFeatureNode(FeatureExtractorBase): @classmethod def feature_type(cls) -> type[BaseFeature]: return MotionFeature @@ -278,42 +265,42 @@ class MotionFeatureNode(FirstFeature): **super().INPUT_TYPES(), "required": { **super().INPUT_TYPES()["required"], + "images": ("IMAGE",), "flow_method": (["Farneback", "LucasKanade", "PyramidalLK"],), "flow_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 10.0, "step": 0.1}), "magnitude_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), } } - RETURN_TYPES = ("FEATURE", "FEATURE_PIPE") + RETURN_TYPES = ("FEATURE",) FUNCTION = "create_feature" - def create_feature(self, video_frames, frame_rate, extraction_method, flow_method, flow_threshold, magnitude_threshold): - feature_pipe = FeaturePipe(frame_rate, video_frames) - num_frames = feature_pipe.frame_count - - self.start_progress(num_frames, desc="Extracting motion features") - + def create_feature(self, images, frame_rate, frame_count, width, height, extraction_method, flow_method, flow_threshold, magnitude_threshold): def progress_callback(current_step, total_steps): self.update_progress(current_step - self.current_progress) + self.start_progress(frame_count, desc="Extracting motion features") + motion_feature = MotionFeature( "motion_feature", - feature_pipe.frame_rate, - feature_pipe.frame_count, - video_frames, + frame_rate, + frame_count, + images, extraction_method, flow_method, flow_threshold, magnitude_threshold, + width=width, + height=height, progress_callback=progress_callback ) motion_feature.extract() self.end_progress() - return (motion_feature, feature_pipe) + return (motion_feature,) -class AreaFeatureNode(FirstFeature): +class AreaFeatureNode(FeatureExtractorBase): @classmethod def feature_type(cls) -> type[BaseFeature]: return AreaFeature @@ -329,15 +316,14 @@ class AreaFeatureNode(FirstFeature): } } - RETURN_TYPES = ("FEATURE", "FEATURE_PIPE") + RETURN_TYPES = ("FEATURE",) FUNCTION = "create_feature" - def create_feature(self, masks, video_frames, frame_rate, extraction_method, threshold): - feature_pipe = FeaturePipe(frame_rate, video_frames) - area_feature = AreaFeature("area_feature", feature_pipe.frame_rate, feature_pipe.frame_count, - masks, feature_type=extraction_method, threshold=threshold) + def create_feature(self, masks, frame_rate, frame_count, width, height, extraction_method, threshold): + area_feature = AreaFeature("area_feature", frame_rate, frame_count, masks, extraction_method, threshold, width=width, height=height) area_feature.extract() - return (area_feature, feature_pipe) + return (area_feature,) + diff --git a/nodes/flex/feature_extractors_audio.py b/nodes/flex/feature_extractors_audio.py index 8fc3151..c5a8b8b 100644 --- a/nodes/flex/feature_extractors_audio.py +++ b/nodes/flex/feature_extractors_audio.py @@ -1,5 +1,4 @@ -from .feature_extractors import FeatureExtractorBase, FirstFeature -from .feature_pipe import FeaturePipe +from .feature_extractors import FeatureExtractorBase from .features_audio import AudioFeature, PitchFeature, PitchRange, BaseFeature, RhythmFeature from ... import RyanOnTheInside from ..audio.audio_nodes import AudioNodeBase @@ -16,18 +15,24 @@ class AudioFeatureExtractor(FeatureExtractorBase): "required": { **super().INPUT_TYPES()["required"], "audio": ("AUDIO",), - "feature_pipe": ("FEATURE_PIPE",), } } - RETURN_TYPES = ("FEATURE", "FEATURE_PIPE") + RETURN_TYPES = ("FEATURE",) FUNCTION = "extract_feature" - - def extract_feature(self, audio, feature_pipe, extraction_method): - feature = AudioFeature(extraction_method, audio, feature_pipe.frame_count, feature_pipe.frame_rate, extraction_method) + def extract_feature(self, audio, frame_rate, frame_count, width, height, extraction_method): + feature = AudioFeature( + feature_name=extraction_method, + audio=audio, + frame_count=frame_count, + frame_rate=frame_rate, + width=width, + height=height, + feature_type=extraction_method + ) feature.extract() - return (feature, feature_pipe) + return (feature,) #todo: create base class in prep for version 2 class AudioFeatureExtractorFirst(FeatureExtractorBase): @@ -42,31 +47,28 @@ class AudioFeatureExtractorFirst(FeatureExtractorBase): "required": { **super().INPUT_TYPES()["required"], "audio": ("AUDIO",), - "width": ("INT", {"default": 512, "min": 64, "max": 4096, "step": 64}), - "height": ("INT", {"default": 512, "min": 64, "max": 4096, "step": 64}), - "frame_rate": ("FLOAT", {"default": 30.0, "min": 0.1, "max": 240.0, "step": 0.1}), } } - RETURN_TYPES = ("FEATURE", "FEATURE_PIPE", "INT") - RETURN_NAMES = ("feature", "feature_pipe", "frame_count") + RETURN_TYPES = ("FEATURE", "INT") + RETURN_NAMES = ("feature", "frame_count") FUNCTION = "extract_feature" CATEGORY = "RyanOnTheInside/FlexFeatures/Audio" - def extract_feature(self, audio, width, height, frame_rate, extraction_method): - empty_frames = AudioNodeBase.create_empty_tensor(audio, frame_rate, height, width, channels=3) - feature_pipe = FeaturePipe(frame_rate, empty_frames) + def extract_feature(self, audio, frame_rate, frame_count, width, height, extraction_method): feature = AudioFeature( feature_name=extraction_method, audio=audio, - frame_count=feature_pipe.frame_count, - frame_rate=feature_pipe.frame_rate, - feature_type=extraction_method + frame_count=frame_count, + frame_rate=frame_rate, + feature_type=extraction_method, + width=width, + height=height ) feature.extract() - return (feature, feature_pipe, feature_pipe.frame_count) + return (feature, frame_count) -class RhythmFeatureExtractor(FirstFeature): +class RhythmFeatureExtractor(FeatureExtractorBase): @classmethod def feature_type(cls) -> type[BaseFeature]: return RhythmFeature @@ -82,27 +84,28 @@ class RhythmFeatureExtractor(FirstFeature): }, } - RETURN_TYPES = ("FEATURE", "FEATURE_PIPE") + RETURN_TYPES = ("FEATURE",) FUNCTION = "extract_feature" CATEGORY = "RyanOnTheInside/FlexFeatures/Audio/Rhythm" - def extract_feature(self, audio, extraction_method, time_signature, frame_rate, video_frames): - feature_pipe = FeaturePipe(frame_rate, video_frames) + def extract_feature(self, audio, extraction_method, time_signature, frame_rate, frame_count, width, height): feature = RhythmFeature( feature_name=extraction_method, audio=audio, - frame_count=feature_pipe.frame_count, - frame_rate=feature_pipe.frame_rate, + frame_count=frame_count, + frame_rate=frame_rate, feature_type=extraction_method, - time_signature=time_signature + time_signature=time_signature, + width=width, + height=height ) feature.extract() - return (feature, feature_pipe) + return (feature,) class PitchFeatureExtractor(FeatureExtractorBase): @classmethod def feature_type(cls) -> type[BaseFeature]: - return PitchFeature + return PitchFeature @classmethod def INPUT_TYPES(cls): @@ -111,7 +114,6 @@ class PitchFeatureExtractor(FeatureExtractorBase): "required": { **super().INPUT_TYPES()["required"], "audio": ("AUDIO",), - "feature_pipe": ("FEATURE_PIPE",), "opt_crepe_model":(["none", "medium", "tiny", "small", "large", "full"], {"default": "medium"}) }, "optional": { @@ -120,23 +122,25 @@ class PitchFeatureExtractor(FeatureExtractorBase): }, } - RETURN_TYPES = ("FEATURE", "FEATURE_PIPE") + RETURN_TYPES = ("FEATURE",) FUNCTION = "extract_feature" - def extract_feature(self, audio, feature_pipe, extraction_method, opt_pitch_range_collections=None, opt_crepe_model=None): + def extract_feature(self, audio, frame_rate, frame_count, width, height, extraction_method, opt_pitch_range_collections=None, opt_crepe_model=None): if opt_pitch_range_collections is None: opt_pitch_range_collections = [] feature = PitchFeature( feature_name=extraction_method, audio=audio, - frame_count=feature_pipe.frame_count, - frame_rate=feature_pipe.frame_rate, + frame_count=frame_count, + frame_rate=frame_rate, pitch_range_collections=opt_pitch_range_collections, feature_type=extraction_method, - crepe_model=opt_crepe_model + crepe_model=opt_crepe_model, + width=width, + height=height ) feature.extract() - return (feature, feature_pipe) + return (feature,) class PitchAbstraction(RyanOnTheInside): CATEGORY="RyanOnTheInside/FlexFeatures/Audio/Pitch" @@ -276,5 +280,4 @@ class PitchRangeByNoteNode(PitchAbstraction): def _midi_to_frequency(self, midi_note): import librosa return librosa.midi_to_hz(midi_note) - - # The _calculate_tolerance method has been removed from here \ No newline at end of file + # The _calculate_tolerance method has been removed from here diff --git a/nodes/flex/feature_extractors_midi.py b/nodes/flex/feature_extractors_midi.py index 9a390d0..f7d6423 100644 --- a/nodes/flex/feature_extractors_midi.py +++ b/nodes/flex/feature_extractors_midi.py @@ -1,6 +1,5 @@ import mido import os -from .feature_pipe import FeaturePipe import folder_paths from server import PromptServer from aiohttp import web @@ -12,9 +11,7 @@ class MIDILoadAndExtract(FeatureExtractorBase): @classmethod def feature_type(cls) -> type[MIDIFeature]: return MIDIFeature - - @classmethod def INPUT_TYPES(cls): return { @@ -22,18 +19,16 @@ class MIDILoadAndExtract(FeatureExtractorBase): **super().INPUT_TYPES()["required"], "midi_file": (folder_paths.get_filename_list("midi_files"),), "track_selection": (["all"],), - "frame_rate": ("FLOAT", {"default": 30, "min": 0.1, "max": 120, "step": 0.1}), - "video_frames": ("IMAGE",), "chord_only": ("BOOLEAN", {"default": False}), "notes": ("STRING", {"default": ""}), }, } - RETURN_TYPES = ("MIDI", "FEATURE", "FEATURE_PIPE") + RETURN_TYPES = ("MIDI", "FEATURE") FUNCTION = "process_midi" - CATEGORY = "RyanOnTheInside/Audio" + CATEGORY = "RyanOnTheInside/FlexFeatures/MIDI" - def process_midi(self, midi_file, track_selection, notes, extraction_method, frame_rate, video_frames, chord_only=False): + def process_midi(self, midi_file, track_selection, notes, extraction_method, frame_rate, frame_count, width, height, chord_only=False): try: midi_path = folder_paths.get_full_path("midi_files", midi_file) if not midi_path or not os.path.exists(midi_path): @@ -42,23 +37,27 @@ class MIDILoadAndExtract(FeatureExtractorBase): midi_data = mido.MidiFile(midi_path) selected_notes = [int(n.strip()) for n in notes.split(',') if n.strip().isdigit()] - feature_pipe = FeaturePipe(frame_rate, video_frames) # Convert friendly attribute name to internal attribute name internal_attribute = MIDIFeature.get_attribute_value(extraction_method) - feature = MIDIFeature(f"midi_{internal_attribute}", midi_data, internal_attribute, - feature_pipe.frame_rate, feature_pipe.frame_count, - notes=selected_notes, chord_only=chord_only) + feature = MIDIFeature( + f"midi_{internal_attribute}", + midi_data, + internal_attribute, + frame_rate, + frame_count, + notes=selected_notes, + chord_only=chord_only, + width=width, + height=height + ) feature.extract() - return (midi_data, feature, feature_pipe) + return (midi_data, feature) except Exception as e: - # error_msg = f"Error in MIDILoadAndExtract.process_midi: {type(e).__name__}: {str(e)}\n" - # error_msg += traceback.format_exc() - # print(error_msg) raise RuntimeError(f"Error processing MIDI file: {type(e).__name__}: {str(e)}") @classmethod diff --git a/nodes/flex/feature_extractors_proximity.py b/nodes/flex/feature_extractors_proximity.py index 5f048ea..d18e9e5 100644 --- a/nodes/flex/feature_extractors_proximity.py +++ b/nodes/flex/feature_extractors_proximity.py @@ -1,40 +1,45 @@ -from .features_proximity import Location, ProximityFeature -from .feature_extractors import FirstFeature +from .features_proximity import Location, ProximityFeature +from .feature_extractors import FeatureExtractorBase from ... import RyanOnTheInside -from .feature_pipe import FeaturePipe import numpy as np import cv2 -class ProximityFeatureNode(FirstFeature): +class ProximityFeatureNode(FeatureExtractorBase): + @classmethod + def feature_type(cls) -> type: + return ProximityFeature + @classmethod def INPUT_TYPES(cls): return { + **super().INPUT_TYPES(), "required": { - "video_frames": ("IMAGE",), - "frame_rate": ("FLOAT", {"default": 30.0, "min": 1.0, "max": 120.0, "step": 0.1}), + **super().INPUT_TYPES()["required"], "anchor_locations": ("LOCATION",), "query_locations": ("LOCATION",), "normalization_method": (["frame", "minmax"],), } } - RETURN_TYPES = ("FEATURE", "FEATURE_PIPE") - RETURN_NAMES = ("proximity_feature", "feature_pipe") + RETURN_TYPES = ("FEATURE",) + RETURN_NAMES = ("proximity_feature",) FUNCTION = "create_feature" + CATEGORY = "RyanOnTheInside/FlexFeatures/Proximity" - def create_feature(self, video_frames, frame_rate, anchor_locations, query_locations, normalization_method): - feature_pipe = FeaturePipe(frame_rate, video_frames) - frame_dimensions = (video_frames.shape[2], video_frames.shape[1]) # width, height + def create_feature(self, frame_rate, frame_count, width, height, anchor_locations, query_locations, normalization_method, extraction_method): + frame_dimensions = (width, height) proximity_feature = ProximityFeature( name="proximity_feature", anchor_locations=anchor_locations, query_locations=query_locations, frame_rate=frame_rate, - frame_count=len(video_frames), + frame_count=frame_count, frame_dimensions=frame_dimensions, - normalization_method=normalization_method + normalization_method=normalization_method, + width=width, + height=height ) # Ensure anchor and query points have the same number of columns @@ -50,11 +55,9 @@ class ProximityFeatureNode(FirstFeature): location.points = location.points[:, :2] print("Depth ignored. Depth must be included for all points or no points.") - # self.start_progress(len(video_frames), desc="Calculating proximity") proximity_feature.extract() - # self.end_progress() - return (proximity_feature, feature_pipe) + return (proximity_feature,) class ProximityFeatureInput(RyanOnTheInside): CATEGORY="RyanOnTheInside/Proximity" diff --git a/nodes/flex/features.py b/nodes/flex/features.py index 4651d7a..578d5b9 100644 --- a/nodes/flex/features.py +++ b/nodes/flex/features.py @@ -6,11 +6,13 @@ from ..masks.mask_utils import calculate_optical_flow from scipy.interpolate import interp1d, make_interp_spline class BaseFeature(ABC): - def __init__(self, name, feature_type, frame_rate, frame_count): + def __init__(self, name, feature_type, frame_rate, frame_count, width, height): self.name = name self.type = feature_type self.frame_rate = frame_rate self.frame_count = frame_count + self.width = width + self.height = height self.data = None self.features = None self.inverted = False @@ -54,8 +56,8 @@ class BaseFeature(ABC): return self class ManualFeature(BaseFeature): - def __init__(self, name, frame_rate, frame_count, start_frame, end_frame, start_value, end_value, method='linear'): - super().__init__(name, "manual", frame_rate, frame_count) + def __init__(self, name, frame_rate, frame_count, start_frame, end_frame, start_value, end_value, method='linear', width=None, height=None): + super().__init__(name, "manual", frame_rate, frame_count, width, height) self.start_frame = start_frame self.end_frame = end_frame self.start_value = start_value @@ -109,11 +111,11 @@ class ManualFeature(BaseFeature): return ease_out class TimeFeature(BaseFeature): - def __init__(self, name, frame_rate, frame_count, effect_type='smooth', speed=1.0, offset=0.0): + def __init__(self, name, frame_rate, frame_count, width, height, effect_type='smooth', speed=1.0, offset=0.0): + super().__init__(name, "time", frame_rate, frame_count, width, height) self.effect_type = effect_type self.speed = speed self.offset = offset - super().__init__(name, "time", frame_rate, frame_count) @classmethod def get_extraction_methods(self): @@ -141,12 +143,12 @@ class TimeFeature(BaseFeature): return self.normalize() class DepthFeature(BaseFeature): - def __init__(self, name, frame_rate, frame_count, depth_maps, feature_name='mean_depth'): + def __init__(self, name, frame_rate, frame_count, depth_maps, feature_name='mean_depth', width=None, height=None): + super().__init__(name, "depth", frame_rate, frame_count, width, height) self.depth_maps = depth_maps self.features = None self.feature_name = feature_name self.available_features = self.get_extraction_methods() - super().__init__(name, "depth", frame_rate, frame_count) @classmethod def get_extraction_methods(self): @@ -214,13 +216,13 @@ class DepthFeature(BaseFeature): raise ValueError(f"Invalid feature name. Available features are: {', '.join(self.available_features)}") class ColorFeature(BaseFeature): - def __init__(self, name, frame_rate, frame_count, images, feature_name='dominant_color'): + def __init__(self, name, frame_rate, frame_count, images, feature_name='dominant_color', width=None, height=None): if images.dim() != 4 or images.shape[-1] != 3: raise ValueError(f"Expected images in BHWC format, but got shape {images.shape}") + super().__init__(name, "color", frame_rate, frame_count, width, height) self.images = images self.feature_name = feature_name self.available_features = self.get_extraction_methods() - super().__init__(name, "color", frame_rate, frame_count) @classmethod def get_extraction_methods(self): @@ -298,11 +300,11 @@ class ColorFeature(BaseFeature): raise ValueError(f"Invalid feature name. Available features are: {', '.join(self.available_features)}") class BrightnessFeature(BaseFeature): - def __init__(self, name, frame_rate, frame_count, images, feature_name='mean_brightness'): + def __init__(self, name, frame_rate, frame_count, images, feature_name='mean_brightness', width=None, height=None): + super().__init__(name, "brightness", frame_rate, frame_count, width, height) self.images = images self.feature_name = feature_name self.available_features = self.get_extraction_methods() - super().__init__(name, "brightness", frame_rate, frame_count) @classmethod def get_extraction_methods(self): @@ -366,7 +368,8 @@ class BrightnessFeature(BaseFeature): raise ValueError(f"Invalid feature name. Available features are: {', '.join(self.available_features)}") class MotionFeature(BaseFeature): - def __init__(self, name, frame_rate, frame_count, images, feature_name='mean_motion', flow_method='Farneback', flow_threshold=0.0, magnitude_threshold=0.0, progress_callback=None): + def __init__(self, name, frame_rate, frame_count, images, feature_name='mean_motion', flow_method='Farneback', flow_threshold=0.0, magnitude_threshold=0.0, width=None, height=None, progress_callback=None): + super().__init__(name, "motion", frame_rate, frame_count, width, height) self.images = images self.feature_name = feature_name self.flow_method = flow_method @@ -374,7 +377,6 @@ class MotionFeature(BaseFeature): self.magnitude_threshold = magnitude_threshold self.available_features = self.get_extraction_methods() self.progress_callback = progress_callback - super().__init__(name, "motion", frame_rate, frame_count) @classmethod def get_extraction_methods(self): @@ -465,12 +467,12 @@ class MotionFeature(BaseFeature): raise ValueError(f"Invalid feature name. Available features are: {', '.join(self.available_features)}") class AreaFeature(BaseFeature): - def __init__(self, name, frame_rate, frame_count, masks, feature_type='total_area', threshold=0.5): + def __init__(self, name, frame_rate, frame_count, masks, feature_type='total_area', threshold=0.5, width=None, height=None): + super().__init__(name, "area", frame_rate, frame_count, width, height) self.masks = masks self.feature_type = feature_type self.threshold = threshold self.available_features = self.get_extraction_methods() - super().__init__(name, "area", frame_rate, frame_count) @classmethod def get_extraction_methods(self): @@ -524,7 +526,7 @@ class AreaFeature(BaseFeature): def set_active_feature(self, feature_name): if feature_name in self.available_features: - self.feature_type = feature_name + self.feature_name = feature_name self.extract() # Re-extract the data with the new feature type else: raise ValueError(f"Invalid feature name. Available features are: {', '.join(self.available_features)}") diff --git a/nodes/flex/features_audio.py b/nodes/flex/features_audio.py index 613bd72..2baf3cf 100644 --- a/nodes/flex/features_audio.py +++ b/nodes/flex/features_audio.py @@ -19,8 +19,8 @@ class BaseAudioFeature(BaseFeature): """Return a list of parameter names that can be modulated.""" return [] - def __init__(self, name, audio, frame_count, frame_rate): - super().__init__(name, "audio", frame_rate, frame_count) + def __init__(self, name, audio, frame_count, frame_rate, width, height): + super().__init__(name, "audio", frame_rate, frame_count, width, height) self.audio = audio self.sample_rate = None self.frame_duration = None @@ -68,8 +68,8 @@ class BaseAudioFeature(BaseFeature): class AudioFeature(BaseAudioFeature): - def __init__(self, feature_name, audio, frame_count, frame_rate, feature_type='amplitude_envelope'): - super().__init__(feature_name, audio, frame_count, frame_rate) + def __init__(self, feature_name, audio, frame_count, frame_rate, width, height, feature_type='amplitude_envelope'): + super().__init__(feature_name, audio, frame_count, frame_rate, width, height) self.feature_type = feature_type self.available_features = self.get_extraction_methods() self.feature_name = feature_type @@ -128,8 +128,8 @@ import numpy as np import librosa class RhythmFeature(BaseAudioFeature): - def __init__(self, feature_name, audio, frame_count, frame_rate, feature_type='beat_locations', time_signature=4): - super().__init__(feature_name, audio, frame_count, frame_rate) + def __init__(self, feature_name, audio, frame_count, frame_rate, width, height, feature_type='beat_locations', time_signature=4): + super().__init__(feature_name, audio, frame_count, frame_rate, width, height) self.feature_type = feature_type self.available_features = self.get_extraction_methods() self.feature_name = feature_type @@ -282,13 +282,15 @@ class PitchFeature(BaseAudioFeature): audio, frame_count, frame_rate, + width, + height, pitch_range_collections=None, feature_type='frequency', window_size=0, vibrato_options=None, crepe_model="none" ): - super().__init__(feature_name, audio, frame_count, frame_rate) + super().__init__(feature_name, audio, frame_count, frame_rate, width, height) self.available_features = self.get_extraction_methods() self.feature_type = feature_type self.pitch_range_collections = pitch_range_collections or [] diff --git a/nodes/flex/features_midi.py b/nodes/flex/features_midi.py index 8916d2a..30de863 100644 --- a/nodes/flex/features_midi.py +++ b/nodes/flex/features_midi.py @@ -26,8 +26,8 @@ class MIDIFeature(BaseFeature): """Return a list of parameter names that can be modulated.""" return list(cls.ATTRIBUTE_MAP.keys()) - def __init__(self, name, midi_data, attribute, frame_rate, frame_count, notes=None, chord_only=False): - super().__init__(name, "midi", frame_rate, frame_count) + def __init__(self, name, midi_data, attribute, frame_rate, frame_count, width, height, notes=None, chord_only=False): + super().__init__(name, "midi", frame_rate, frame_count, width, height) self.midi_data = midi_data self.attribute = attribute self.notes = set(notes) if notes is not None else set() diff --git a/nodes/flex/features_proximity.py b/nodes/flex/features_proximity.py index 2b941d4..84aae2e 100644 --- a/nodes/flex/features_proximity.py +++ b/nodes/flex/features_proximity.py @@ -6,14 +6,15 @@ import numpy as np from .features import BaseFeature class ProximityFeature(BaseFeature): - def __init__(self, name, anchor_locations, query_locations, frame_rate, frame_count, frame_dimensions, normalization_method='frame'): - super().__init__(name, "proximity", frame_rate, frame_count) + def __init__(self, name, anchor_locations, query_locations, frame_rate, frame_count, frame_dimensions, width, height, normalization_method='frame'): + super().__init__(name, "proximity", frame_rate, frame_count, width, height) self.anchor_locations = anchor_locations self.query_locations = query_locations self.frame_diagonal = np.sqrt(frame_dimensions[0]**2 + frame_dimensions[1]**2) self.proximity_values = None self.normalization_method = normalization_method + @classmethod def get_extraction_methods(self): return ["normalization_method"] diff --git a/nodes/flex/flex_base.py b/nodes/flex/flex_base.py index ac302bb..513609f 100644 --- a/nodes/flex/flex_base.py +++ b/nodes/flex/flex_base.py @@ -18,7 +18,6 @@ class FlexBase(ABC): }, "optional": { "opt_feature": ("FEATURE",), - "opt_feature_pipe": ("FEATURE_PIPE",), } } diff --git a/nodes/images/flex_image_base.py b/nodes/images/flex_image_base.py index 33469b6..9e30661 100644 --- a/nodes/images/flex_image_base.py +++ b/nodes/images/flex_image_base.py @@ -55,16 +55,10 @@ class FlexImageBase(RyanOnTheInside, FlexBase): feature_param, feature_mode, opt_feature=None, - opt_feature_pipe=None, **kwargs ): - if (opt_feature is None) != (opt_feature_pipe is None): - raise ValueError( - "Both opt_feature and opt_feature_pipe must be provided together, or neither should be provided." - ) - - if opt_feature is None and opt_feature_pipe is None: - # Process all frames without modulation + # Process all frames without modulation if no feature is provided + if opt_feature is None: num_frames = images.shape[0] images_np = images.cpu().numpy() @@ -84,8 +78,8 @@ class FlexImageBase(RyanOnTheInside, FlexBase): self.update_progress() else: # Process frames with modulation based on feature values - num_frames = opt_feature_pipe.frame_count images_np = images.cpu().numpy() + num_frames = opt_feature.frame_count self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}") diff --git a/nodes/latents/flex_latent_base.py b/nodes/latents/flex_latent_base.py index 5ed0905..fe1aefb 100644 --- a/nodes/latents/flex_latent_base.py +++ b/nodes/latents/flex_latent_base.py @@ -28,14 +28,8 @@ class FlexLatentBase(RyanOnTheInside, FlexBase): feature_param, feature_mode, opt_feature=None, - opt_feature_pipe=None, **kwargs ): - if (opt_feature is None) != (opt_feature_pipe is None): - raise ValueError( - "Both opt_feature and opt_feature_pipe must be provided together, or neither should be provided." - ) - latents_np = latents["samples"].cpu().numpy() num_frames = latents_np.shape[0] self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}") @@ -46,7 +40,7 @@ class FlexLatentBase(RyanOnTheInside, FlexBase): feature_value = 1.0 # Default feature value apply_effect = True # Default to applying the effect - if opt_feature is not None and opt_feature_pipe is not None: + if opt_feature is not None: feature_value = opt_feature.get_value_at_frame(i) apply_effect = feature_value >= feature_threshold diff --git a/nodes/masks/flex_mask_base.py b/nodes/masks/flex_mask_base.py index 381d718..cd895d1 100644 --- a/nodes/masks/flex_mask_base.py +++ b/nodes/masks/flex_mask_base.py @@ -76,17 +76,12 @@ class FlexMaskBase(FlexBase, MaskBase): """ return self.process_mask(mask, feature_value, strength, **kwargs) - def apply_effect(self, masks, opt_feature=None, opt_feature_pipe=None, strength=1.0, feature_threshold=0.0, mask_strength=1.0, invert=False, subtract_original=0.0, grow_with_blur=0.0, **kwargs): + def apply_effect(self, masks, opt_feature=None, strength=1.0, feature_threshold=0.0, mask_strength=1.0, invert=False, subtract_original=0.0, grow_with_blur=0.0, **kwargs): """Main entry point for the Flex system. This method implements the required FlexBase.apply_effect method and routes to our mask-specific implementation. """ - if (opt_feature is None) != (opt_feature_pipe is None): - raise ValueError( - "Both feature and feature_pipe must be provided together, or neither should be provided." - ) - - num_frames = opt_feature_pipe.frame_count if opt_feature_pipe else masks.shape[0] + num_frames = masks.shape[0] original_masks = masks.clone() self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}") @@ -126,15 +121,14 @@ class FlexMaskBase(FlexBase, MaskBase): # Use mask_strength instead of strength for the final mask operation return (super().apply_mask_operation(processed_masks, original_masks, mask_strength, invert, subtract_original, grow_with_blur, **kwargs),) - def main_function(self, masks, opt_feature=None, opt_feature_pipe=None, strength=1.0, feature_threshold=0.0, mask_strength=1.0, invert=False, subtract_original=0.0, grow_with_blur=0.0, **kwargs): + def main_function(self, masks, opt_feature=None, strength=1.0, feature_threshold=0.0, mask_strength=1.0, invert=False, subtract_original=0.0, grow_with_blur=0.0, **kwargs): """Implementation of MaskBase's abstract main_function. Explicitly forwards all parameters to apply_effect to maintain parameter names. """ return self.apply_effect( masks=masks, - feature=opt_feature, - feature_pipe=opt_feature_pipe, + opt_feature=opt_feature, strength=strength, feature_threshold=feature_threshold, mask_strength=mask_strength, diff --git a/nodes/masks/flex_masks_normal.py b/nodes/masks/flex_masks_normal.py index 80c09d9..3d338af 100644 --- a/nodes/masks/flex_masks_normal.py +++ b/nodes/masks/flex_masks_normal.py @@ -23,7 +23,7 @@ class FlexMaskNormalBase(FlexMaskBase): normal_array = normal_map.cpu().numpy() * 2 - 1 return normal_array - def main_function(self, masks, feature, feature_pipe, strength, feature_threshold, invert, subtract_original, grow_with_blur, normal_map, **kwargs): + def main_function(self, masks, opt_feature=None, strength=1.0, feature_threshold=0.0, invert=False, subtract_original=0.0, grow_with_blur=0.0, normal_map=None, **kwargs): # This method should be overridden by child classes raise NotImplementedError("Subclasses must implement main_function") diff --git a/nodes/preprocessors/pose.py b/nodes/preprocessors/pose.py index 61241ab..b4de441 100644 --- a/nodes/preprocessors/pose.py +++ b/nodes/preprocessors/pose.py @@ -10,7 +10,6 @@ class PoseInterpolator(RyanOnTheInside): "pose_1": ("POSE_KEYPOINT",), "pose_2": ("POSE_KEYPOINT",), "feature": ("FEATURE",), - "feature_pipe": ("FEATURE_PIPE",), "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), "interpolation_mode": (["Linear", "Spherical"], {"default": "Linear"}), "omit_missing_points": ("BOOLEAN", {"default": False}), @@ -21,9 +20,9 @@ class PoseInterpolator(RyanOnTheInside): FUNCTION = "interpolate_poses" CATEGORY = "RyanOnTheInside/Poses" - def interpolate_poses(self, pose_1, pose_2, feature, feature_pipe, strength, interpolation_mode, omit_missing_points): + def interpolate_poses(self, pose_1, pose_2, feature, strength, interpolation_mode, omit_missing_points): print("Debug: Starting interpolate_poses method") - num_frames = feature_pipe.frame_count + num_frames = feature.frame_count self.progress_bar = ProgressBar(num_frames) result_poses = [] diff --git a/nodes/video/flex_video.py b/nodes/video/flex_video.py index 64f18fa..585505c 100644 --- a/nodes/video/flex_video.py +++ b/nodes/video/flex_video.py @@ -1,7 +1,6 @@ import numpy as np import torch from .video_base import FlexVideoBase -from ..flex.feature_pipe import FeaturePipe from scipy.interpolate import interp1d from ..masks.mask_utils import calculate_optical_flow import cv2 @@ -15,33 +14,15 @@ class FlexVideoDirection(FlexVideoBase): def get_modifiable_params(cls): return ["direction"] - @classmethod - def INPUT_TYPES(cls): - inputs = super().INPUT_TYPES() - # Make feature_pipe optional - inputs.setdefault("optional", {}) - inputs["optional"].update({ - "feature_pipe": ("FEATURE_PIPE",), - }) - return inputs - - def apply_effect_internal( self, video: np.ndarray, feature_values: np.ndarray, - feature_pipe=None, **kwargs, ) -> np.ndarray: num_frames = video.shape[0] - # Use the frame count from the feature pipe if provided, otherwise fallback to the input video length - if feature_pipe is not None: - target_frame_count = feature_pipe.frame_count - else: - target_frame_count = num_frames - - # Normalize feature values to the range [0, 1] over the length of the feature pipe (or input video) + # Normalize feature values to the range [0, 1] normalized_features = np.clip(feature_values, 0.0, 1.0) # Map feature values to frame indices in the input video @@ -50,16 +31,6 @@ class FlexVideoDirection(FlexVideoBase): # Ensure frame indices stay within valid bounds frame_indices = np.clip(frame_indices, 0, num_frames - 1) - # If a feature pipe is provided, the output video should be the length of the feature pipe - if target_frame_count != num_frames: - # Adjust the output video length to match the feature pipe length - # Select frames based on the feature value mapping - frame_indices = np.interp( - np.linspace(0, 1, target_frame_count), - np.linspace(0, 1, len(feature_values)), - frame_indices - ).astype(int) - # Create the processed video by selecting frames based on the mapped indices processed_video = video[frame_indices] diff --git a/nodes/video/flex_video_speed.py b/nodes/video/flex_video_speed.py index b15e977..7e65615 100644 --- a/nodes/video/flex_video_speed.py +++ b/nodes/video/flex_video_speed.py @@ -8,7 +8,6 @@ import torch from .vfi_utils import preprocess_frames, postprocess_frames from .video_base import FlexVideoBase import numpy as np -from ..flex.feature_pipe import FeaturePipe from scipy.interpolate import interp1d from ..masks.mask_utils import calculate_optical_flow import cv2 @@ -31,7 +30,6 @@ class FlexVideoSpeed(FlexVideoBase): return { "required": { **super().INPUT_TYPES()["required"], - "feature_pipe": ("FEATURE_PIPE",), "speed_factor": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.1}), "interpolation_mode": (["none", "linear", "Farneback", "rife47", "rife49"],), "fast_mode": ("BOOLEAN", {"default": True}), @@ -44,12 +42,12 @@ class FlexVideoSpeed(FlexVideoBase): def get_modifiable_params(cls): return ["speed_factor"] - def apply_effect_internal(self, video: np.ndarray, feature_values: np.ndarray, feature_pipe: FeaturePipe, + def apply_effect_internal(self, video: np.ndarray, feature_values: np.ndarray, speed_factor: float, interpolation_mode: str, fast_mode: bool, ensemble: bool, - scale_factor: float, **kwargs): + scale_factor: float, opt_feature=None, **kwargs): num_frames = video.shape[0] - total_duration = feature_pipe.frame_count / feature_pipe.frame_rate - frame_rate = feature_pipe.frame_rate + frame_rate = opt_feature.frame_rate + total_duration = num_frames / frame_rate # Ensure feature_values is the same length as video frames if len(feature_values) != num_frames: @@ -71,7 +69,7 @@ class FlexVideoSpeed(FlexVideoBase): normalized_cumulative_times = cumulative_times * (total_duration / cumulative_times[-1]) # Create an array of the target timestamps that the final video must have - target_timestamps = np.linspace(0, total_duration, feature_pipe.frame_count) + target_timestamps = np.linspace(0, total_duration, num_frames) # Interpolate the adjusted frames based on the normalized timestamps frame_indices = np.interp(target_timestamps, normalized_cumulative_times, np.arange(num_frames)) diff --git a/nodes/video/video_base.py b/nodes/video/video_base.py index 31c4630..bc052bb 100644 --- a/nodes/video/video_base.py +++ b/nodes/video/video_base.py @@ -1,64 +1,33 @@ import torch import numpy as np from abc import ABC, abstractmethod -from ... import RyanOnTheInside +from ..flex.flex_base import FlexBase from comfy.utils import ProgressBar -class FlexVideoBase(RyanOnTheInside, ABC): +class FlexVideoBase(FlexBase, ABC): @classmethod def INPUT_TYPES(cls): return { + **super().INPUT_TYPES(), "required": { + **super().INPUT_TYPES()["required"], "images": ("IMAGE",), - "feature": ("FEATURE",), - "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.01}), - "feature_mode": (["relative", "absolute"], {"default": "relative"}), - "feature_param": (cls.get_modifiable_params(),), - "feature_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), - }, - "optional": { - "feature_pipe": ("FEATURE_PIPE",) } } - CATEGORY = "RyanOnTheInside/FlexVideo" + CATEGORY = "RyanOnTheInside/FlexBase" RETURN_TYPES = ("IMAGE",) FUNCTION = "apply_effect" - - def __init__(self): - self.progress_bar = None - def start_progress(self, total_steps, desc="Processing"): - self.progress_bar = ProgressBar(total_steps) + def apply_effect(self, images, strength, feature_mode, feature_threshold, opt_feature=None, **kwargs): + if opt_feature is None: + return (images,) - def update_progress(self): - if self.progress_bar: - self.progress_bar.update(1) - - def end_progress(self): - self.progress_bar = None - - @classmethod - @abstractmethod - def get_modifiable_params(cls): - """Return a list of parameter names that can be modulated.""" - return [] - - #TODO implement parame name - def modulate_param(self, param_name, param_value, feature_value, strength, mode): - if mode == "relative": - # Adjust parameter relative to its value and the feature - return param_value * (1 + (feature_value - 0.5) * 2 * strength) - else: # absolute - # Adjust parameter directly based on the feature - return param_value * feature_value * strength - - def apply_effect(self, images, feature, strength, feature_mode, feature_threshold, feature_pipe=None, **kwargs): images_np = images.cpu().numpy() # Convert tensor to numpy array num_frames = images_np.shape[0] - target_frame_count = feature_pipe.frame_count if feature_pipe is not None else num_frames - feature_values = np.array([feature.get_value_at_frame(i) for i in range(target_frame_count)]) + # Get feature values for each frame + feature_values = np.array([opt_feature.get_value_at_frame(i) for i in range(num_frames)]) # Apply threshold to feature values feature_values[feature_values < feature_threshold] = 0 @@ -71,7 +40,7 @@ class FlexVideoBase(RyanOnTheInside, ABC): kwargs[param_name] = self.modulate_param(param_name, param_value, avg_feature_value, strength, feature_mode) # Apply the effect to the entire video - processed_video = self.apply_effect_internal(images_np, feature_values=feature_values, feature_pipe=feature_pipe, **kwargs) + processed_video = self.apply_effect_internal(images_np, feature_values=feature_values, **kwargs) # Convert the numpy array back to a tensor and ensure it's in BHWC format result_tensor = torch.from_numpy(processed_video).float()