nodes/ryanontheinside-ComfyUI_RyanOnTheInside
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feature pipe removed

Browse Source
This commit is contained in:
ryanontheinside
2025-01-04 03:34:49 +00:00
parent ecc05c2b1d
commit 2adfd1ca8e
19 changed files with 209 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files
__init__.py
+4
View File
@@ -101,6 +101,7 @@ from .nodes.flex.feature_extractors import(
AreaFeatureNode,
ManualFeatureNode,
ManualFeatureFromPipe,
ManualFeatureDraw,
)
from .nodes.flex.feature_extractors_audio import(
@@ -111,6 +112,7 @@ from .nodes.flex.feature_extractors_audio import(
PitchRangeByNoteNode,
PitchFeatureExtractor,
RhythmFeatureExtractor,
)
from .nodes.flex.feature_extractors_midi import(
@@ -401,6 +403,7 @@ NODE_CLASS_MAPPINGS = {
"AudioFeatureExtractorFirst": AudioFeatureExtractorFirst,
"PitchFeatureExtractor": PitchFeatureExtractor,
"RhythmFeatureExtractor": RhythmFeatureExtractor,
"PitchRange": PitchRangeNode,
"PitchRangePreset": PitchRangePresetNode,
"PitchRangeByNoteNode": PitchRangeByNoteNode,
@@ -408,6 +411,7 @@ NODE_CLASS_MAPPINGS = {
"TimeFeatureNode": TimeFeatureNode,
"ManualFeatureNode": ManualFeatureNode,
"ManualFeatureFromPipe": ManualFeatureFromPipe,
"ManualFeatureDraw": ManualFeatureDraw,
"DepthFeatureNode": DepthFeatureNode,
"ColorFeatureNode": ColorFeatureNode,
"BrightnessFeatureNode": BrightnessFeatureNode,
nodes/depth/depth_base.py
+2 -8
View File
@@ -46,18 +46,12 @@ class FlexDepthBase(RyanOnTheInside, FlexBase):
feature_param,
feature_mode,
opt_feature=None,
opt_feature_pipe=None,
**kwargs
):
if (opt_feature is None) != (opt_feature_pipe is None):
raise ValueError(
"Both opt_feature and opt_feature_pipe must be provided together, or neither should be provided."
)
num_frames = depth_maps.shape[0]
depth_maps_np = depth_maps.cpu().numpy()
if opt_feature is None and opt_feature_pipe is None:
if opt_feature is None:
self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}")
result = []
@@ -74,7 +68,7 @@ class FlexDepthBase(RyanOnTheInside, FlexBase):
result.append(processed_depth_map)
self.update_progress()
else:
num_frames = opt_feature_pipe.frame_count
num_frames = opt_feature.frame_count
self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}")
result = []
nodes/flex/feature_extractors.py
+68 -82
View File
@@ -21,6 +21,10 @@ class FeatureExtractorBase(RyanOnTheInside, ABC):
return {
"required": {
"extraction_method": (feature_class.get_extraction_methods(), {"default": feature_class.get_extraction_methods()[0]}),
"frame_rate": ("FLOAT", {"default": 30.0, "min": 1.0, "max": 120.0, "step": 0.1}),
"frame_count": ("INT", {"default": 30, "min": 1}),
"width": ("INT", {"default": 512, "min": 64, "max": 4096, "step": 64}),
"height": ("INT", {"default": 512, "min": 64, "max": 4096, "step": 64}),
}
}
@@ -50,6 +54,7 @@ class FeatureExtractorBase(RyanOnTheInside, ABC):
self.tqdm_bar = None
self.current_progress = 0
self.total_steps = 0
CATEGORY="RyanOnTheInside/FlexFeatures"
class ManualFeatureNode(FeatureExtractorBase):
@@ -62,32 +67,29 @@ class ManualFeatureNode(FeatureExtractorBase):
return {
**super().INPUT_TYPES(),
"required": {
"frame_rate": ("FLOAT", {"default": 30.0, "min": 1.0, "max": 120.0, "step": 0.1}),
"frame_count": ("INT", {"default": 30, "min": 1}),
**super().INPUT_TYPES()["required"],
"frame_numbers": ("STRING", {"default": "0,10,20"}),
"values": ("STRING", {"default": "0.0,0.5,1.0"}),
"last_value": ("FLOAT", {"default": 1.0}),
"width": ("INT", {"default": 1920, "min": 1}),
"height": ("INT", {"default": 1080, "min": 1}),
"interpolation_method": (["none", "linear", "ease_in", "ease_out"], {"default": "none"}),
}
}
RETURN_TYPES = ("FEATURE", "FEATURE_PIPE")
RETURN_TYPES = ("FEATURE",)
FUNCTION = "create_feature"
def _apply_interpolation(self, feature_pipe, frame_numbers, values, last_value, interpolation_method):
def _apply_interpolation(self, frame_count, frame_rate, frame_numbers, values, last_value, interpolation_method, height=None, width=None):
"""Shared interpolation logic"""
manual_feature = ManualFeature("manual_feature", feature_pipe.frame_rate, feature_pipe.frame_count,
frame_numbers[0], frame_numbers[-1], values[0], values[-1],
method=interpolation_method)
manual_feature = ManualFeature("manual_feature", frame_rate, frame_count,
frame_numbers[0], frame_numbers[-1], values[0], values[-1],
method=interpolation_method, height=height, width=width)
manual_feature.data = np.zeros(feature_pipe.frame_count, dtype=np.float32)
manual_feature.data = np.zeros(frame_count, dtype=np.float32)
interpolation_kind = 'linear' if len(frame_numbers) < 3 else 'quadratic'
if interpolation_method == 'none':
for frame, value in zip(frame_numbers, values):
if 0 <= frame < feature_pipe.frame_count:
if 0 <= frame < frame_count:
manual_feature.data[frame] = value
else:
raise ValueError(f"Frame number {frame} is out of bounds.")
@@ -100,7 +102,7 @@ class ManualFeatureNode(FeatureExtractorBase):
reversed_values = [values[-1] - (v - values[0]) for v in values]
f = interp1d(frame_numbers, reversed_values, kind=interpolation_kind, fill_value="extrapolate")
x = np.arange(feature_pipe.frame_count)
x = np.arange(frame_count)
manual_feature.data = f(x)
if interpolation_method == 'ease_out':
@@ -108,7 +110,7 @@ class ManualFeatureNode(FeatureExtractorBase):
return manual_feature
def create_feature(self, frame_rate, frame_count, frame_numbers, values, last_value, width, height, interpolation_method):
def create_feature(self, frame_rate, frame_count, frame_numbers, values, last_value, width, height, interpolation_method, extraction_method):
# Parse inputs
frame_numbers = list(map(int, frame_numbers.split(',')))
values = list(map(float, values.split(',')))
@@ -120,14 +122,10 @@ class ManualFeatureNode(FeatureExtractorBase):
if max(frame_numbers) >= frame_count:
raise ValueError(f"Frame numbers must be less than frame_count ({frame_count})")
# Create feature pipe with specified frame_count
video_frames = torch.zeros((frame_count, height, width, 3), dtype=torch.float32)
feature_pipe = FeaturePipe(frame_rate, video_frames)
# Apply interpolation
manual_feature = self._apply_interpolation(feature_pipe, frame_numbers, values, last_value, interpolation_method)
manual_feature = self._apply_interpolation(frame_count, frame_rate, frame_numbers, values, last_value, interpolation_method, height, width)
return (manual_feature, feature_pipe)
return (manual_feature,)
class ManualFeatureFromPipe(ManualFeatureNode):
@classmethod
@@ -157,31 +155,12 @@ class ManualFeatureFromPipe(ManualFeatureNode):
values.append(last_value)
# Apply interpolation using parent class method
manual_feature = self._apply_interpolation(feature_pipe, frame_numbers, values, last_value, interpolation_method)
manual_feature = self._apply_interpolation(feature_pipe.frame_count, feature_pipe.frame_rate, frame_numbers, values, last_value, interpolation_method)
return (manual_feature, feature_pipe)
class FirstFeature(FeatureExtractorBase):
@classmethod
def feature_type(cls) -> type[BaseFeature]:
pass
@classmethod
def INPUT_TYPES(cls):
return {
**super().INPUT_TYPES(),
"required": {
**super().INPUT_TYPES()["required"],
"video_frames": ("IMAGE",),
"frame_rate": ("FLOAT", {"default": 30.0, "min": 1.0, "max": 120.0, "step": 0.1}),
}
}
RETURN_TYPES = ("FEATURE", "FEATURE_PIPE")
FUNCTION = "create_feature"
class TimeFeatureNode(FirstFeature):
class TimeFeatureNode(FeatureExtractorBase):
@classmethod
def feature_type(cls) -> type[BaseFeature]:
return TimeFeature
@@ -197,16 +176,16 @@ class TimeFeatureNode(FirstFeature):
}
}
RETURN_TYPES = ("FEATURE",)
FUNCTION = "create_feature"
def create_feature(self, extraction_method, speed, offset, video_frames, frame_rate):
feature_pipe = FeaturePipe(frame_rate, video_frames)
time_feature = TimeFeature("time_effect", feature_pipe.frame_rate, feature_pipe.frame_count,
effect_type=extraction_method, speed=speed, offset=offset)
def create_feature(self, extraction_method, speed, offset, frame_rate, frame_count, width, height):
time_feature = TimeFeature("time_effect", frame_rate, frame_count, width, height,
effect_type=extraction_method, speed=speed, offset=offset)
time_feature.extract()
return (time_feature, feature_pipe)
return (time_feature,)
class DepthFeatureNode(FirstFeature):
class DepthFeatureNode(FeatureExtractorBase):
@classmethod
def feature_type(cls) -> type[BaseFeature]:
return DepthFeature
@@ -221,13 +200,15 @@ class DepthFeatureNode(FirstFeature):
}
}
def create_feature(self, depth_maps, frame_rate, video_frames, extraction_method):
feature_pipe = FeaturePipe(frame_rate, video_frames)
depth_feature = DepthFeature("depth_feature", feature_pipe.frame_rate, feature_pipe.frame_count, depth_maps, extraction_method)
depth_feature.extract()
return (depth_feature, feature_pipe)
RETURN_TYPES = ("FEATURE",)
FUNCTION = "create_feature"
class ColorFeatureNode(FirstFeature):
def create_feature(self, depth_maps, frame_rate, frame_count, width, height, extraction_method):
depth_feature = DepthFeature("depth_feature", frame_rate, frame_count, depth_maps, extraction_method, width=width, height=height)
depth_feature.extract()
return (depth_feature,)
class ColorFeatureNode(FeatureExtractorBase):
@classmethod
def feature_type(cls) -> type[BaseFeature]:
return ColorFeature
@@ -238,16 +219,19 @@ class ColorFeatureNode(FirstFeature):
**super().INPUT_TYPES(),
"required": {
**super().INPUT_TYPES()["required"],
"images": ("IMAGE",),
}
}
def create_feature(self, video_frames, frame_rate, extraction_method):
feature_pipe = FeaturePipe(frame_rate, video_frames)
color_feature = ColorFeature("color_feature", feature_pipe.frame_rate, feature_pipe.frame_count, video_frames, extraction_method)
color_feature.extract()
return (color_feature, feature_pipe)
RETURN_TYPES = ("FEATURE",)
FUNCTION = "create_feature"
class BrightnessFeatureNode(FirstFeature):
def create_feature(self, images, frame_rate, frame_count, width, height, extraction_method):
color_feature = ColorFeature("color_feature", frame_rate, frame_count, images, extraction_method, width=width, height=height)
color_feature.extract()
return (color_feature,)
class BrightnessFeatureNode(FeatureExtractorBase):
@classmethod
def feature_type(cls) -> type[BaseFeature]:
return BrightnessFeature
@@ -258,16 +242,19 @@ class BrightnessFeatureNode(FirstFeature):
**super().INPUT_TYPES(),
"required": {
**super().INPUT_TYPES()["required"],
"images": ("IMAGE",),
}
}
def create_feature(self, video_frames, frame_rate, extraction_method):
feature_pipe = FeaturePipe(frame_rate, video_frames)
brightness_feature = BrightnessFeature("brightness_feature", feature_pipe.frame_rate, feature_pipe.frame_count, video_frames, extraction_method)
RETURN_TYPES = ("FEATURE",)
FUNCTION = "create_feature"
def create_feature(self, images, frame_rate, frame_count, width, height, extraction_method):
brightness_feature = BrightnessFeature("brightness_feature", frame_rate, frame_count, images, extraction_method, width=width, height=height)
brightness_feature.extract()
return (brightness_feature, feature_pipe)
return (brightness_feature,)
class MotionFeatureNode(FirstFeature):
class MotionFeatureNode(FeatureExtractorBase):
@classmethod
def feature_type(cls) -> type[BaseFeature]:
return MotionFeature
@@ -278,42 +265,42 @@ class MotionFeatureNode(FirstFeature):
**super().INPUT_TYPES(),
"required": {
**super().INPUT_TYPES()["required"],
"images": ("IMAGE",),
"flow_method": (["Farneback", "LucasKanade", "PyramidalLK"],),
"flow_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 10.0, "step": 0.1}),
"magnitude_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
}
}
RETURN_TYPES = ("FEATURE", "FEATURE_PIPE")
RETURN_TYPES = ("FEATURE",)
FUNCTION = "create_feature"
def create_feature(self, video_frames, frame_rate, extraction_method, flow_method, flow_threshold, magnitude_threshold):
feature_pipe = FeaturePipe(frame_rate, video_frames)
num_frames = feature_pipe.frame_count
self.start_progress(num_frames, desc="Extracting motion features")
def create_feature(self, images, frame_rate, frame_count, width, height, extraction_method, flow_method, flow_threshold, magnitude_threshold):
def progress_callback(current_step, total_steps):
self.update_progress(current_step - self.current_progress)
self.start_progress(frame_count, desc="Extracting motion features")
motion_feature = MotionFeature(
"motion_feature",
feature_pipe.frame_rate,
feature_pipe.frame_count,
video_frames,
frame_rate,
frame_count,
images,
extraction_method,
flow_method,
flow_threshold,
magnitude_threshold,
width=width,
height=height,
progress_callback=progress_callback
)
motion_feature.extract()
self.end_progress()
return (motion_feature, feature_pipe)
return (motion_feature,)
class AreaFeatureNode(FirstFeature):
class AreaFeatureNode(FeatureExtractorBase):
@classmethod
def feature_type(cls) -> type[BaseFeature]:
return AreaFeature
@@ -329,15 +316,14 @@ class AreaFeatureNode(FirstFeature):
}
}
RETURN_TYPES = ("FEATURE", "FEATURE_PIPE")
RETURN_TYPES = ("FEATURE",)
FUNCTION = "create_feature"
def create_feature(self, masks, video_frames, frame_rate, extraction_method, threshold):
feature_pipe = FeaturePipe(frame_rate, video_frames)
area_feature = AreaFeature("area_feature", feature_pipe.frame_rate, feature_pipe.frame_count,
masks, feature_type=extraction_method, threshold=threshold)
def create_feature(self, masks, frame_rate, frame_count, width, height, extraction_method, threshold):
area_feature = AreaFeature("area_feature", frame_rate, frame_count, masks, extraction_method, threshold, width=width, height=height)
area_feature.extract()
return (area_feature, feature_pipe)
return (area_feature,)
nodes/flex/feature_extractors_audio.py
+41 -38
View File
@@ -1,5 +1,4 @@
from .feature_extractors import FeatureExtractorBase, FirstFeature
from .feature_pipe import FeaturePipe
from .feature_extractors import FeatureExtractorBase
from .features_audio import AudioFeature, PitchFeature, PitchRange, BaseFeature, RhythmFeature
from ... import RyanOnTheInside
from ..audio.audio_nodes import AudioNodeBase
@@ -16,18 +15,24 @@ class AudioFeatureExtractor(FeatureExtractorBase):
"required": {
**super().INPUT_TYPES()["required"],
"audio": ("AUDIO",),
"feature_pipe": ("FEATURE_PIPE",),
}
}
RETURN_TYPES = ("FEATURE", "FEATURE_PIPE")
RETURN_TYPES = ("FEATURE",)
FUNCTION = "extract_feature"
def extract_feature(self, audio, feature_pipe, extraction_method):
feature = AudioFeature(extraction_method, audio, feature_pipe.frame_count, feature_pipe.frame_rate, extraction_method)
def extract_feature(self, audio, frame_rate, frame_count, width, height, extraction_method):
feature = AudioFeature(
feature_name=extraction_method,
audio=audio,
frame_count=frame_count,
frame_rate=frame_rate,
width=width,
height=height,
feature_type=extraction_method
)
feature.extract()
return (feature, feature_pipe)
return (feature,)
#todo: create base class in prep for version 2
class AudioFeatureExtractorFirst(FeatureExtractorBase):
@@ -42,31 +47,28 @@ class AudioFeatureExtractorFirst(FeatureExtractorBase):
"required": {
**super().INPUT_TYPES()["required"],
"audio": ("AUDIO",),
"width": ("INT", {"default": 512, "min": 64, "max": 4096, "step": 64}),
"height": ("INT", {"default": 512, "min": 64, "max": 4096, "step": 64}),
"frame_rate": ("FLOAT", {"default": 30.0, "min": 0.1, "max": 240.0, "step": 0.1}),
}
}
RETURN_TYPES = ("FEATURE", "FEATURE_PIPE", "INT")
RETURN_NAMES = ("feature", "feature_pipe", "frame_count")
RETURN_TYPES = ("FEATURE", "INT")
RETURN_NAMES = ("feature", "frame_count")
FUNCTION = "extract_feature"
CATEGORY = "RyanOnTheInside/FlexFeatures/Audio"
def extract_feature(self, audio, width, height, frame_rate, extraction_method):
empty_frames = AudioNodeBase.create_empty_tensor(audio, frame_rate, height, width, channels=3)
feature_pipe = FeaturePipe(frame_rate, empty_frames)
def extract_feature(self, audio, frame_rate, frame_count, width, height, extraction_method):
feature = AudioFeature(
feature_name=extraction_method,
audio=audio,
frame_count=feature_pipe.frame_count,
frame_rate=feature_pipe.frame_rate,
feature_type=extraction_method
frame_count=frame_count,
frame_rate=frame_rate,
feature_type=extraction_method,
width=width,
height=height
)
feature.extract()
return (feature, feature_pipe, feature_pipe.frame_count)
return (feature, frame_count)
class RhythmFeatureExtractor(FirstFeature):
class RhythmFeatureExtractor(FeatureExtractorBase):
@classmethod
def feature_type(cls) -> type[BaseFeature]:
return RhythmFeature
@@ -82,27 +84,28 @@ class RhythmFeatureExtractor(FirstFeature):
},
}
RETURN_TYPES = ("FEATURE", "FEATURE_PIPE")
RETURN_TYPES = ("FEATURE",)
FUNCTION = "extract_feature"
CATEGORY = "RyanOnTheInside/FlexFeatures/Audio/Rhythm"
def extract_feature(self, audio, extraction_method, time_signature, frame_rate, video_frames):
feature_pipe = FeaturePipe(frame_rate, video_frames)
def extract_feature(self, audio, extraction_method, time_signature, frame_rate, frame_count, width, height):
feature = RhythmFeature(
feature_name=extraction_method,
audio=audio,
frame_count=feature_pipe.frame_count,
frame_rate=feature_pipe.frame_rate,
frame_count=frame_count,
frame_rate=frame_rate,
feature_type=extraction_method,
time_signature=time_signature
time_signature=time_signature,
width=width,
height=height
)
feature.extract()
return (feature, feature_pipe)
return (feature,)
class PitchFeatureExtractor(FeatureExtractorBase):
@classmethod
def feature_type(cls) -> type[BaseFeature]:
return PitchFeature
return PitchFeature
@classmethod
def INPUT_TYPES(cls):
@@ -111,7 +114,6 @@ class PitchFeatureExtractor(FeatureExtractorBase):
"required": {
**super().INPUT_TYPES()["required"],
"audio": ("AUDIO",),
"feature_pipe": ("FEATURE_PIPE",),
"opt_crepe_model":(["none", "medium", "tiny", "small", "large", "full"], {"default": "medium"})
},
"optional": {
@@ -120,23 +122,25 @@ class PitchFeatureExtractor(FeatureExtractorBase):
},
}
RETURN_TYPES = ("FEATURE", "FEATURE_PIPE")
RETURN_TYPES = ("FEATURE",)
FUNCTION = "extract_feature"
def extract_feature(self, audio, feature_pipe, extraction_method, opt_pitch_range_collections=None, opt_crepe_model=None):
def extract_feature(self, audio, frame_rate, frame_count, width, height, extraction_method, opt_pitch_range_collections=None, opt_crepe_model=None):
if opt_pitch_range_collections is None:
opt_pitch_range_collections = []
feature = PitchFeature(
feature_name=extraction_method,
audio=audio,
frame_count=feature_pipe.frame_count,
frame_rate=feature_pipe.frame_rate,
frame_count=frame_count,
frame_rate=frame_rate,
pitch_range_collections=opt_pitch_range_collections,
feature_type=extraction_method,
crepe_model=opt_crepe_model
crepe_model=opt_crepe_model,
width=width,
height=height
)
feature.extract()
return (feature, feature_pipe)
return (feature,)
class PitchAbstraction(RyanOnTheInside):
CATEGORY="RyanOnTheInside/FlexFeatures/Audio/Pitch"
@@ -276,5 +280,4 @@ class PitchRangeByNoteNode(PitchAbstraction):
def _midi_to_frequency(self, midi_note):
import librosa
return librosa.midi_to_hz(midi_note)
# The _calculate_tolerance method has been removed from here
# The _calculate_tolerance method has been removed from here
nodes/flex/feature_extractors_midi.py
+15 -16
View File
@@ -1,6 +1,5 @@
import mido
import os
from .feature_pipe import FeaturePipe
import folder_paths
from server import PromptServer
from aiohttp import web
@@ -12,9 +11,7 @@ class MIDILoadAndExtract(FeatureExtractorBase):
@classmethod
def feature_type(cls) -> type[MIDIFeature]:
return MIDIFeature
@classmethod
def INPUT_TYPES(cls):
return {
@@ -22,18 +19,16 @@ class MIDILoadAndExtract(FeatureExtractorBase):
**super().INPUT_TYPES()["required"],
"midi_file": (folder_paths.get_filename_list("midi_files"),),
"track_selection": (["all"],),
"frame_rate": ("FLOAT", {"default": 30, "min": 0.1, "max": 120, "step": 0.1}),
"video_frames": ("IMAGE",),
"chord_only": ("BOOLEAN", {"default": False}),
"notes": ("STRING", {"default": ""}),
},
}
RETURN_TYPES = ("MIDI", "FEATURE", "FEATURE_PIPE")
RETURN_TYPES = ("MIDI", "FEATURE")
FUNCTION = "process_midi"
CATEGORY = "RyanOnTheInside/Audio"
CATEGORY = "RyanOnTheInside/FlexFeatures/MIDI"
def process_midi(self, midi_file, track_selection, notes, extraction_method, frame_rate, video_frames, chord_only=False):
def process_midi(self, midi_file, track_selection, notes, extraction_method, frame_rate, frame_count, width, height, chord_only=False):
try:
midi_path = folder_paths.get_full_path("midi_files", midi_file)
if not midi_path or not os.path.exists(midi_path):
@@ -42,23 +37,27 @@ class MIDILoadAndExtract(FeatureExtractorBase):
midi_data = mido.MidiFile(midi_path)
selected_notes = [int(n.strip()) for n in notes.split(',') if n.strip().isdigit()]
feature_pipe = FeaturePipe(frame_rate, video_frames)
# Convert friendly attribute name to internal attribute name
internal_attribute = MIDIFeature.get_attribute_value(extraction_method)
feature = MIDIFeature(f"midi_{internal_attribute}", midi_data, internal_attribute,
feature_pipe.frame_rate, feature_pipe.frame_count,
notes=selected_notes, chord_only=chord_only)
feature = MIDIFeature(
f"midi_{internal_attribute}",
midi_data,
internal_attribute,
frame_rate,
frame_count,
notes=selected_notes,
chord_only=chord_only,
width=width,
height=height
)
feature.extract()
return (midi_data, feature, feature_pipe)
return (midi_data, feature)
except Exception as e:
# error_msg = f"Error in MIDILoadAndExtract.process_midi: {type(e).__name__}: {str(e)}\n"
# error_msg += traceback.format_exc()
# print(error_msg)
raise RuntimeError(f"Error processing MIDI file: {type(e).__name__}: {str(e)}")
@classmethod
nodes/flex/feature_extractors_proximity.py
+19 -16
View File
@@ -1,40 +1,45 @@
from .features_proximity import Location, ProximityFeature
from .feature_extractors import FirstFeature
from .features_proximity import Location, ProximityFeature
from .feature_extractors import FeatureExtractorBase
from ... import RyanOnTheInside
from .feature_pipe import FeaturePipe
import numpy as np
import cv2
class ProximityFeatureNode(FirstFeature):
class ProximityFeatureNode(FeatureExtractorBase):
@classmethod
def feature_type(cls) -> type:
return ProximityFeature
@classmethod
def INPUT_TYPES(cls):
return {
**super().INPUT_TYPES(),
"required": {
"video_frames": ("IMAGE",),
"frame_rate": ("FLOAT", {"default": 30.0, "min": 1.0, "max": 120.0, "step": 0.1}),
**super().INPUT_TYPES()["required"],
"anchor_locations": ("LOCATION",),
"query_locations": ("LOCATION",),
"normalization_method": (["frame", "minmax"],),
}
}
RETURN_TYPES = ("FEATURE", "FEATURE_PIPE")
RETURN_NAMES = ("proximity_feature", "feature_pipe")
RETURN_TYPES = ("FEATURE",)
RETURN_NAMES = ("proximity_feature",)
FUNCTION = "create_feature"
CATEGORY = "RyanOnTheInside/FlexFeatures/Proximity"
def create_feature(self, video_frames, frame_rate, anchor_locations, query_locations, normalization_method):
feature_pipe = FeaturePipe(frame_rate, video_frames)
frame_dimensions = (video_frames.shape[2], video_frames.shape[1]) # width, height
def create_feature(self, frame_rate, frame_count, width, height, anchor_locations, query_locations, normalization_method, extraction_method):
frame_dimensions = (width, height)
proximity_feature = ProximityFeature(
name="proximity_feature",
anchor_locations=anchor_locations,
query_locations=query_locations,
frame_rate=frame_rate,
frame_count=len(video_frames),
frame_count=frame_count,
frame_dimensions=frame_dimensions,
normalization_method=normalization_method
normalization_method=normalization_method,
width=width,
height=height
)
# Ensure anchor and query points have the same number of columns
@@ -50,11 +55,9 @@ class ProximityFeatureNode(FirstFeature):
location.points = location.points[:, :2]
print("Depth ignored. Depth must be included for all points or no points.")
# self.start_progress(len(video_frames), desc="Calculating proximity")
proximity_feature.extract()
# self.end_progress()
return (proximity_feature, feature_pipe)
return (proximity_feature,)
class ProximityFeatureInput(RyanOnTheInside):
CATEGORY="RyanOnTheInside/Proximity"
nodes/flex/features.py
+18 -16
View File
@@ -6,11 +6,13 @@ from ..masks.mask_utils import calculate_optical_flow
from scipy.interpolate import interp1d, make_interp_spline
class BaseFeature(ABC):
def __init__(self, name, feature_type, frame_rate, frame_count):
def __init__(self, name, feature_type, frame_rate, frame_count, width, height):
self.name = name
self.type = feature_type
self.frame_rate = frame_rate
self.frame_count = frame_count
self.width = width
self.height = height
self.data = None
self.features = None
self.inverted = False
@@ -54,8 +56,8 @@ class BaseFeature(ABC):
return self
class ManualFeature(BaseFeature):
def __init__(self, name, frame_rate, frame_count, start_frame, end_frame, start_value, end_value, method='linear'):
super().__init__(name, "manual", frame_rate, frame_count)
def __init__(self, name, frame_rate, frame_count, start_frame, end_frame, start_value, end_value, method='linear', width=None, height=None):
super().__init__(name, "manual", frame_rate, frame_count, width, height)
self.start_frame = start_frame
self.end_frame = end_frame
self.start_value = start_value
@@ -109,11 +111,11 @@ class ManualFeature(BaseFeature):
return ease_out
class TimeFeature(BaseFeature):
def __init__(self, name, frame_rate, frame_count, effect_type='smooth', speed=1.0, offset=0.0):
def __init__(self, name, frame_rate, frame_count, width, height, effect_type='smooth', speed=1.0, offset=0.0):
super().__init__(name, "time", frame_rate, frame_count, width, height)
self.effect_type = effect_type
self.speed = speed
self.offset = offset
super().__init__(name, "time", frame_rate, frame_count)
@classmethod
def get_extraction_methods(self):
@@ -141,12 +143,12 @@ class TimeFeature(BaseFeature):
return self.normalize()
class DepthFeature(BaseFeature):
def __init__(self, name, frame_rate, frame_count, depth_maps, feature_name='mean_depth'):
def __init__(self, name, frame_rate, frame_count, depth_maps, feature_name='mean_depth', width=None, height=None):
super().__init__(name, "depth", frame_rate, frame_count, width, height)
self.depth_maps = depth_maps
self.features = None
self.feature_name = feature_name
self.available_features = self.get_extraction_methods()
super().__init__(name, "depth", frame_rate, frame_count)
@classmethod
def get_extraction_methods(self):
@@ -214,13 +216,13 @@ class DepthFeature(BaseFeature):
raise ValueError(f"Invalid feature name. Available features are: {', '.join(self.available_features)}")
class ColorFeature(BaseFeature):
def __init__(self, name, frame_rate, frame_count, images, feature_name='dominant_color'):
def __init__(self, name, frame_rate, frame_count, images, feature_name='dominant_color', width=None, height=None):
if images.dim() != 4 or images.shape[-1] != 3:
raise ValueError(f"Expected images in BHWC format, but got shape {images.shape}")
super().__init__(name, "color", frame_rate, frame_count, width, height)
self.images = images
self.feature_name = feature_name
self.available_features = self.get_extraction_methods()
super().__init__(name, "color", frame_rate, frame_count)
@classmethod
def get_extraction_methods(self):
@@ -298,11 +300,11 @@ class ColorFeature(BaseFeature):
raise ValueError(f"Invalid feature name. Available features are: {', '.join(self.available_features)}")
class BrightnessFeature(BaseFeature):
def __init__(self, name, frame_rate, frame_count, images, feature_name='mean_brightness'):
def __init__(self, name, frame_rate, frame_count, images, feature_name='mean_brightness', width=None, height=None):
super().__init__(name, "brightness", frame_rate, frame_count, width, height)
self.images = images
self.feature_name = feature_name
self.available_features = self.get_extraction_methods()
super().__init__(name, "brightness", frame_rate, frame_count)
@classmethod
def get_extraction_methods(self):
@@ -366,7 +368,8 @@ class BrightnessFeature(BaseFeature):
raise ValueError(f"Invalid feature name. Available features are: {', '.join(self.available_features)}")
class MotionFeature(BaseFeature):
def __init__(self, name, frame_rate, frame_count, images, feature_name='mean_motion', flow_method='Farneback', flow_threshold=0.0, magnitude_threshold=0.0, progress_callback=None):
def __init__(self, name, frame_rate, frame_count, images, feature_name='mean_motion', flow_method='Farneback', flow_threshold=0.0, magnitude_threshold=0.0, width=None, height=None, progress_callback=None):
super().__init__(name, "motion", frame_rate, frame_count, width, height)
self.images = images
self.feature_name = feature_name
self.flow_method = flow_method
@@ -374,7 +377,6 @@ class MotionFeature(BaseFeature):
self.magnitude_threshold = magnitude_threshold
self.available_features = self.get_extraction_methods()
self.progress_callback = progress_callback
super().__init__(name, "motion", frame_rate, frame_count)
@classmethod
def get_extraction_methods(self):
@@ -465,12 +467,12 @@ class MotionFeature(BaseFeature):
raise ValueError(f"Invalid feature name. Available features are: {', '.join(self.available_features)}")
class AreaFeature(BaseFeature):
def __init__(self, name, frame_rate, frame_count, masks, feature_type='total_area', threshold=0.5):
def __init__(self, name, frame_rate, frame_count, masks, feature_type='total_area', threshold=0.5, width=None, height=None):
super().__init__(name, "area", frame_rate, frame_count, width, height)
self.masks = masks
self.feature_type = feature_type
self.threshold = threshold
self.available_features = self.get_extraction_methods()
super().__init__(name, "area", frame_rate, frame_count)
@classmethod
def get_extraction_methods(self):
@@ -524,7 +526,7 @@ class AreaFeature(BaseFeature):
def set_active_feature(self, feature_name):
if feature_name in self.available_features:
self.feature_type = feature_name
self.feature_name = feature_name
self.extract() # Re-extract the data with the new feature type
else:
raise ValueError(f"Invalid feature name. Available features are: {', '.join(self.available_features)}")
nodes/flex/features_audio.py
+9 -7
View File
@@ -19,8 +19,8 @@ class BaseAudioFeature(BaseFeature):
"""Return a list of parameter names that can be modulated."""
return []
def __init__(self, name, audio, frame_count, frame_rate):
super().__init__(name, "audio", frame_rate, frame_count)
def __init__(self, name, audio, frame_count, frame_rate, width, height):
super().__init__(name, "audio", frame_rate, frame_count, width, height)
self.audio = audio
self.sample_rate = None
self.frame_duration = None
@@ -68,8 +68,8 @@ class BaseAudioFeature(BaseFeature):
class AudioFeature(BaseAudioFeature):
def __init__(self, feature_name, audio, frame_count, frame_rate, feature_type='amplitude_envelope'):
super().__init__(feature_name, audio, frame_count, frame_rate)
def __init__(self, feature_name, audio, frame_count, frame_rate, width, height, feature_type='amplitude_envelope'):
super().__init__(feature_name, audio, frame_count, frame_rate, width, height)
self.feature_type = feature_type
self.available_features = self.get_extraction_methods()
self.feature_name = feature_type
@@ -128,8 +128,8 @@ import numpy as np
import librosa
class RhythmFeature(BaseAudioFeature):
def __init__(self, feature_name, audio, frame_count, frame_rate, feature_type='beat_locations', time_signature=4):
super().__init__(feature_name, audio, frame_count, frame_rate)
def __init__(self, feature_name, audio, frame_count, frame_rate, width, height, feature_type='beat_locations', time_signature=4):
super().__init__(feature_name, audio, frame_count, frame_rate, width, height)
self.feature_type = feature_type
self.available_features = self.get_extraction_methods()
self.feature_name = feature_type
@@ -282,13 +282,15 @@ class PitchFeature(BaseAudioFeature):
audio,
frame_count,
frame_rate,
width,
height,
pitch_range_collections=None,
feature_type='frequency',
window_size=0,
vibrato_options=None,
crepe_model="none"
):
super().__init__(feature_name, audio, frame_count, frame_rate)
super().__init__(feature_name, audio, frame_count, frame_rate, width, height)
self.available_features = self.get_extraction_methods()
self.feature_type = feature_type
self.pitch_range_collections = pitch_range_collections or []
nodes/flex/features_midi.py
+2 -2
View File
@@ -26,8 +26,8 @@ class MIDIFeature(BaseFeature):
"""Return a list of parameter names that can be modulated."""
return list(cls.ATTRIBUTE_MAP.keys())
def __init__(self, name, midi_data, attribute, frame_rate, frame_count, notes=None, chord_only=False):
super().__init__(name, "midi", frame_rate, frame_count)
def __init__(self, name, midi_data, attribute, frame_rate, frame_count, width, height, notes=None, chord_only=False):
super().__init__(name, "midi", frame_rate, frame_count, width, height)
self.midi_data = midi_data
self.attribute = attribute
self.notes = set(notes) if notes is not None else set()
nodes/flex/features_proximity.py
+3 -2
View File
@@ -6,14 +6,15 @@ import numpy as np
from .features import BaseFeature
class ProximityFeature(BaseFeature):
def __init__(self, name, anchor_locations, query_locations, frame_rate, frame_count, frame_dimensions, normalization_method='frame'):
super().__init__(name, "proximity", frame_rate, frame_count)
def __init__(self, name, anchor_locations, query_locations, frame_rate, frame_count, frame_dimensions, width, height, normalization_method='frame'):
super().__init__(name, "proximity", frame_rate, frame_count, width, height)
self.anchor_locations = anchor_locations
self.query_locations = query_locations
self.frame_diagonal = np.sqrt(frame_dimensions[0]**2 + frame_dimensions[1]**2)
self.proximity_values = None
self.normalization_method = normalization_method
@classmethod
def get_extraction_methods(self):
return ["normalization_method"]
nodes/flex/flex_base.py
-1
View File
@@ -18,7 +18,6 @@ class FlexBase(ABC):
},
"optional": {
"opt_feature": ("FEATURE",),
"opt_feature_pipe": ("FEATURE_PIPE",),
}
}
nodes/images/flex_image_base.py
+3 -9
View File
@@ -55,16 +55,10 @@ class FlexImageBase(RyanOnTheInside, FlexBase):
feature_param,
feature_mode,
opt_feature=None,
opt_feature_pipe=None,
**kwargs
):
if (opt_feature is None) != (opt_feature_pipe is None):
raise ValueError(
"Both opt_feature and opt_feature_pipe must be provided together, or neither should be provided."
)
if opt_feature is None and opt_feature_pipe is None:
# Process all frames without modulation
# Process all frames without modulation if no feature is provided
if opt_feature is None:
num_frames = images.shape[0]
images_np = images.cpu().numpy()
@@ -84,8 +78,8 @@ class FlexImageBase(RyanOnTheInside, FlexBase):
self.update_progress()
else:
# Process frames with modulation based on feature values
num_frames = opt_feature_pipe.frame_count
images_np = images.cpu().numpy()
num_frames = opt_feature.frame_count
self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}")
nodes/latents/flex_latent_base.py
+1 -7
View File
@@ -28,14 +28,8 @@ class FlexLatentBase(RyanOnTheInside, FlexBase):
feature_param,
feature_mode,
opt_feature=None,
opt_feature_pipe=None,
**kwargs
):
if (opt_feature is None) != (opt_feature_pipe is None):
raise ValueError(
"Both opt_feature and opt_feature_pipe must be provided together, or neither should be provided."
)
latents_np = latents["samples"].cpu().numpy()
num_frames = latents_np.shape[0]
self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}")
@@ -46,7 +40,7 @@ class FlexLatentBase(RyanOnTheInside, FlexBase):
feature_value = 1.0 # Default feature value
apply_effect = True # Default to applying the effect
if opt_feature is not None and opt_feature_pipe is not None:
if opt_feature is not None:
feature_value = opt_feature.get_value_at_frame(i)
apply_effect = feature_value >= feature_threshold
nodes/masks/flex_mask_base.py
+4 -10
View File
@@ -76,17 +76,12 @@ class FlexMaskBase(FlexBase, MaskBase):
"""
return self.process_mask(mask, feature_value, strength, **kwargs)
def apply_effect(self, masks, opt_feature=None, opt_feature_pipe=None, strength=1.0, feature_threshold=0.0, mask_strength=1.0, invert=False, subtract_original=0.0, grow_with_blur=0.0, **kwargs):
def apply_effect(self, masks, opt_feature=None, strength=1.0, feature_threshold=0.0, mask_strength=1.0, invert=False, subtract_original=0.0, grow_with_blur=0.0, **kwargs):
"""Main entry point for the Flex system.
This method implements the required FlexBase.apply_effect method and routes to our mask-specific implementation.
"""
if (opt_feature is None) != (opt_feature_pipe is None):
raise ValueError(
"Both feature and feature_pipe must be provided together, or neither should be provided."
)
num_frames = opt_feature_pipe.frame_count if opt_feature_pipe else masks.shape[0]
num_frames = masks.shape[0]
original_masks = masks.clone()
self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}")
@@ -126,15 +121,14 @@ class FlexMaskBase(FlexBase, MaskBase):
# Use mask_strength instead of strength for the final mask operation
return (super().apply_mask_operation(processed_masks, original_masks, mask_strength, invert, subtract_original, grow_with_blur, **kwargs),)
def main_function(self, masks, opt_feature=None, opt_feature_pipe=None, strength=1.0, feature_threshold=0.0, mask_strength=1.0, invert=False, subtract_original=0.0, grow_with_blur=0.0, **kwargs):
def main_function(self, masks, opt_feature=None, strength=1.0, feature_threshold=0.0, mask_strength=1.0, invert=False, subtract_original=0.0, grow_with_blur=0.0, **kwargs):
"""Implementation of MaskBase's abstract main_function.
Explicitly forwards all parameters to apply_effect to maintain parameter names.
"""
return self.apply_effect(
masks=masks,
feature=opt_feature,
feature_pipe=opt_feature_pipe,
opt_feature=opt_feature,
strength=strength,
feature_threshold=feature_threshold,
mask_strength=mask_strength,
nodes/masks/flex_masks_normal.py
+1 -1
View File
@@ -23,7 +23,7 @@ class FlexMaskNormalBase(FlexMaskBase):
normal_array = normal_map.cpu().numpy() * 2 - 1
return normal_array
def main_function(self, masks, feature, feature_pipe, strength, feature_threshold, invert, subtract_original, grow_with_blur, normal_map, **kwargs):
def main_function(self, masks, opt_feature=None, strength=1.0, feature_threshold=0.0, invert=False, subtract_original=0.0, grow_with_blur=0.0, normal_map=None, **kwargs):
# This method should be overridden by child classes
raise NotImplementedError("Subclasses must implement main_function")
nodes/preprocessors/pose.py
+2 -3
View File
@@ -10,7 +10,6 @@ class PoseInterpolator(RyanOnTheInside):
"pose_1": ("POSE_KEYPOINT",),
"pose_2": ("POSE_KEYPOINT",),
"feature": ("FEATURE",),
"feature_pipe": ("FEATURE_PIPE",),
"strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),
"interpolation_mode": (["Linear", "Spherical"], {"default": "Linear"}),
"omit_missing_points": ("BOOLEAN", {"default": False}),
@@ -21,9 +20,9 @@ class PoseInterpolator(RyanOnTheInside):
FUNCTION = "interpolate_poses"
CATEGORY = "RyanOnTheInside/Poses"
def interpolate_poses(self, pose_1, pose_2, feature, feature_pipe, strength, interpolation_mode, omit_missing_points):
def interpolate_poses(self, pose_1, pose_2, feature, strength, interpolation_mode, omit_missing_points):
print("Debug: Starting interpolate_poses method")
num_frames = feature_pipe.frame_count
num_frames = feature.frame_count
self.progress_bar = ProgressBar(num_frames)
result_poses = []
nodes/video/flex_video.py
+1 -30
View File
@@ -1,7 +1,6 @@
import numpy as np
import torch
from .video_base import FlexVideoBase
from ..flex.feature_pipe import FeaturePipe
from scipy.interpolate import interp1d
from ..masks.mask_utils import calculate_optical_flow
import cv2
@@ -15,33 +14,15 @@ class FlexVideoDirection(FlexVideoBase):
def get_modifiable_params(cls):
return ["direction"]
@classmethod
def INPUT_TYPES(cls):
inputs = super().INPUT_TYPES()
# Make feature_pipe optional
inputs.setdefault("optional", {})
inputs["optional"].update({
"feature_pipe": ("FEATURE_PIPE",),
})
return inputs
def apply_effect_internal(
self,
video: np.ndarray,
feature_values: np.ndarray,
feature_pipe=None,
**kwargs,
) -> np.ndarray:
num_frames = video.shape[0]
# Use the frame count from the feature pipe if provided, otherwise fallback to the input video length
if feature_pipe is not None:
target_frame_count = feature_pipe.frame_count
else:
target_frame_count = num_frames
# Normalize feature values to the range [0, 1] over the length of the feature pipe (or input video)
# Normalize feature values to the range [0, 1]
normalized_features = np.clip(feature_values, 0.0, 1.0)
# Map feature values to frame indices in the input video
@@ -50,16 +31,6 @@ class FlexVideoDirection(FlexVideoBase):
# Ensure frame indices stay within valid bounds
frame_indices = np.clip(frame_indices, 0, num_frames - 1)
# If a feature pipe is provided, the output video should be the length of the feature pipe
if target_frame_count != num_frames:
# Adjust the output video length to match the feature pipe length
# Select frames based on the feature value mapping
frame_indices = np.interp(
np.linspace(0, 1, target_frame_count),
np.linspace(0, 1, len(feature_values)),
frame_indices
).astype(int)
# Create the processed video by selecting frames based on the mapped indices
processed_video = video[frame_indices]
nodes/video/flex_video_speed.py
+5 -7
View File
@@ -8,7 +8,6 @@ import torch
from .vfi_utils import preprocess_frames, postprocess_frames
from .video_base import FlexVideoBase
import numpy as np
from ..flex.feature_pipe import FeaturePipe
from scipy.interpolate import interp1d
from ..masks.mask_utils import calculate_optical_flow
import cv2
@@ -31,7 +30,6 @@ class FlexVideoSpeed(FlexVideoBase):
return {
"required": {
**super().INPUT_TYPES()["required"],
"feature_pipe": ("FEATURE_PIPE",),
"speed_factor": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.1}),
"interpolation_mode": (["none", "linear", "Farneback", "rife47", "rife49"],),
"fast_mode": ("BOOLEAN", {"default": True}),
@@ -44,12 +42,12 @@ class FlexVideoSpeed(FlexVideoBase):
def get_modifiable_params(cls):
return ["speed_factor"]
def apply_effect_internal(self, video: np.ndarray, feature_values: np.ndarray, feature_pipe: FeaturePipe,
def apply_effect_internal(self, video: np.ndarray, feature_values: np.ndarray,
speed_factor: float, interpolation_mode: str, fast_mode: bool, ensemble: bool,
scale_factor: float, **kwargs):
scale_factor: float, opt_feature=None, **kwargs):
num_frames = video.shape[0]
total_duration = feature_pipe.frame_count / feature_pipe.frame_rate
frame_rate = feature_pipe.frame_rate
frame_rate = opt_feature.frame_rate
total_duration = num_frames / frame_rate
# Ensure feature_values is the same length as video frames
if len(feature_values) != num_frames:
@@ -71,7 +69,7 @@ class FlexVideoSpeed(FlexVideoBase):
normalized_cumulative_times = cumulative_times * (total_duration / cumulative_times[-1])
# Create an array of the target timestamps that the final video must have
target_timestamps = np.linspace(0, total_duration, feature_pipe.frame_count)
target_timestamps = np.linspace(0, total_duration, num_frames)
# Interpolate the adjusted frames based on the normalized timestamps
frame_indices = np.interp(target_timestamps, normalized_cumulative_times, np.arange(num_frames))
nodes/video/video_base.py
+11 -42
View File
@@ -1,64 +1,33 @@
import torch
import numpy as np
from abc import ABC, abstractmethod
from ... import RyanOnTheInside
from ..flex.flex_base import FlexBase
from comfy.utils import ProgressBar
class FlexVideoBase(RyanOnTheInside, ABC):
class FlexVideoBase(FlexBase, ABC):
@classmethod
def INPUT_TYPES(cls):
return {
**super().INPUT_TYPES(),
"required": {
**super().INPUT_TYPES()["required"],
"images": ("IMAGE",),
"feature": ("FEATURE",),
"strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.01}),
"feature_mode": (["relative", "absolute"], {"default": "relative"}),
"feature_param": (cls.get_modifiable_params(),),
"feature_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
},
"optional": {
"feature_pipe": ("FEATURE_PIPE",)
}
}
CATEGORY = "RyanOnTheInside/FlexVideo"
CATEGORY = "RyanOnTheInside/FlexBase"
RETURN_TYPES = ("IMAGE",)
FUNCTION = "apply_effect"
def __init__(self):
self.progress_bar = None
def start_progress(self, total_steps, desc="Processing"):
self.progress_bar = ProgressBar(total_steps)
def apply_effect(self, images, strength, feature_mode, feature_threshold, opt_feature=None, **kwargs):
if opt_feature is None:
return (images,)
def update_progress(self):
if self.progress_bar:
self.progress_bar.update(1)
def end_progress(self):
self.progress_bar = None
@classmethod
@abstractmethod
def get_modifiable_params(cls):
"""Return a list of parameter names that can be modulated."""
return []
#TODO implement parame name
def modulate_param(self, param_name, param_value, feature_value, strength, mode):
if mode == "relative":
# Adjust parameter relative to its value and the feature
return param_value * (1 + (feature_value - 0.5) * 2 * strength)
else: # absolute
# Adjust parameter directly based on the feature
return param_value * feature_value * strength
def apply_effect(self, images, feature, strength, feature_mode, feature_threshold, feature_pipe=None, **kwargs):
images_np = images.cpu().numpy() # Convert tensor to numpy array
num_frames = images_np.shape[0]
target_frame_count = feature_pipe.frame_count if feature_pipe is not None else num_frames
feature_values = np.array([feature.get_value_at_frame(i) for i in range(target_frame_count)])
# Get feature values for each frame
feature_values = np.array([opt_feature.get_value_at_frame(i) for i in range(num_frames)])
# Apply threshold to feature values
feature_values[feature_values < feature_threshold] = 0
@@ -71,7 +40,7 @@ class FlexVideoBase(RyanOnTheInside, ABC):
kwargs[param_name] = self.modulate_param(param_name, param_value, avg_feature_value, strength, feature_mode)
# Apply the effect to the entire video
processed_video = self.apply_effect_internal(images_np, feature_values=feature_values, feature_pipe=feature_pipe, **kwargs)
processed_video = self.apply_effect_internal(images_np, feature_values=feature_values, **kwargs)
# Convert the numpy array back to a tensor and ensure it's in BHWC format
result_tensor = torch.from_numpy(processed_video).float()