nodes/ryanontheinside-ComfyUI_RyanOnTheInside
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

video refactored

Browse Source
This commit is contained in:
ryanontheinstide
2025-01-17 16:29:08 -05:00
parent f4dd7eeaf9
commit 07ec17ff45
4 changed files with 152 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nodes/flex/flex_base.py
+19 -22
View File
@@ -52,13 +52,9 @@ class FlexBase(ABC):
def get_feature_value(self, frame_index: int, feature=None, param_name=None):
"""Get feature value from either a provided feature or a scheduled parameter"""
"""Get feature value from a provided feature"""
if feature is not None:
return feature.get_value_at_frame(frame_index)
elif self.parameter_scheduler and param_name:
feature_seq = self.parameter_scheduler.get_as_feature(param_name)
if feature_seq is not None:
return feature_seq[frame_index]
return None
@classmethod
@@ -112,10 +108,10 @@ class FlexBase(ABC):
def process_parameters(self, frame_index: int = 0, feature_value: float = None,
feature_param: str = None, feature_mode: str = "relative", **kwargs) -> dict:
"""Process parameters considering both scheduling and feature modulation"""
"""Process parameters considering both scheduling and feature modulation.
feature_value is treated as pure input and never modulated."""
# Initialize parameter scheduler if not already done
if self.parameter_scheduler is None:
# Determine frame count from the first array/list parameter we find
frame_count = kwargs.get('frame_count', 1)
for value in kwargs.values():
if isinstance(value, (list, tuple, np.ndarray)):
@@ -129,12 +125,10 @@ class FlexBase(ABC):
# Get all parameters that could be scheduled
processed_kwargs = {}
# Helper function to process schedulable parameters (strength, feature_threshold)
# Helper function to process schedulable parameters
def process_schedulable_param(param_name: str, default_value: float) -> float:
value = kwargs.get(param_name, default_value)
if isinstance(value, (list, tuple, np.ndarray)):
try:
value = float(value[frame_index])
except (IndexError, TypeError):
@@ -161,43 +155,46 @@ class FlexBase(ABC):
try:
# Handle different types of inputs
if isinstance(value, (list, tuple, np.ndarray)):
# Convert numpy arrays to lists if needed
if isinstance(value, np.ndarray):
if value.ndim > 1: # If multi-dimensional array
value = value.flatten().tolist() # Flatten and convert to list
if value.ndim > 1:
value = value.flatten().tolist()
else:
value = value.tolist()
# Use frame_index to get the current value
try:
base_value = float(value[frame_index])
except (IndexError, TypeError):
base_value = float(value[0]) # Fallback to first value
base_value = float(value[0])
else:
# Single value
base_value = float(value)
# Apply feature modulation if this is the target parameter
if param_name == feature_param and feature_value is not None:
# Only apply modulation if feature value meets threshold
# Only modulate if:
# 1. This parameter is the target parameter
# 2. We have a feature value
# 3. The parameter isn't the feature value itself
if (param_name == feature_param and
feature_value is not None and
feature_param != 'feature_value'): # Changed this line
if feature_value >= feature_threshold:
processed_value = self.modulate_param(param_name, base_value, feature_value, strength, feature_mode)
else:
processed_value = base_value
# Convert back to int if needed
if input_types[param_name][0] == "INT":
processed_kwargs[param_name] = int(processed_value)
else:
processed_kwargs[param_name] = processed_value
else:
# Convert to int if needed
if input_types[param_name][0] == "INT":
processed_kwargs[param_name] = int(base_value)
else:
processed_kwargs[param_name] = base_value
except (ValueError, TypeError):
# If conversion fails, pass through unchanged
processed_kwargs[param_name] = value
# Ensure feature_value is passed through unmodified
if feature_value is not None:
processed_kwargs['feature_value'] = feature_value
return processed_kwargs
nodes/video/flex_video.py
+71 -20
View File
@@ -22,18 +22,53 @@ class FlexVideoDirection(FlexVideoBase):
feature_values: np.ndarray,
**kwargs,
) -> np.ndarray:
num_frames = video.shape[0]
num_output_frames = len(feature_values)
num_input_frames = video.shape[0]
# Normalize feature values to the range [0, 1]
normalized_features = np.clip(feature_values, 0.0, 1.0)
# Ensure strength and threshold arrays match feature length
strength = kwargs.get('strength')
feature_threshold = kwargs.get('feature_threshold')
# If arrays are wrong length (because they were based on video length), resize them
if len(strength) != num_output_frames:
if len(strength) == 1: # If it's a single value expanded
strength = np.full(num_output_frames, strength[0])
else: # If it's a different length array
strength = np.broadcast_to(strength, (num_output_frames,))
if len(feature_threshold) != num_output_frames:
if len(feature_threshold) == 1:
feature_threshold = np.full(num_output_frames, feature_threshold[0])
else:
feature_threshold = np.broadcast_to(feature_threshold, (num_output_frames,))
# Normalize feature values to 0-1
feature_min = np.min(feature_values)
feature_max = np.max(feature_values)
if feature_max > feature_min:
normalized_features = (feature_values - feature_min) / (feature_max - feature_min)
normalized_threshold = (feature_threshold - feature_min) / (feature_max - feature_min)
else:
normalized_features = np.full_like(feature_values, 0.5)
normalized_threshold = feature_threshold
# Map feature values to frame indices in the input video
frame_indices = (normalized_features * (num_frames - 1)).astype(int)
# Ensure frame indices stay within valid bounds
frame_indices = np.clip(frame_indices, 0, num_frames - 1)
# Create the processed video by selecting frames based on the mapped indices
# Create output video array matching feature length
processed_video = np.empty((num_output_frames, *video.shape[1:]), dtype=video.dtype)
# Handle thresholding
above_threshold = normalized_features >= normalized_threshold
# For frames below threshold, use first frame
frame_positions = np.where(
above_threshold,
normalized_features * (num_input_frames - 1) * strength,
np.zeros_like(normalized_features)
)
# Convert to integer indices with clipping
frame_indices = np.clip(frame_positions.astype(int), 0, num_input_frames - 1)
# Create the output video by sampling from input video
processed_video = video[frame_indices]
return processed_video
@@ -66,31 +101,47 @@ class FlexVideoSeek(FlexVideoBase):
) -> np.ndarray:
num_frames = video.shape[0]
processed_video = np.empty_like(video)
strength = kwargs.get('strength', 1.0)
strength = kwargs.get('strength', np.ones(num_frames))
feature_threshold = kwargs.get('feature_threshold', np.zeros(num_frames))
seek_speed = 1.0
# Reverse the video if the reverse parameter is True
if reverse:
video = video[::-1]
# Clip feature values between 0 and 1
feature_values_clipped = np.clip(feature_values, 0.0, 1.0)
# Create a mask for values above threshold (element-wise comparison)
above_threshold = feature_values >= feature_threshold
# For values below threshold, we'll use 0 speed (stay on current frame)
# For values above threshold, we'll use the normalized feature value
feature_values_masked = np.where(above_threshold,
np.clip(feature_values, 0.0, 1.0),
0.0)
# Apply seek_speed to feature values
adjusted_speeds = feature_values_clipped * seek_speed * strength
# Apply seek_speed and strength (element-wise multiplication)
adjusted_speeds = feature_values_masked * seek_speed * strength
# Ensure the total adjusted speed matches the number of frames
# If all speeds are 0 (all below threshold), use uniform movement
total_speed = np.sum(adjusted_speeds)
if total_speed == 0:
adjusted_speeds = np.ones(num_frames)
else:
adjusted_speeds = adjusted_speeds / total_speed * num_frames
# Calculate cumulative frame positions
# Calculate cumulative positions
cumulative_positions = np.cumsum(adjusted_speeds)
# Map frame indices based on cumulative positions
frame_indices = np.clip(cumulative_positions.astype(int), 0, num_frames - 1)
# For frames where feature is below threshold, maintain previous frame
frame_indices = np.zeros(num_frames, dtype=int)
last_valid_idx = 0
for i in range(num_frames):
if above_threshold[i]:
frame_idx = int(np.clip(cumulative_positions[i], 0, num_frames - 1))
last_valid_idx = frame_idx
else:
frame_idx = last_valid_idx
frame_indices[i] = frame_idx
# Create the processed video by selecting frames based on the mapped indices
for idx in range(num_frames):
nodes/video/flex_video_speed.py
+7
View File
@@ -57,6 +57,13 @@ class FlexVideoSpeed(FlexVideoBase):
if len(feature_values) != num_frames:
raise ValueError("feature_values length must match the number of video frames")
# Get threshold from kwargs
feature_threshold = kwargs.get('feature_threshold', 0.0)
# Apply threshold - zero out values below threshold
above_threshold = feature_values >= feature_threshold
feature_values = np.where(above_threshold, feature_values, 0.0)
# Adjust feature values based on speed_factor (element-wise comparison)
adjusted_feature_values = np.where(speed_factor >= 0, 1 - feature_values, feature_values)
nodes/video/video_base.py
+55 -60
View File
@@ -5,6 +5,7 @@ from ..flex.flex_base import FlexBase
from comfy.utils import ProgressBar
from ...tooltips import apply_tooltips
#TODO: maybe implement a batch version of flex base when the time comes. Currently, this differs significantly from its siblings
@apply_tooltips
class FlexVideoBase(FlexBase, ABC):
@classmethod
@@ -21,74 +22,68 @@ class FlexVideoBase(FlexBase, ABC):
RETURN_TYPES = ("IMAGE",)
FUNCTION = "apply_effect"
def apply_effect(self, images, strength, feature_mode, feature_threshold, opt_feature=None, **kwargs):
def apply_effect(self, images, strength, feature_mode, feature_threshold, feature_param, opt_feature=None, **kwargs):
images_np = images.cpu().numpy()
num_frames = images_np.shape[0]
# Get feature length first
if opt_feature is not None:
num_frames = opt_feature.frame_count
else:
num_frames = images_np.shape[0] # Fallback to video length if no feature
self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}")
# First pass: collect all feature values
# Handle non-array parameters directly
processed_kwargs = {k: v for k, v in kwargs.items()
if not isinstance(v, (list, tuple, np.ndarray))}
# Process parameters frame by frame and collect feature values
feature_values = []
for i in range(num_frames):
for i in range(num_frames): # Now using feature length!
feature_value = self.get_feature_value(i, opt_feature)
feature_values.append(0.5 if feature_value is None else feature_value)
feature_values = np.array(feature_values)
feature_value = 0.5 if feature_value is None else feature_value
feature_values.append(feature_value)
# Process parameters based on feature values #TODO: really should be if param_name == feature_param
processed_kwargs = {}
for param_name in self.get_modifiable_params():
if param_name in kwargs:
param_values = []
for i in range(num_frames):
param_value = kwargs[param_name]
if isinstance(param_value, (list, tuple, np.ndarray)):
try:
base_value = float(param_value[i])
except (IndexError, TypeError):
base_value = float(param_value[0])
else:
base_value = float(param_value)
modulated_value = self.modulate_param(
param_name, base_value, feature_values[i], strength, feature_mode
)
param_values.append(modulated_value)
processed_kwargs[param_name] = np.array(param_values)
# Add remaining kwargs
for key, value in kwargs.items():
if key not in processed_kwargs and key != 'frame_index':
if isinstance(value, (list, tuple, np.ndarray)):
try:
processed_values = [value[i] for i in range(num_frames)]
except (IndexError, TypeError):
processed_values = [value[0]] * num_frames
processed_kwargs[key] = np.array(processed_values)
else:
processed_kwargs[key] = value
# Process the entire video at once
if opt_feature is None or np.all(feature_values >= feature_threshold):
processed_video = self.apply_effect_internal(
images_np,
feature_values=feature_values,
opt_feature=opt_feature,
**processed_kwargs
# Process parameters
frame_kwargs = self.process_parameters(
frame_index=i,
feature_value=feature_value,
feature_threshold=feature_threshold,
strength=strength,
feature_param=feature_param,
feature_mode=feature_mode,
**kwargs
)
else:
# Only apply effect to frames where feature value is above threshold
mask = feature_values >= feature_threshold
processed_video = images_np.copy()
if np.any(mask):
processed_frames = self.apply_effect_internal(
images_np[mask],
feature_values=feature_values[mask],
opt_feature=opt_feature,
**{k: v[mask] if isinstance(v, np.ndarray) else v for k, v in processed_kwargs.items()}
)
processed_video[mask] = processed_frames
# Store modulated values and array values
for k, v in frame_kwargs.items():
if k not in processed_kwargs:
processed_kwargs[k] = []
if isinstance(processed_kwargs[k], list):
processed_kwargs[k].append(v)
# Convert collected values to arrays
feature_values = np.array(feature_values)
for k, v in processed_kwargs.items():
if isinstance(v, list):
processed_kwargs[k] = np.array(v)
# Convert strength and feature_threshold to arrays right before passing to child
strength = np.asarray(strength)
feature_threshold = np.asarray(feature_threshold)
if strength.ndim == 0:
strength = np.full(num_frames, strength)
if feature_threshold.ndim == 0:
feature_threshold = np.full(num_frames, feature_threshold)
processed_kwargs['strength'] = strength
processed_kwargs['feature_threshold'] = feature_threshold
# Let child classes handle the video processing and threshold checks
processed_video = self.apply_effect_internal(
images_np,
feature_values=feature_values,
opt_feature=opt_feature,
**processed_kwargs
)
self.end_progress()