diff --git a/nodes/flex/flex_base.py b/nodes/flex/flex_base.py index 6d9b138..60bf5a6 100644 --- a/nodes/flex/flex_base.py +++ b/nodes/flex/flex_base.py @@ -52,13 +52,9 @@ class FlexBase(ABC): def get_feature_value(self, frame_index: int, feature=None, param_name=None): - """Get feature value from either a provided feature or a scheduled parameter""" + """Get feature value from a provided feature""" if feature is not None: return feature.get_value_at_frame(frame_index) - elif self.parameter_scheduler and param_name: - feature_seq = self.parameter_scheduler.get_as_feature(param_name) - if feature_seq is not None: - return feature_seq[frame_index] return None @classmethod @@ -112,10 +108,10 @@ class FlexBase(ABC): def process_parameters(self, frame_index: int = 0, feature_value: float = None, feature_param: str = None, feature_mode: str = "relative", **kwargs) -> dict: - """Process parameters considering both scheduling and feature modulation""" + """Process parameters considering both scheduling and feature modulation. + feature_value is treated as pure input and never modulated.""" # Initialize parameter scheduler if not already done if self.parameter_scheduler is None: - # Determine frame count from the first array/list parameter we find frame_count = kwargs.get('frame_count', 1) for value in kwargs.values(): if isinstance(value, (list, tuple, np.ndarray)): @@ -129,12 +125,10 @@ class FlexBase(ABC): # Get all parameters that could be scheduled processed_kwargs = {} - # Helper function to process schedulable parameters (strength, feature_threshold) + # Helper function to process schedulable parameters def process_schedulable_param(param_name: str, default_value: float) -> float: value = kwargs.get(param_name, default_value) if isinstance(value, (list, tuple, np.ndarray)): - - try: value = float(value[frame_index]) except (IndexError, TypeError): @@ -161,43 +155,46 @@ class FlexBase(ABC): try: # Handle different types of inputs if isinstance(value, (list, tuple, np.ndarray)): - # Convert numpy arrays to lists if needed if isinstance(value, np.ndarray): - if value.ndim > 1: # If multi-dimensional array - value = value.flatten().tolist() # Flatten and convert to list + if value.ndim > 1: + value = value.flatten().tolist() else: value = value.tolist() - # Use frame_index to get the current value try: base_value = float(value[frame_index]) except (IndexError, TypeError): - base_value = float(value[0]) # Fallback to first value + base_value = float(value[0]) else: - # Single value base_value = float(value) - # Apply feature modulation if this is the target parameter - if param_name == feature_param and feature_value is not None: - # Only apply modulation if feature value meets threshold + # Only modulate if: + # 1. This parameter is the target parameter + # 2. We have a feature value + # 3. The parameter isn't the feature value itself + if (param_name == feature_param and + feature_value is not None and + feature_param != 'feature_value'): # Changed this line + if feature_value >= feature_threshold: processed_value = self.modulate_param(param_name, base_value, feature_value, strength, feature_mode) else: processed_value = base_value - # Convert back to int if needed if input_types[param_name][0] == "INT": processed_kwargs[param_name] = int(processed_value) else: processed_kwargs[param_name] = processed_value else: - # Convert to int if needed if input_types[param_name][0] == "INT": processed_kwargs[param_name] = int(base_value) else: processed_kwargs[param_name] = base_value except (ValueError, TypeError): - # If conversion fails, pass through unchanged processed_kwargs[param_name] = value + # Ensure feature_value is passed through unmodified + if feature_value is not None: + processed_kwargs['feature_value'] = feature_value + return processed_kwargs diff --git a/nodes/video/flex_video.py b/nodes/video/flex_video.py index c2543d0..874f00c 100644 --- a/nodes/video/flex_video.py +++ b/nodes/video/flex_video.py @@ -22,18 +22,53 @@ class FlexVideoDirection(FlexVideoBase): feature_values: np.ndarray, **kwargs, ) -> np.ndarray: - num_frames = video.shape[0] + num_output_frames = len(feature_values) + num_input_frames = video.shape[0] - # Normalize feature values to the range [0, 1] - normalized_features = np.clip(feature_values, 0.0, 1.0) + # Ensure strength and threshold arrays match feature length + strength = kwargs.get('strength') + feature_threshold = kwargs.get('feature_threshold') + + # If arrays are wrong length (because they were based on video length), resize them + if len(strength) != num_output_frames: + if len(strength) == 1: # If it's a single value expanded + strength = np.full(num_output_frames, strength[0]) + else: # If it's a different length array + strength = np.broadcast_to(strength, (num_output_frames,)) + + if len(feature_threshold) != num_output_frames: + if len(feature_threshold) == 1: + feature_threshold = np.full(num_output_frames, feature_threshold[0]) + else: + feature_threshold = np.broadcast_to(feature_threshold, (num_output_frames,)) + + # Normalize feature values to 0-1 + feature_min = np.min(feature_values) + feature_max = np.max(feature_values) + if feature_max > feature_min: + normalized_features = (feature_values - feature_min) / (feature_max - feature_min) + normalized_threshold = (feature_threshold - feature_min) / (feature_max - feature_min) + else: + normalized_features = np.full_like(feature_values, 0.5) + normalized_threshold = feature_threshold - # Map feature values to frame indices in the input video - frame_indices = (normalized_features * (num_frames - 1)).astype(int) - - # Ensure frame indices stay within valid bounds - frame_indices = np.clip(frame_indices, 0, num_frames - 1) - - # Create the processed video by selecting frames based on the mapped indices + # Create output video array matching feature length + processed_video = np.empty((num_output_frames, *video.shape[1:]), dtype=video.dtype) + + # Handle thresholding + above_threshold = normalized_features >= normalized_threshold + + # For frames below threshold, use first frame + frame_positions = np.where( + above_threshold, + normalized_features * (num_input_frames - 1) * strength, + np.zeros_like(normalized_features) + ) + + # Convert to integer indices with clipping + frame_indices = np.clip(frame_positions.astype(int), 0, num_input_frames - 1) + + # Create the output video by sampling from input video processed_video = video[frame_indices] return processed_video @@ -66,31 +101,47 @@ class FlexVideoSeek(FlexVideoBase): ) -> np.ndarray: num_frames = video.shape[0] processed_video = np.empty_like(video) - strength = kwargs.get('strength', 1.0) + strength = kwargs.get('strength', np.ones(num_frames)) + feature_threshold = kwargs.get('feature_threshold', np.zeros(num_frames)) seek_speed = 1.0 # Reverse the video if the reverse parameter is True if reverse: video = video[::-1] - # Clip feature values between 0 and 1 - feature_values_clipped = np.clip(feature_values, 0.0, 1.0) + # Create a mask for values above threshold (element-wise comparison) + above_threshold = feature_values >= feature_threshold + + # For values below threshold, we'll use 0 speed (stay on current frame) + # For values above threshold, we'll use the normalized feature value + feature_values_masked = np.where(above_threshold, + np.clip(feature_values, 0.0, 1.0), + 0.0) - # Apply seek_speed to feature values - adjusted_speeds = feature_values_clipped * seek_speed * strength + # Apply seek_speed and strength (element-wise multiplication) + adjusted_speeds = feature_values_masked * seek_speed * strength - # Ensure the total adjusted speed matches the number of frames + # If all speeds are 0 (all below threshold), use uniform movement total_speed = np.sum(adjusted_speeds) if total_speed == 0: adjusted_speeds = np.ones(num_frames) else: adjusted_speeds = adjusted_speeds / total_speed * num_frames - # Calculate cumulative frame positions + # Calculate cumulative positions cumulative_positions = np.cumsum(adjusted_speeds) - - # Map frame indices based on cumulative positions - frame_indices = np.clip(cumulative_positions.astype(int), 0, num_frames - 1) + + # For frames where feature is below threshold, maintain previous frame + frame_indices = np.zeros(num_frames, dtype=int) + last_valid_idx = 0 + + for i in range(num_frames): + if above_threshold[i]: + frame_idx = int(np.clip(cumulative_positions[i], 0, num_frames - 1)) + last_valid_idx = frame_idx + else: + frame_idx = last_valid_idx + frame_indices[i] = frame_idx # Create the processed video by selecting frames based on the mapped indices for idx in range(num_frames): diff --git a/nodes/video/flex_video_speed.py b/nodes/video/flex_video_speed.py index 4b51c24..7e3fca8 100644 --- a/nodes/video/flex_video_speed.py +++ b/nodes/video/flex_video_speed.py @@ -57,6 +57,13 @@ class FlexVideoSpeed(FlexVideoBase): if len(feature_values) != num_frames: raise ValueError("feature_values length must match the number of video frames") + # Get threshold from kwargs + feature_threshold = kwargs.get('feature_threshold', 0.0) + + # Apply threshold - zero out values below threshold + above_threshold = feature_values >= feature_threshold + feature_values = np.where(above_threshold, feature_values, 0.0) + # Adjust feature values based on speed_factor (element-wise comparison) adjusted_feature_values = np.where(speed_factor >= 0, 1 - feature_values, feature_values) diff --git a/nodes/video/video_base.py b/nodes/video/video_base.py index 19dbd31..aab6f5f 100644 --- a/nodes/video/video_base.py +++ b/nodes/video/video_base.py @@ -5,6 +5,7 @@ from ..flex.flex_base import FlexBase from comfy.utils import ProgressBar from ...tooltips import apply_tooltips +#TODO: maybe implement a batch version of flex base when the time comes. Currently, this differs significantly from its siblings @apply_tooltips class FlexVideoBase(FlexBase, ABC): @classmethod @@ -21,74 +22,68 @@ class FlexVideoBase(FlexBase, ABC): RETURN_TYPES = ("IMAGE",) FUNCTION = "apply_effect" - def apply_effect(self, images, strength, feature_mode, feature_threshold, opt_feature=None, **kwargs): + def apply_effect(self, images, strength, feature_mode, feature_threshold, feature_param, opt_feature=None, **kwargs): images_np = images.cpu().numpy() - - num_frames = images_np.shape[0] - + + # Get feature length first + if opt_feature is not None: + num_frames = opt_feature.frame_count + else: + num_frames = images_np.shape[0] # Fallback to video length if no feature self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}") - # First pass: collect all feature values + # Handle non-array parameters directly + processed_kwargs = {k: v for k, v in kwargs.items() + if not isinstance(v, (list, tuple, np.ndarray))} + + # Process parameters frame by frame and collect feature values feature_values = [] - for i in range(num_frames): + for i in range(num_frames): # Now using feature length! feature_value = self.get_feature_value(i, opt_feature) - feature_values.append(0.5 if feature_value is None else feature_value) - feature_values = np.array(feature_values) + feature_value = 0.5 if feature_value is None else feature_value + feature_values.append(feature_value) - # Process parameters based on feature values #TODO: really should be if param_name == feature_param - processed_kwargs = {} - for param_name in self.get_modifiable_params(): - if param_name in kwargs: - param_values = [] - for i in range(num_frames): - param_value = kwargs[param_name] - if isinstance(param_value, (list, tuple, np.ndarray)): - try: - base_value = float(param_value[i]) - except (IndexError, TypeError): - base_value = float(param_value[0]) - else: - base_value = float(param_value) - - modulated_value = self.modulate_param( - param_name, base_value, feature_values[i], strength, feature_mode - ) - param_values.append(modulated_value) - processed_kwargs[param_name] = np.array(param_values) - - # Add remaining kwargs - for key, value in kwargs.items(): - if key not in processed_kwargs and key != 'frame_index': - if isinstance(value, (list, tuple, np.ndarray)): - try: - processed_values = [value[i] for i in range(num_frames)] - except (IndexError, TypeError): - processed_values = [value[0]] * num_frames - processed_kwargs[key] = np.array(processed_values) - else: - processed_kwargs[key] = value - - # Process the entire video at once - if opt_feature is None or np.all(feature_values >= feature_threshold): - processed_video = self.apply_effect_internal( - images_np, - feature_values=feature_values, - opt_feature=opt_feature, - **processed_kwargs + # Process parameters + frame_kwargs = self.process_parameters( + frame_index=i, + feature_value=feature_value, + feature_threshold=feature_threshold, + strength=strength, + feature_param=feature_param, + feature_mode=feature_mode, + **kwargs ) - else: - # Only apply effect to frames where feature value is above threshold - mask = feature_values >= feature_threshold - processed_video = images_np.copy() - if np.any(mask): - processed_frames = self.apply_effect_internal( - images_np[mask], - feature_values=feature_values[mask], - opt_feature=opt_feature, - **{k: v[mask] if isinstance(v, np.ndarray) else v for k, v in processed_kwargs.items()} - ) - processed_video[mask] = processed_frames + # Store modulated values and array values + for k, v in frame_kwargs.items(): + if k not in processed_kwargs: + processed_kwargs[k] = [] + if isinstance(processed_kwargs[k], list): + processed_kwargs[k].append(v) + + # Convert collected values to arrays + feature_values = np.array(feature_values) + for k, v in processed_kwargs.items(): + if isinstance(v, list): + processed_kwargs[k] = np.array(v) + + # Convert strength and feature_threshold to arrays right before passing to child + strength = np.asarray(strength) + feature_threshold = np.asarray(feature_threshold) + if strength.ndim == 0: + strength = np.full(num_frames, strength) + if feature_threshold.ndim == 0: + feature_threshold = np.full(num_frames, feature_threshold) + processed_kwargs['strength'] = strength + processed_kwargs['feature_threshold'] = feature_threshold + + # Let child classes handle the video processing and threshold checks + processed_video = self.apply_effect_internal( + images_np, + feature_values=feature_values, + opt_feature=opt_feature, + **processed_kwargs + ) self.end_progress()