Video target

video as target

__init__.py

@@ -171,7 +171,13 @@ from .nodes.images.flex_images import (
     FlexImageTiltShift,
     FlexImageParallax,
     FlexImageContrast,
-    
+
+)
+
+from .nodes.video.flex_video import (
+    FlexVideoSpeed,
+    FlexVideoDirection,
+    FlexVideoFrameBlend,
 )
 
 from .nodes.depth.depth_base import(
@@ -215,6 +221,11 @@ os.makedirs(midi_path, exist_ok=True)
 
 NODE_CLASS_MAPPINGS = {
     
+    #video
+    "FlexVideoSpeed":              FlexVideoSpeed,
+    "FlexVideoDirection":          FlexVideoDirection,
+    "FlexVideoFrameBlend":         FlexVideoFrameBlend,
+
     ###temporal
     "MaskMorph":                    MaskMorph,
     "MaskTransform":                MaskTransform,
@@ -315,6 +326,7 @@ NODE_CLASS_MAPPINGS = {
     "FlexImageTiltShift":           FlexImageTiltShift,
     "FlexImageParallax":            FlexImageParallax,
     "FlexImageContrast":            FlexImageContrast,
+
     #opacity xp 
     # "FlexDepthBasedMaskOpacity":  FlexDepthBasedMaskOpacity,
     # "DepthBasedMaskOpacity":      DepthBasedMaskOpacity,
@@ -344,6 +356,11 @@ NODE_CLASS_MAPPINGS = {
 WEB_DIRECTORY = "./web/js"
 
 NODE_DISPLAY_NAME_MAPPINGS = {
+
+    "FlexVideoSpeed":            "**BETA** Flex Video Speed",
+    "FlexVideoDirection":        "Flex Video Direction",
+    "FlexVideoFrameBlend":       "**BETA**Flex Video Frame Blend",
+
     "MaskMorph": "Temporal Mask Morph",
     "MaskTransform":"Temporal Mask Transform",
     "MaskMath":"Temporal Mask Math",

node_configs/node_configs.py

@@ -1,3 +1,8 @@
+
+
+NODE_CONFIGS = {}
+
+
 #NOTE: this abstraction allows for the documentation to be both centrally managed and inherited
 from abc import ABCMeta
 class NodeConfigMeta(type):
@@ -14,7 +19,7 @@ class CombinedMeta(NodeConfigMeta, ABCMeta):
 def add_node_config(node_name, config):
     NODE_CONFIGS[node_name] = config
 
-NODE_CONFIGS = {}
+
 
 add_node_config("MaskBase", {
     "BASE_DESCRIPTION": """
@@ -1287,4 +1292,49 @@ This node overlays pitch-related information on video frames, including:
 
 The information is displayed as text on each frame, allowing for easy visualization of pitch characteristics alongside the video content.
 """
+})
+
+add_node_config("FlexVideoBase", {
+    "TOP_DESCRIPTION": "Base class for flexible video effect nodes.",
+    "ADDITIONAL_INFO": """
+- `images`: Input video frames (IMAGE type)
+- `feature`: Feature used to modulate the effect (FEATURE type)
+- `strength`: Overall strength of the effect (0.0 to 2.0)
+- `feature_mode`: How the feature modulates the parameter ("relative" or "absolute")
+- `feature_param`: Parameter to be modulated by the feature
+- `feature_threshold`: Minimum feature value to apply the effect (0.0 to 1.0)
+- `feature_pipe`: (Sometimes Optional) Feature pipe containing frame information (FEATURE_PIPE type)
+"""
+})
+
+add_node_config("FlexVideoDirection", {
+    "TOP_DESCRIPTION": "Applies a direction-based effect to video frames based on feature values.",
+    "ADDITIONAL_INFO": """
+- Inherits parameters from FlexVideoBase
+- `feature_pipe`: (Optional) Feature pipe containing frame information (FEATURE_PIPE type)
+
+This node creates a video effect by selecting frames based on the input feature values. It can create effects like reversing, speeding up, or creating loops in the video sequence.
+"""
+})
+
+add_node_config("FlexVideoSpeed", {
+    "TOP_DESCRIPTION": "Adjusts the playback speed of video frames based on feature values.",
+    "ADDITIONAL_INFO": """
+- Inherits parameters from FlexVideoBase
+- `feature_pipe`: Feature pipe containing frame information (FEATURE_PIPE type)
+- `max_speed_percent`: Maximum speed as a percentage of the original speed (1.0 to 1000.0)
+- `duration_adjustment_method`: Method to adjust video duration ("Interpolate" or "Truncate/Repeat")
+
+This node modifies the playback speed of the video based on the input feature values. It can create effects like slow motion, fast forward, or variable speed playback.
+"""
+})
+
+add_node_config("FlexVideoFrameBlend", {
+    "TOP_DESCRIPTION": "Applies frame blending effect to video frames based on feature values.",
+    "ADDITIONAL_INFO": """
+- Inherits parameters from FlexVideoBase
+- `blend_strength`: Strength of the frame blending effect (0.0 to 1.0)
+
+This node creates a frame blending effect, where each frame is blended with the next frame. The strength of the blend is modulated by the input feature values, allowing for dynamic motion blur effects.
+"""
 })

nodes/video/flex_video.py

@@ -0,0 +1,218 @@
+import numpy as np
+from .video_base import FlexVideoBase
+
+class FlexVideoDirection(FlexVideoBase):
+
+    @classmethod
+    def get_modifiable_params(cls):
+        return ["direction"]
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        inputs = super().INPUT_TYPES()
+        # Make feature_pipe optional
+        inputs.setdefault("optional", {})
+        inputs["optional"].update({
+            "feature_pipe": ("FEATURE_PIPE",),
+        })
+        return inputs
+
+
+    def apply_effect_internal(
+        self,
+        video: np.ndarray,
+        feature_values: np.ndarray,
+        feature_pipe=None,
+        **kwargs,
+    ) -> np.ndarray:
+        num_frames = video.shape[0]
+        
+        # Use the frame count from the feature pipe if provided, otherwise fallback to the input video length
+        if feature_pipe is not None:
+            target_frame_count = feature_pipe.frame_count
+        else:
+            target_frame_count = num_frames
+
+        # Normalize feature values to the range [0, 1] over the length of the feature pipe (or input video)
+        normalized_features = np.clip(feature_values, 0.0, 1.0)
+
+        # Map feature values to frame indices in the input video
+        frame_indices = (normalized_features * (num_frames - 1)).astype(int)
+
+        # Ensure frame indices stay within valid bounds
+        frame_indices = np.clip(frame_indices, 0, num_frames - 1)
+
+        # If a feature pipe is provided, the output video should be the length of the feature pipe
+        if target_frame_count != num_frames:
+            # Adjust the output video length to match the feature pipe length
+            # Select frames based on the feature value mapping
+            frame_indices = np.interp(
+                np.linspace(0, 1, target_frame_count),
+                np.linspace(0, 1, len(feature_values)),
+                frame_indices
+            ).astype(int)
+
+        # Create the processed video by selecting frames based on the mapped indices
+        processed_video = video[frame_indices]
+
+        return processed_video
+
+class FlexVideoSpeed(FlexVideoBase):
+
+    @classmethod
+    def get_modifiable_params(cls):
+        return ["max_speed_percent"]
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                **super().INPUT_TYPES()["required"],
+                "feature_pipe": ("FEATURE_PIPE",),
+                "max_speed_percent": ("FLOAT", {"default": 500.0, "min": 1.0, "max": 1000.0, "step": 1.0}),
+                "duration_adjustment_method": (["Interpolate", "Truncate/Repeat"],),
+            }
+        }
+
+    FUNCTION = "apply_effect"
+
+    def apply_effect_internal(
+        self,
+        video: np.ndarray,
+        feature_values: np.ndarray,
+        max_speed_percent: float,
+        feature_pipe=None,
+        duration_adjustment_method="Interpolate",
+        **kwargs,
+    ) -> np.ndarray:
+        strength = kwargs.get('strength', 1.0)
+        feature_mode = kwargs.get('feature_mode', 'relative')
+        num_frames = video.shape[0]
+        original_frame_rate = feature_pipe.frame_rate
+        num_features = len(feature_values)
+
+        # Adjust video duration to match the number of features
+        if num_frames != num_features:
+            if duration_adjustment_method == "Interpolate":
+                video = self.interpolate_video(video, num_features)
+            else:  # "Truncate/Repeat"
+                video = self.truncate_or_repeat_video(video, num_features)
+            num_frames = num_features
+
+        # Convert max_speed_percent to a speed factor
+        max_speed_factor = max_speed_percent / 100.0
+
+        # Apply strength and feature_mode to adjust speed
+        if feature_mode == "relative":
+            # Feature value of 0.5 means original speed
+            speed_factors = 1.0 + (feature_values - 0.5) * 2 * (max_speed_factor - 1.0) * strength
+        else:  # "absolute"
+            # Feature values directly map to speed, scaled by max_speed_factor
+            speed_factors = feature_values * max_speed_factor * strength
+
+        # Ensure speed factors are within a reasonable range
+        speed_factors = np.clip(speed_factors, 1e-6, max_speed_factor)
+
+        # Compute time intervals between frames based on adjusted speed
+        time_intervals = 1.0 / (original_frame_rate * speed_factors)
+
+        # Compute cumulative time
+        cumulative_time = np.cumsum(time_intervals)
+        total_time = cumulative_time[-1]
+
+        # Generate new time stamps for the output frames
+        output_time_stamps = np.linspace(0, total_time, num_frames)
+
+        # Map output time stamps to input frame indices via interpolation
+        new_frame_indices = np.interp(output_time_stamps, cumulative_time, np.arange(num_frames))
+
+        # Interpolate frames
+        processed_video = np.empty_like(video)
+        self.start_progress(len(new_frame_indices))
+        for idx, t in enumerate(new_frame_indices):
+            lower_idx = int(np.floor(t))
+            upper_idx = min(lower_idx + 1, num_frames - 1)
+            weight = t - lower_idx
+            processed_video[idx] = (1 - weight) * video[lower_idx] + weight * video[upper_idx]
+            self.update_progress()
+        self.end_progress()
+        return processed_video
+    
+    def interpolate_video(self, video: np.ndarray, target_frames: int) -> np.ndarray:
+        """Interpolate video to match the target number of frames."""
+        orig_frames = video.shape[0]
+        if orig_frames == target_frames:
+            return video
+        
+        new_video = np.empty((target_frames, *video.shape[1:]), dtype=video.dtype)
+        for i in range(target_frames):
+            t = i / (target_frames - 1) * (orig_frames - 1)
+            idx1, idx2 = int(t), min(int(t) + 1, orig_frames - 1)
+            weight = t - idx1
+            new_video[i] = (1 - weight) * video[idx1] + weight * video[idx2]
+        
+        return new_video
+
+    def truncate_or_repeat_video(self, video: np.ndarray, target_frames: int) -> np.ndarray:
+        """Truncate or repeat video frames to match the target number of frames."""
+        orig_frames = video.shape[0]
+        if orig_frames == target_frames:
+            return video
+        
+        if orig_frames > target_frames:
+            return video[:target_frames]
+        else:
+            repeats = target_frames // orig_frames
+            remainder = target_frames % orig_frames
+            return np.concatenate([np.tile(video, (repeats, 1, 1, 1)), video[:remainder]])
+class FlexVideoFrameBlend(FlexVideoBase):
+
+    @classmethod
+    def get_modifiable_params(cls):
+        return ["blend_strength"]
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        inputs = super().INPUT_TYPES()
+        inputs["required"].update({
+            "blend_strength": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}),
+        })
+        return inputs
+
+    FUNCTION = "apply_effect"
+
+    def apply_effect_internal(
+        self,
+        video: np.ndarray,
+        feature_values: np.ndarray,
+        blend_strength: float,
+        **kwargs,
+    ) -> np.ndarray:
+        num_frames = video.shape[0]
+        strength = kwargs.get('strength', 1.0)
+        feature_mode = kwargs.get('feature_mode', 'relative')
+        processed_video = np.empty_like(video)
+
+        # Adjust blend strength based on feature values
+        if feature_mode == "relative":
+            adjusted_blend = blend_strength + (feature_values - 0.5) * 2 * strength
+        else:  # "absolute"
+            adjusted_blend = feature_values * strength
+
+        adjusted_blend = np.clip(adjusted_blend, 0.0, 1.0)
+
+        self.start_progress(num_frames)
+        for idx in range(num_frames):
+            current_frame = video[idx]
+            if idx < num_frames - 1:
+                next_frame = video[idx + 1]
+            else:
+                next_frame = video[idx]  # Last frame, no next frame available
+
+            alpha = adjusted_blend[idx]
+            blended_frame = (1 - alpha) * current_frame + alpha * next_frame
+            processed_video[idx] = np.clip(blended_frame, 0.0, 1.0)
+            self.update_progress()
+        self.end_progress() 
+
+        return processed_video

nodes/video/video_base.py

@@ -0,0 +1,86 @@
+import torch
+import numpy as np
+from abc import ABC, abstractmethod
+from ... import RyanOnTheInside
+from comfy.utils import ProgressBar
+
+class FlexVideoBase(RyanOnTheInside, ABC):
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "images": ("IMAGE",),
+                "feature": ("FEATURE",),
+                "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.01}),
+                "feature_mode": (["relative", "absolute"], {"default": "relative"}),
+                "feature_param": (cls.get_modifiable_params(),),
+                "feature_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
+            },
+            "optional": {
+                "feature_pipe": ("FEATURE_PIPE",)
+            }
+        }
+
+    CATEGORY = "RyanOnTheInside/VideoEffects"
+    RETURN_TYPES = ("IMAGE",)
+    FUNCTION = "apply_effect"
+    
+    def __init__(self):
+        self.progress_bar = None
+
+    def start_progress(self, total_steps, desc="Processing"):
+        self.progress_bar = ProgressBar(total_steps)
+
+    def update_progress(self):
+        if self.progress_bar:
+            self.progress_bar.update(1)
+
+    def end_progress(self):
+        self.progress_bar = None
+
+    @classmethod
+    @abstractmethod
+    def get_modifiable_params(cls):
+        """Return a list of parameter names that can be modulated."""
+        return []
+
+    #TODO implement parame name
+    def modulate_param(self, param_name, param_value, feature_value, strength, mode):
+        if mode == "relative":
+            # Adjust parameter relative to its value and the feature
+            return param_value * (1 + (feature_value - 0.5) * 2 * strength)
+        else:  # absolute
+            # Adjust parameter directly based on the feature
+            return param_value * feature_value * strength
+
+    def apply_effect(self, images, feature, strength, feature_mode, feature_threshold, feature_pipe=None, **kwargs):
+        images_np = images.cpu().numpy()  # Convert tensor to numpy array
+        num_frames = images_np.shape[0]
+
+        target_frame_count = feature_pipe.frame_count if feature_pipe is not None else num_frames
+        feature_values = np.array([feature.get_value_at_frame(i) for i in range(target_frame_count)])
+        
+        # Apply threshold to feature values
+        feature_values[feature_values < feature_threshold] = 0
+
+        # Modulate parameters based on the feature values
+        for param_name in self.get_modifiable_params():
+            if param_name in kwargs:
+                param_value = kwargs[param_name]
+                avg_feature_value = np.mean(feature_values)
+                kwargs[param_name] = self.modulate_param(param_name, param_value, avg_feature_value, strength, feature_mode)
+
+        # Apply the effect to the entire video
+        processed_video = self.apply_effect_internal(images_np, feature_values=feature_values, feature_pipe=feature_pipe, **kwargs)
+
+        # Convert the numpy array back to a tensor and ensure it's in BHWC format
+        result_tensor = torch.from_numpy(processed_video).float()
+        if result_tensor.shape[1] == 3:  # If in BCHW format, convert to BHWC
+            result_tensor = result_tensor.permute(0, 2, 3, 1)
+
+        return (result_tensor,)
+
+    @abstractmethod
+    def apply_effect_internal(self, video: np.ndarray, **kwargs) -> np.ndarray:
+        """Apply the effect to the entire video. To be implemented by child classes."""
+        pass

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "comfyui_ryanonyheinside"
 description = "Custom nodes introducing particle simulations, optical flow, audio manipulation & reactivity, and temporal masks"
-version = "1.8.5"
+version = "1.8.6"
 license = {file = "LICENSE"}
 dependencies = ["opencv-python==4.10.0","scipy","torchaudio", "pillow","librosa","pymunk==6.8.1","matplotlib","openunmix","mido"]
 

readme.md

@@ -32,10 +32,12 @@
 
 *Examples showcasing various effects using particle emitters, vortices, and other node features*
 
-##### 🆕 Recent Updates
--9/27/24 - MIDI feature bug fix
--9/25/24 - **FlexImageContrast**: Adds contrast and brightness to images, with an option to preserve luminosity.
--9/15/24 - **alot** Depth From Shape, Audio Pitch Feature, Pitch Range  filters, new MIDI keybord for Pitch Range specification, Image from audio, mask from audio, Improved depth chamber, wave propagation, emanating rings,   and a lot more 
+##### 🆕 Recent Updates:
+
+- 9/28/24 - **Video TARGET**: Adds video as a feature target! While all feature targets are inherently *tiiime* based, this additional allows for *video-level* modulation as opposed to modulating individual frames to varying degrees over time. Novel!
+- 9/27/24 - MIDI feature bug fix
+- 9/25/24 - **FlexImageContrast**: Adds contrast and brightness to images, with an option to preserve luminosity.
+- 9/15/24 - **alot** Depth From Shape, Audio Pitch Feature, Pitch Range  filters, new MIDI keybord for Pitch Range specification, Image from audio, mask from audio, Improved depth chamber, wave propagation, emanating rings,   and a lot more 
 - 9/8/24 - **Area Feature**: Adds area  as a  driving reactivity feature!
 - 9/7/24 - **Proximity Feature**: Adds proximity as a driving reactivity feature! Allows for the distince of objects from one another to control other nodes.
 - 9/5/24 - **FlexImageParallax**: Add illusory depth.