Squashed commit of the following:
commitf16db92525Author: ryanontheinstide <ryanfosdick87@gmail.com> Date: Tue Sep 17 09:15:11 2024 -0400 temporarily added back depth shape modifier commit6f0d93dda3Author: ryanontheinstide <ryanfosdick87@gmail.com> Date: Mon Sep 16 09:52:01 2024 -0400 depth targets commitf9c586173fAuthor: ryanontheinstide <ryanfosdick87@gmail.com> Date: Mon Sep 16 07:38:21 2024 -0400 bugfix commitbbd18f6732Author: ryanontheinstide <ryanfosdick87@gmail.com> Date: Sun Sep 15 21:06:12 2024 -0400 Depth from shape, Pitch Feature, Pitch Range specification, etc commitc16d5f0100Author: ryanontheinstide <ryanfosdick87@gmail.com> Date: Tue Sep 10 07:15:48 2024 -0400 for inagy commit92b411d254Author: ryanontheinstide <ryanfosdick87@gmail.com> Date: Mon Sep 9 19:56:24 2024 -0400 Added proximity feature, area feature, depth from shape commit7902b6a5bfAuthor: ryanontheinstide <ryanfosdick87@gmail.com> Date: Thu Sep 5 05:44:14 2024 -0400 parallax, depthchamber commitb2ec476287Author: ryanontheinstide <ryanfosdick87@gmail.com> Date: Wed Sep 4 09:20:09 2024 -0400 depth chamber improvement commit1312348002Author: ryanontheinstide <ryanfosdick87@gmail.com> Date: Tue Sep 3 16:12:30 2024 -0400 Squash commit6f0332bafdAuthor: ryanontheinstide <ryanfosdick87@gmail.com> Date: Mon Sep 2 11:28:46 2024 -0400 swap device commit5e3a59b2a1Author: ryanontheinstide <ryanfosdick87@gmail.com> Date: Mon Sep 2 08:20:05 2024 -0400 utility nodes commit4442ea14deAuthor: ryanontheinstide <ryanfosdick87@gmail.com> Date: Sun Sep 1 09:35:42 2024 -0400 improved wave propagation in Emanating Rings commit7fe5befaf6Author: ryanontheinstide <ryanfosdick87@gmail.com> Date: Sat Aug 31 17:22:26 2024 -0400 Added random shapes commitc8597d35f9Author: ryanontheinstide <ryanfosdick87@gmail.com> Date: Sat Aug 31 10:01:55 2024 -0400 Rings and wave propagation, skip features for flex images, etc commit40981b910fAuthor: ryanontheinstide <ryanfosdick87@gmail.com> Date: Thu Aug 29 17:41:51 2024 -0400 bugfix commit98b21aaa50Author: ryanontheinstide <ryanfosdick87@gmail.com> Date: Thu Aug 29 12:24:22 2024 -0400 feature modulation of particle systems commitd79ac2936cAuthor: ryanontheinstide <ryanfosdick87@gmail.com> Date: Wed Aug 28 18:47:15 2024 -0400 Squashed commit of the following: commit5c3ab23192Author: ryanontheinstide <ryanfosdick87@gmail.com> Date: Wed Aug 28 18:45:53 2024 -0400 direction amplitude, colorgrade bug fixes commitff153b25ebAuthor: ryanontheinstide <ryanfosdick87@gmail.com> Date: Wed Aug 28 17:00:40 2024 -0400 added emission radius commit618f63bc2bAuthor: ryanontheinstide <ryanfosdick87@gmail.com> Date: Wed Aug 28 15:37:17 2024 -0400 add random flag to particle modulators commit67b273dceeAuthor: ryanontheinstide <ryanfosdick87@gmail.com> Date: Wed Aug 28 15:26:46 2024 -0400 commit27cee31feaAuthor: ryanontheinstide <ryanfosdick87@gmail.com> Date: Wed Aug 28 13:36:22 2024 -0400 damn version commitd5883b5529Author: ryanontheinstide <ryanfosdick87@gmail.com> Date: Wed Aug 28 13:32:49 2024 -0400 Squashed commit of the following: more feature modulators and flex mask targets commite0603a88f9Author: ryanontheinstide <ryanfosdick87@gmail.com> Date: Tue Aug 27 19:20:05 2024 -0400 version update commitea2ca47a82Author: ryanontheinstide <ryanfosdick87@gmail.com> Date: Tue Aug 27 19:18:27 2024 -0400 get torch device commit0f9b32e13eAuthor: ryanontheinstide <ryanfosdick87@gmail.com> Date: Tue Aug 27 18:41:06 2024 -0400 bugfix commit8cc1a292f3Author: ryanontheinstide <ryanfosdick87@gmail.com> Date: Tue Aug 27 18:37:42 2024 -0400 bugfix commit7de6853a28Author: ryanontheinstide <ryanfosdick87@gmail.com> Date: Tue Aug 27 17:58:23 2024 -0400 Added flex opacity and voronoi scheduled commitf435740018Author: ryanontheinstide <ryanfosdick87@gmail.com> Date: Tue Aug 27 06:35:15 2024 -0400 bugfix - arbitrary video audio lengths commit607703988aAuthor: ryanontheinstide <ryanfosdick87@gmail.com> Date: Mon Aug 26 10:32:13 2024 -0400 Added flex, audio, midi, help system commite239c5a19dAuthor: ryanontheinstide <ryanfosdick87@gmail.com> Date: Sat Aug 17 18:00:05 2024 -0400 version, temporal control for particle size,color,speed
This commit is contained in:
ryanontheinstide
@@ -173,6 +173,12 @@ from .nodes.images.flex_images import (
|
||||
|
||||
)
|
||||
|
||||
from .nodes.depth.depth_base import(
|
||||
DepthInjection,
|
||||
DepthBlender,
|
||||
DepthRippleEffect,
|
||||
)
|
||||
|
||||
from .nodes.flex.feature_externals import (
|
||||
FeatureToWeightsStrategy,
|
||||
DepthShapeModifier,
|
||||
@@ -279,6 +285,9 @@ NODE_CLASS_MAPPINGS = {
|
||||
"AreaFeatureNode": AreaFeatureNode,
|
||||
|
||||
"FeatureToWeightsStrategy": FeatureToWeightsStrategy,
|
||||
"DepthInjection": DepthInjection,
|
||||
"DepthRippleEffect": DepthRippleEffect,
|
||||
"DepthBlender": DepthBlender,
|
||||
"DepthShapeModifier": DepthShapeModifier,
|
||||
|
||||
#feature modulation
|
||||
|
||||
@@ -1148,6 +1148,33 @@ add_node_config("ImageIntervalSelectPercentage", {
|
||||
"""
|
||||
})
|
||||
|
||||
add_node_config("DepthInjection", {
|
||||
"TOP_DESCRIPTION": "Modifies depth maps based on mask contours, creating spherical gradients.",
|
||||
"ADDITIONAL_INFO": """
|
||||
- `depth_map`: Input depth map (IMAGE type)
|
||||
- `mask`: Input mask to define areas for depth modification (MASK type)
|
||||
- `gradient_steepness`: Controls the steepness of the spherical gradient (0.1 to 10.0). Higher values create sharper transitions.
|
||||
- `depth_min`: Minimum depth value for the modified areas (0.0 to 1.0)
|
||||
- `depth_max`: Maximum depth value for the modified areas (0.0 to 1.0)
|
||||
- `strength`: Overall strength of the depth modification effect (0.0 to 1.0)
|
||||
|
||||
This node modifies depth maps by creating spherical gradients based on the contours of the input mask. It's useful for adding depth variations to specific areas of an image, such as creating a sense of volume for masked objects.
|
||||
|
||||
The process involves:
|
||||
1. Finding contours in the mask
|
||||
2. Generating spherical gradients for each contour
|
||||
3. Scaling the gradients to the specified depth range
|
||||
4. Blending the modified depth with the original depth map
|
||||
|
||||
This node can be particularly effective for:
|
||||
- Adding depth to flat objects in a scene
|
||||
- Creating a sense of volume for masked areas
|
||||
- Fine-tuning depth maps for more realistic 3D effects
|
||||
|
||||
Note: The node currently doesn't use the feature modulation capabilities, but these could be added in future versions for dynamic depth modifications.
|
||||
"""
|
||||
})
|
||||
|
||||
add_node_config("ImageChunks", {
|
||||
"TOP_DESCRIPTION": "Concatenates images into a grid.",
|
||||
"ADDITIONAL_INFO": """
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,286 @@
|
||||
import torch
|
||||
import numpy as np
|
||||
from abc import ABC, abstractmethod
|
||||
from tqdm import tqdm
|
||||
from comfy.utils import ProgressBar
|
||||
from ... import RyanOnTheInside
|
||||
import cv2
|
||||
|
||||
class FlexDepthBase(RyanOnTheInside, ABC):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"depth_maps": ("IMAGE",),
|
||||
"feature": ("FEATURE",),
|
||||
"feature_pipe": ("FEATURE_PIPE",),
|
||||
"strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||||
"feature_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||||
"feature_param": (cls.get_modifiable_params(), {"default": cls.get_modifiable_params()[0]}),
|
||||
"feature_mode": (["relative", "absolute"], {"default": "relative"}),
|
||||
}
|
||||
}
|
||||
|
||||
CATEGORY = "RyanOnTheInside/DepthEffects"
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "apply_effect"
|
||||
|
||||
def __init__(self):
|
||||
self.progress_bar = None
|
||||
|
||||
def start_progress(self, total_steps, desc="Processing"):
|
||||
self.progress_bar = ProgressBar(total_steps)
|
||||
|
||||
def update_progress(self):
|
||||
if self.progress_bar:
|
||||
self.progress_bar.update(1)
|
||||
|
||||
def end_progress(self):
|
||||
self.progress_bar = None
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def get_modifiable_params(cls):
|
||||
"""Return a list of parameter names that can be modulated."""
|
||||
return []
|
||||
|
||||
def modulate_param(self, param_name, param_value, feature_value, strength, mode):
|
||||
if mode == "relative":
|
||||
return param_value * (1 + (feature_value - 0.5) * strength)
|
||||
else: # absolute
|
||||
return param_value * feature_value * strength
|
||||
|
||||
def apply_effect(self, depth_maps, feature, feature_pipe, strength, feature_threshold, feature_param, feature_mode, **kwargs):
|
||||
num_frames = feature_pipe.frame_count
|
||||
depth_maps_np = depth_maps.cpu().numpy()
|
||||
|
||||
self.start_progress(num_frames, desc=f"Applying {self.__class__.__name__}")
|
||||
|
||||
result = []
|
||||
for i in range(num_frames):
|
||||
depth_map = depth_maps_np[i]
|
||||
feature_value = feature.get_value_at_frame(i)
|
||||
kwargs['frame_index'] = i
|
||||
if feature_value >= feature_threshold:
|
||||
processed_depth_map = self.process_depth_map(depth_map, feature_value, strength,
|
||||
feature_param=feature_param,
|
||||
feature_mode=feature_mode,
|
||||
**kwargs)
|
||||
else:
|
||||
processed_depth_map = depth_map
|
||||
|
||||
result.append(processed_depth_map)
|
||||
self.update_progress()
|
||||
|
||||
self.end_progress()
|
||||
|
||||
# Convert the list of numpy arrays to a single numpy array
|
||||
result_np = np.stack(result)
|
||||
|
||||
# Convert the numpy array to a PyTorch tensor
|
||||
result_tensor = torch.from_numpy(result_np).float()
|
||||
|
||||
# Ensure the tensor is in BHWC format
|
||||
if result_tensor.shape[1] != depth_maps.shape[1]: # Adjust as needed
|
||||
result_tensor = result_tensor.permute(0, 2, 3, 1)
|
||||
|
||||
return (result_tensor,)
|
||||
|
||||
def process_depth_map(self, depth_map: np.ndarray, feature_value: float, strength: float,
|
||||
feature_param: str, feature_mode: str, **kwargs) -> np.ndarray:
|
||||
# Modulate the selected parameter
|
||||
for param_name in self.get_modifiable_params():
|
||||
if param_name in kwargs:
|
||||
if param_name == feature_param:
|
||||
kwargs[param_name] = self.modulate_param(param_name, kwargs[param_name],
|
||||
feature_value, strength, feature_mode)
|
||||
|
||||
# Call the child class's implementation
|
||||
return self.apply_effect_internal(depth_map, **kwargs)
|
||||
|
||||
@abstractmethod
|
||||
def apply_effect_internal(self, depth_map: np.ndarray, **kwargs) -> np.ndarray:
|
||||
"""Apply the effect to the depth map. To be implemented by child classes."""
|
||||
pass
|
||||
|
||||
|
||||
class DepthInjection(FlexDepthBase):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
**super().INPUT_TYPES(),
|
||||
"required": {
|
||||
**super().INPUT_TYPES()["required"],
|
||||
"mask": ("MASK",),
|
||||
"gradient_steepness": ("FLOAT", {"default": 2.0, "min": 0.1, "max": 10.0, "step": 0.1}),
|
||||
"depth_min": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||||
"depth_max": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||||
# "strength" is already included in the base class
|
||||
}
|
||||
}
|
||||
|
||||
CATEGORY = "RyanOnTheInside/DepthModifiers"
|
||||
|
||||
@classmethod
|
||||
def get_modifiable_params(cls):
|
||||
return ["gradient_steepness", "depth_min", "depth_max", "strength", "None"]
|
||||
|
||||
def apply_effect_internal(self, depth_map: np.ndarray, mask, frame_index, **kwargs) -> np.ndarray:
|
||||
gradient_steepness = kwargs.get('gradient_steepness')
|
||||
depth_min = kwargs.get('depth_min')
|
||||
depth_max = kwargs.get('depth_max')
|
||||
strength = kwargs.get('strength', 1.0)
|
||||
|
||||
h, w, c = depth_map.shape
|
||||
|
||||
# Get mask for this frame
|
||||
mask_i = mask[frame_index].cpu().numpy().astype(np.uint8)
|
||||
|
||||
# Label connected components in the mask
|
||||
num_labels, labels = cv2.connectedComponents(mask_i)
|
||||
|
||||
# Initialize the gradient map
|
||||
gradient_map = np.zeros((h, w), dtype=np.float32)
|
||||
|
||||
for label in range(1, num_labels): # Skip background label 0
|
||||
# Create a mask for this component
|
||||
component_mask = (labels == label).astype(np.uint8)
|
||||
|
||||
# Compute distance transform inside the component
|
||||
dist_transform = cv2.distanceTransform(component_mask, cv2.DIST_L2, 5)
|
||||
max_dist = np.max(dist_transform)
|
||||
|
||||
if max_dist > 0:
|
||||
# Normalize distance transform to [0,1]
|
||||
normalized_dist = dist_transform / max_dist
|
||||
|
||||
# Apply gradient steepness
|
||||
shape_gradient = np.power(normalized_dist, gradient_steepness)
|
||||
|
||||
# Apply gradient only to this component
|
||||
gradient_map += shape_gradient * component_mask.astype(np.float32)
|
||||
|
||||
# Scale gradient to depth range
|
||||
depth_gradient = depth_min + gradient_map * (depth_max - depth_min)
|
||||
|
||||
# Apply gradient to depth map
|
||||
modified_depth = depth_map.copy()
|
||||
depth_gradient_expanded = np.expand_dims(depth_gradient, axis=-1).repeat(c, axis=-1)
|
||||
mask_i_expanded = np.expand_dims(mask_i, axis=-1).repeat(c, axis=-1).astype(bool)
|
||||
modified_depth = np.where(mask_i_expanded,
|
||||
depth_gradient_expanded,
|
||||
modified_depth)
|
||||
|
||||
# Blend modified depth with original depth
|
||||
blend_mask = mask_i_expanded * strength
|
||||
modified_depth = depth_map * (1 - blend_mask) + modified_depth * blend_mask
|
||||
|
||||
return modified_depth
|
||||
|
||||
|
||||
class DepthBlender(FlexDepthBase):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
**super().INPUT_TYPES(),
|
||||
"required": {
|
||||
**super().INPUT_TYPES()["required"],
|
||||
"other_depth_maps": ("IMAGE",),
|
||||
"blend_mode": (["add", "subtract", "multiply", "average"], {"default": "average"}),
|
||||
# "strength" is already included in the base class
|
||||
}
|
||||
}
|
||||
|
||||
CATEGORY = "RyanOnTheInside/DepthModifiers"
|
||||
|
||||
@classmethod
|
||||
def get_modifiable_params(cls):
|
||||
return ["strength", "None"]
|
||||
|
||||
def apply_effect_internal(self, depth_map: np.ndarray, other_depth_maps, frame_index, **kwargs) -> np.ndarray:
|
||||
blend_mode = kwargs.get('blend_mode')
|
||||
strength = kwargs.get('strength', 1.0)
|
||||
|
||||
# Get the other depth map for the current frame
|
||||
other_depth_map = other_depth_maps[frame_index].cpu().numpy()
|
||||
|
||||
if blend_mode == "add":
|
||||
blended_depth = depth_map + other_depth_map
|
||||
elif blend_mode == "subtract":
|
||||
blended_depth = depth_map - other_depth_map
|
||||
elif blend_mode == "multiply":
|
||||
blended_depth = depth_map * other_depth_map
|
||||
elif blend_mode == "average":
|
||||
blended_depth = (depth_map + other_depth_map) / 2
|
||||
else:
|
||||
blended_depth = depth_map # Default to original depth map
|
||||
|
||||
# Blend with the original depth map based on strength
|
||||
modified_depth = depth_map * (1 - strength) + blended_depth * strength
|
||||
|
||||
# Ensure depth values are within valid range
|
||||
modified_depth = np.clip(modified_depth, 0.0, 1.0)
|
||||
|
||||
return modified_depth
|
||||
|
||||
|
||||
class DepthRippleEffect(FlexDepthBase):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
**super().INPUT_TYPES(),
|
||||
"required": {
|
||||
**super().INPUT_TYPES()["required"],
|
||||
"ripple_amplitude": ("FLOAT", {"default": 0.05, "min": 0.0, "max": 0.5, "step": 0.01}),
|
||||
"ripple_frequency": ("FLOAT", {"default": 20.0, "min": 1.0, "max": 100.0, "step": 1.0}),
|
||||
"ripple_phase": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 6.2832, "step": 0.1}), # 2π
|
||||
"curvature": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||||
# "strength" is already included in the base class
|
||||
}
|
||||
}
|
||||
|
||||
CATEGORY = "RyanOnTheInside/DepthModifiers"
|
||||
|
||||
@classmethod
|
||||
def get_modifiable_params(cls):
|
||||
return ["ripple_amplitude", "ripple_frequency", "ripple_phase", "curvature", "strength", "None"]
|
||||
|
||||
def apply_effect_internal(self, depth_map: np.ndarray, frame_index, **kwargs) -> np.ndarray:
|
||||
ripple_amplitude = kwargs.get('ripple_amplitude')
|
||||
ripple_frequency = kwargs.get('ripple_frequency')
|
||||
ripple_phase = kwargs.get('ripple_phase')
|
||||
curvature = kwargs.get('curvature')
|
||||
strength = kwargs.get('strength', 1.0)
|
||||
|
||||
h, w, c = depth_map.shape
|
||||
|
||||
# Create a coordinate grid normalized to [0,1]
|
||||
y, x = np.meshgrid(np.linspace(0, 1, h), np.linspace(0, 1, w), indexing='ij')
|
||||
|
||||
# Center coordinates around 0
|
||||
x_c = x - 0.5
|
||||
y_c = y - 0.5
|
||||
|
||||
# Compute distance from center for circular ripples
|
||||
distance = np.sqrt(x_c**2 + y_c**2)
|
||||
|
||||
# Compute linear ripple pattern (lines)
|
||||
linear_pattern = x_c + y_c
|
||||
|
||||
# Interpolate between linear and circular patterns
|
||||
pattern = (1 - curvature) * linear_pattern + curvature * distance
|
||||
|
||||
# Compute ripple effect
|
||||
ripple = ripple_amplitude * np.sin(2 * np.pi * ripple_frequency * pattern + ripple_phase)
|
||||
|
||||
# Apply ripple to depth map
|
||||
ripple = np.expand_dims(ripple, axis=-1).repeat(c, axis=-1)
|
||||
modified_depth = depth_map + ripple
|
||||
|
||||
# Blend with the original depth map based on strength
|
||||
modified_depth = depth_map * (1 - strength) + modified_depth * strength
|
||||
|
||||
# Ensure depth values are within valid range
|
||||
modified_depth = np.clip(modified_depth, 0.0, 1.0)
|
||||
|
||||
return modified_depth
|
||||
@@ -41,7 +41,7 @@ class FeatureToWeightsStrategy(FlexExternalModulator):
|
||||
|
||||
return (weights_strategy,)
|
||||
|
||||
|
||||
#TODO: sub somthing else
|
||||
class DepthShapeModifier(FlexExternalModulator):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
import mido
|
||||
import os
|
||||
from .feature_pipe import FeaturePipe
|
||||
import folder_paths
|
||||
from server import PromptServer
|
||||
from aiohttp import web
|
||||
import shutil
|
||||
from .midi_feature import MIDIFeature
|
||||
from .feature_extractors import FeatureExtractorBase
|
||||
|
||||
class MIDILoadAndExtract(FeatureExtractorBase):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"midi_file": (folder_paths.get_filename_list("midi_files"),),
|
||||
"track_selection": (["all"],),
|
||||
"attribute": (MIDIFeature.get_attribute_names(), {"default": "Note On/Off"}),
|
||||
"frame_rate": ("FLOAT", {"default": 30, "min": 0.1, "max": 120, "step": 0.1}),
|
||||
"video_frames": ("IMAGE",),
|
||||
"chord_only": ("BOOLEAN", {"default": False}),
|
||||
"notes": ("STRING", {"default": ""}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("MIDI", "FEATURE", "FEATURE_PIPE")
|
||||
FUNCTION = "process_midi"
|
||||
CATEGORY = "RyanOnTheInside/Audio"
|
||||
|
||||
def process_midi(self, midi_file, track_selection, notes, attribute, frame_rate, video_frames, chord_only=False):
|
||||
try:
|
||||
midi_path = folder_paths.get_full_path("midi_files", midi_file)
|
||||
if not midi_path or not os.path.exists(midi_path):
|
||||
raise FileNotFoundError(f"MIDI file not found: {midi_file}")
|
||||
|
||||
midi_data = mido.MidiFile(midi_path)
|
||||
|
||||
selected_notes = [int(n.strip()) for n in notes.split(',') if n.strip().isdigit()]
|
||||
feature_pipe = FeaturePipe(frame_rate, video_frames)
|
||||
|
||||
# Convert friendly attribute name to internal attribute name
|
||||
internal_attribute = MIDIFeature.get_attribute_value(attribute)
|
||||
|
||||
feature = MIDIFeature(f"midi_{internal_attribute}", midi_data, internal_attribute,
|
||||
feature_pipe.frame_rate, feature_pipe.frame_count,
|
||||
notes=selected_notes, chord_only=chord_only)
|
||||
|
||||
feature.extract()
|
||||
|
||||
return (midi_data, feature, feature_pipe)
|
||||
|
||||
except Exception as e:
|
||||
# error_msg = f"Error in MIDILoadAndExtract.process_midi: {type(e).__name__}: {str(e)}\n"
|
||||
# error_msg += traceback.format_exc()
|
||||
# print(error_msg)
|
||||
raise RuntimeError(f"Error processing MIDI file: {type(e).__name__}: {str(e)}")
|
||||
|
||||
@classmethod
|
||||
def analyze_midi(cls, midi_path):
|
||||
midi_data = mido.MidiFile(midi_path)
|
||||
tracks = ["all"]
|
||||
all_notes = set()
|
||||
track_notes = {}
|
||||
for i, track in enumerate(midi_data.tracks):
|
||||
track_notes[str(i)] = set()
|
||||
for msg in track:
|
||||
if msg.type == 'note_on':
|
||||
track_notes[str(i)].add(msg.note)
|
||||
all_notes.add(msg.note)
|
||||
if len(track_notes[str(i)]) == 0:
|
||||
tracks.append(f"{i}: (Empty)")
|
||||
else:
|
||||
tracks.append(f"{i}: {track.name or f'Track {i}'}")
|
||||
|
||||
return {
|
||||
"tracks": tracks,
|
||||
"all_notes": ",".join(map(str, sorted(set(all_notes)))),
|
||||
"track_notes": {k: ",".join(map(str, sorted(v))) for k, v in track_notes.items()}
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, midi_file, track_selection, notes, attribute, frame_rate, video_frames):
|
||||
if not folder_paths.exists_and_is_file(midi_file):
|
||||
return "MIDI file not found: {}".format(midi_file)
|
||||
|
||||
if notes != "all":
|
||||
try:
|
||||
note_list = [int(n.strip()) for n in notes.split(',') if n.strip()]
|
||||
if not all(0 <= n <= 127 for n in note_list):
|
||||
return "Invalid note value. All notes must be between 0 and 127."
|
||||
except ValueError:
|
||||
return "Invalid notes format. Please provide comma-separated integers or 'all'."
|
||||
|
||||
return True
|
||||
|
||||
|
||||
routes = PromptServer.instance.routes
|
||||
@PromptServer.instance.routes.post('/get_track_notes')
|
||||
async def get_track_notes(request):
|
||||
data = await request.json()
|
||||
midi_file = data.get('midi_file')
|
||||
|
||||
if not midi_file:
|
||||
return web.json_response({"error": "Missing required parameters"}, status=400)
|
||||
|
||||
midi_path = folder_paths.get_full_path("midi_files", midi_file)
|
||||
if not midi_path or not os.path.exists(midi_path):
|
||||
return web.json_response({"error": "MIDI file not found"}, status=404)
|
||||
|
||||
analysis = MIDILoadAndExtract.analyze_midi(midi_path)
|
||||
return web.json_response(analysis)
|
||||
|
||||
@routes.post('/upload_midi')
|
||||
async def upload_midi(request):
|
||||
data = await request.post()
|
||||
midi_file = data['file']
|
||||
|
||||
if midi_file and midi_file.filename:
|
||||
safe_filename = os.path.basename(midi_file.filename)
|
||||
|
||||
midi_dir = folder_paths.get_folder_paths("midi_files")[0]
|
||||
|
||||
if not os.path.exists(midi_dir):
|
||||
os.makedirs(midi_dir, exist_ok=True)
|
||||
|
||||
midi_path = os.path.join(midi_dir, safe_filename)
|
||||
|
||||
with open(midi_path, 'wb') as f:
|
||||
shutil.copyfileobj(midi_file.file, f)
|
||||
|
||||
midi_files = folder_paths.get_filename_list("midi_files")
|
||||
analysis = MIDILoadAndExtract.analyze_midi(midi_path)
|
||||
|
||||
return web.json_response({
|
||||
"status": "success",
|
||||
"uploaded_file": safe_filename,
|
||||
"midi_files": midi_files,
|
||||
"analysis": analysis
|
||||
})
|
||||
else:
|
||||
return web.json_response({"status": "error", "message": "No file uploaded"}, status=400)
|
||||
|
||||
@PromptServer.instance.routes.post('/refresh_midi_data')
|
||||
async def refresh_midi_data(request):
|
||||
data = await request.json()
|
||||
midi_file = data.get('midi_file')
|
||||
track_selection = data.get('track_selection')
|
||||
|
||||
if not midi_file:
|
||||
return web.json_response({"error": "Missing required parameters"}, status=400)
|
||||
|
||||
midi_path = folder_paths.get_full_path("midi_files", midi_file)
|
||||
if not midi_path or not os.path.exists(midi_path):
|
||||
return web.json_response({"error": "MIDI file not found"}, status=404)
|
||||
|
||||
analysis = MIDILoadAndExtract.analyze_midi(midi_path)
|
||||
|
||||
# Filter notes based on track selection
|
||||
if track_selection != "all":
|
||||
track_index = track_selection.split(':')[0]
|
||||
analysis['all_notes'] = analysis['track_notes'].get(track_index, "")
|
||||
|
||||
return web.json_response(analysis)
|
||||
@@ -0,0 +1,92 @@
|
||||
from .features import BaseFeature
|
||||
import torch
|
||||
import numpy as np
|
||||
|
||||
import numpy as np
|
||||
from .features import BaseFeature
|
||||
|
||||
class ProximityFeature(BaseFeature):
|
||||
def __init__(self, name, anchor_locations, query_locations, frame_rate, frame_count, frame_dimensions, normalization_method='frame'):
|
||||
super().__init__(name, "proximity", frame_rate, frame_count)
|
||||
self.anchor_locations = anchor_locations
|
||||
self.query_locations = query_locations
|
||||
self.frame_diagonal = np.sqrt(frame_dimensions[0]**2 + frame_dimensions[1]**2)
|
||||
self.proximity_values = None
|
||||
self.normalization_method = normalization_method
|
||||
|
||||
def extract(self):
|
||||
from scipy.spatial.distance import cdist
|
||||
|
||||
proximities = []
|
||||
for anchor, query in zip(self.anchor_locations, self.query_locations):
|
||||
if len(anchor) == 0 or len(query) == 0:
|
||||
proximities.append(self.frame_diagonal if self.normalization_method == 'frame' else float('inf'))
|
||||
else:
|
||||
distances = cdist(anchor.points, query.points)
|
||||
min_distance = np.min(distances)
|
||||
proximities.append(min_distance)
|
||||
|
||||
proximities = np.array(proximities)
|
||||
|
||||
if self.normalization_method == 'frame':
|
||||
self.proximity_values = 1 - np.clip(proximities / self.frame_diagonal, 0, 1)
|
||||
elif self.normalization_method == 'minmax':
|
||||
min_proximity = np.min(proximities)
|
||||
max_proximity = np.max(proximities)
|
||||
if max_proximity > min_proximity:
|
||||
self.proximity_values = 1 - (proximities - min_proximity) / (max_proximity - min_proximity)
|
||||
else:
|
||||
self.proximity_values = np.ones_like(proximities)
|
||||
else:
|
||||
raise ValueError(f"Unknown normalization method: {self.normalization_method}")
|
||||
|
||||
self.smooth_proximities()
|
||||
return self
|
||||
|
||||
def smooth_proximities(self, window_size=5):
|
||||
kernel = np.ones(window_size) / window_size
|
||||
self.proximity_values = np.convolve(self.proximity_values, kernel, mode='same')
|
||||
|
||||
def get_value_at_frame(self, frame_index):
|
||||
return self.proximity_values[frame_index]
|
||||
|
||||
class Location:
|
||||
def __init__(self, x, y, z=None):
|
||||
x = np.asarray(x).reshape(-1)
|
||||
y = np.asarray(y).reshape(-1)
|
||||
|
||||
if z is not None:
|
||||
z = np.asarray(z).reshape(-1)
|
||||
# Ensure z has the same length as x and y, with a small tolerance for floating-point imprecision
|
||||
if abs(len(z) - len(x)) <= 1: # Allow for off-by-one errors
|
||||
z = z[:len(x)] if len(z) > len(x) else np.pad(z, (0, len(x) - len(z)), 'constant', constant_values=np.nan)
|
||||
elif len(z) != len(x):
|
||||
raise ValueError(f"Mismatch in dimensions: x,y have length {len(x)}, but z has length {len(z)}")
|
||||
self.points = np.column_stack((x, y, z))
|
||||
else:
|
||||
self.points = np.column_stack((x, y))
|
||||
|
||||
def __len__(self):
|
||||
return len(self.points)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
return self.points[idx]
|
||||
|
||||
@property
|
||||
def x(self):
|
||||
return self.points[:, 0]
|
||||
|
||||
@property
|
||||
def y(self):
|
||||
return self.points[:, 1]
|
||||
|
||||
@property
|
||||
def z(self):
|
||||
return self.points[:, 2] if self.points.shape[1] > 2 else None
|
||||
|
||||
@classmethod
|
||||
def from_tensor(cls, tensor):
|
||||
return cls(tensor[:, 0], tensor[:, 1], tensor[:, 2] if tensor.shape[1] > 2 else None)
|
||||
|
||||
def to_tensor(self):
|
||||
return torch.from_numpy(self.points)
|
||||
@@ -0,0 +1,279 @@
|
||||
from .proximity_feature import Location, ProximityFeature
|
||||
from .feature_extractors import FirstFeature
|
||||
from ... import RyanOnTheInside
|
||||
from .feature_pipe import FeaturePipe
|
||||
import numpy as np
|
||||
import cv2
|
||||
import torch
|
||||
from scipy.spatial.distance import cdist # Add this import
|
||||
|
||||
class ProximityFeatureNode(FirstFeature):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"video_frames": ("IMAGE",),
|
||||
"frame_rate": ("FLOAT", {"default": 30.0, "min": 1.0, "max": 120.0, "step": 0.1}),
|
||||
"anchor_locations": ("LOCATION",),
|
||||
"query_locations": ("LOCATION",),
|
||||
"normalization_method": (["frame", "minmax"],),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("FEATURE", "FEATURE_PIPE")
|
||||
RETURN_NAMES = ("proximity_feature", "feature_pipe")
|
||||
FUNCTION = "create_feature"
|
||||
|
||||
def create_feature(self, video_frames, frame_rate, anchor_locations, query_locations, normalization_method):
|
||||
feature_pipe = FeaturePipe(frame_rate, video_frames)
|
||||
frame_dimensions = (video_frames.shape[2], video_frames.shape[1]) # width, height
|
||||
|
||||
proximity_feature = ProximityFeature(
|
||||
name="proximity_feature",
|
||||
anchor_locations=anchor_locations,
|
||||
query_locations=query_locations,
|
||||
frame_rate=frame_rate,
|
||||
frame_count=len(video_frames),
|
||||
frame_dimensions=frame_dimensions,
|
||||
normalization_method=normalization_method
|
||||
)
|
||||
|
||||
# Ensure anchor and query points have the same number of columns
|
||||
anchor_dim = anchor_locations[0].points.shape[1]
|
||||
query_dim = query_locations[0].points.shape[1]
|
||||
|
||||
if anchor_dim != query_dim:
|
||||
if anchor_dim == 3:
|
||||
for location in anchor_locations:
|
||||
location.points = location.points[:, :2]
|
||||
elif query_dim == 3:
|
||||
for location in query_locations:
|
||||
location.points = location.points[:, :2]
|
||||
print("Depth ignored. Depth must be included for all points or no points.")
|
||||
|
||||
# self.start_progress(len(video_frames), desc="Calculating proximity")
|
||||
proximity_feature.extract()
|
||||
# self.end_progress()
|
||||
|
||||
return (proximity_feature, feature_pipe)
|
||||
|
||||
class ProximityFeatureInput(RyanOnTheInside):
|
||||
CATEGORY="RyanOnTheInside/Proximity"
|
||||
|
||||
class LocationFromMask(ProximityFeatureInput):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"masks": ("MASK",),
|
||||
"method": (["mask_boundary", "mask_center"],),
|
||||
},
|
||||
"optional": {
|
||||
"depth_maps": ("IMAGE",),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("LOCATION",)
|
||||
FUNCTION = "generate_locations"
|
||||
|
||||
def generate_locations(self, masks, method, depth_maps=None):
|
||||
locations = []
|
||||
for i in range(masks.shape[0]):
|
||||
mask = masks[i].cpu().numpy()
|
||||
if np.sum(mask) == 0: # Check if the mask is empty
|
||||
print(f"No mask found in frame {i}.")
|
||||
locations.append(Location(np.array([-1]), np.array([-1]), np.array([-1]) if depth_maps is not None else None))
|
||||
continue
|
||||
|
||||
if method == "mask_boundary":
|
||||
contours, _ = cv2.findContours(mask.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
||||
if contours:
|
||||
boundary = np.vstack(contours).squeeze()
|
||||
else:
|
||||
boundary = np.array([[0, 0]]) # Fallback to a single point
|
||||
elif method == "mask_center":
|
||||
y, x = np.where(mask > 0.5)
|
||||
if len(x) > 0 and len(y) > 0:
|
||||
center_x, center_y = np.mean(x), np.mean(y)
|
||||
boundary = np.array([[center_x, center_y]])
|
||||
else:
|
||||
boundary = np.array([[mask.shape[1] / 2, mask.shape[0] / 2]])
|
||||
|
||||
if depth_maps is not None:
|
||||
depth_map = depth_maps[i].cpu().numpy()
|
||||
# Ensure boundary points are within the depth map dimensions
|
||||
valid_points = (boundary[:, 0] < depth_map.shape[1]) & (boundary[:, 1] < depth_map.shape[0])
|
||||
boundary = boundary[valid_points]
|
||||
|
||||
if boundary.size == 0: # Check if boundary is empty after filtering
|
||||
print(f"No valid boundary points in frame {i}.")
|
||||
locations.append(Location(np.array([-1]), np.array([-1]), np.array([-1])))
|
||||
continue
|
||||
|
||||
# Extract depth values for valid boundary points
|
||||
z = np.mean(depth_map[boundary[:, 1].astype(int), boundary[:, 0].astype(int)], axis=-1)
|
||||
|
||||
location = Location(boundary[:, 0], boundary[:, 1], z)
|
||||
else:
|
||||
location = Location(boundary[:, 0], boundary[:, 1])
|
||||
|
||||
locations.append(location)
|
||||
|
||||
return (locations,)
|
||||
|
||||
class LocationFromPoint(ProximityFeatureInput):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"x": ("FLOAT", {"default": 0.0,"min":0.0,"step":0.01}),
|
||||
"y": ("FLOAT", {"default": 0.0,"min":0.0,"step":0.01}),
|
||||
"z": ("FLOAT", {"default": 0.0, "max":1.0, "min":0.0,"step":0.01}),
|
||||
"batch_count": ("INT", {"default": 1, "min": 1}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("LOCATION",)
|
||||
FUNCTION = "generate_locations"
|
||||
|
||||
def generate_locations(self, x, y, batch_count, z):
|
||||
locations = []
|
||||
for _ in range(batch_count):
|
||||
location = Location(np.array([x]), np.array([y]), np.array([z]))
|
||||
locations.append(location)
|
||||
|
||||
return (locations,)
|
||||
|
||||
class LocationTransform(ProximityFeatureInput):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"locations": ("LOCATION",),
|
||||
"feature": ("FEATURE",),
|
||||
"transformation_type": (["translate", "scale"],),
|
||||
"transformation_value": ("FLOAT", {"default": 1.0}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("LOCATION",)
|
||||
FUNCTION = "transform_locations"
|
||||
|
||||
def transform_locations(self, locations, feature, transformation_type, transformation_value):
|
||||
transformed_locations = []
|
||||
for frame_index, location in enumerate(locations):
|
||||
feature_value = feature.get_value_at_frame(frame_index)
|
||||
if transformation_type == "translate":
|
||||
new_x = location.x + feature_value * transformation_value
|
||||
new_y = location.y + feature_value * transformation_value
|
||||
if location.z is not None:
|
||||
new_z = location.z + feature_value * transformation_value
|
||||
transformed_location = Location(new_x, new_y, new_z)
|
||||
else:
|
||||
transformed_location = Location(new_x, new_y)
|
||||
elif transformation_type == "scale":
|
||||
new_x = location.x * feature_value * transformation_value
|
||||
new_y = location.y * feature_value * transformation_value
|
||||
if location.z is not None:
|
||||
new_z = location.z * feature_value * transformation_value
|
||||
transformed_location = Location(new_x, new_y, new_z)
|
||||
else:
|
||||
transformed_location = Location(new_x, new_y)
|
||||
transformed_locations.append(transformed_location)
|
||||
|
||||
return (transformed_locations,)
|
||||
|
||||
|
||||
|
||||
class ProximityVisualizer(RyanOnTheInside):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"video_frames": ("IMAGE",),
|
||||
"anchor_locations": ("LOCATION",),
|
||||
"query_locations": ("LOCATION",),
|
||||
"proximity_feature": ("FEATURE",),
|
||||
"anchor_color": ("STRING", {"default": "(255,0,0)"}),
|
||||
"query_color": ("STRING", {"default": "(0,255,0)"}),
|
||||
"line_color": ("STRING", {"default": "(0,0,255)"}),
|
||||
"text_color": ("STRING", {"default": "(255,255,255)"}),
|
||||
"font_scale": ("FLOAT", {"default": 0.5, "min": 0.1, "max": 2.0, "step": 0.1}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "visualize_proximity"
|
||||
|
||||
def visualize_proximity(self, video_frames, anchor_locations, query_locations, proximity_feature,
|
||||
anchor_color, query_color, line_color, text_color, font_scale):
|
||||
anchor_color = self.parse_color(anchor_color)
|
||||
query_color = self.parse_color(query_color)
|
||||
line_color = self.parse_color(line_color)
|
||||
text_color = self.parse_color(text_color)
|
||||
|
||||
output_frames = []
|
||||
height, width = video_frames.shape[1:3] # Extract height and width from video_frames
|
||||
|
||||
# Calculate the frame diagonal from the video frames
|
||||
frame_diagonal = np.sqrt(width**2 + height**2)
|
||||
scale_factor = frame_diagonal / proximity_feature.frame_diagonal
|
||||
for frame_index in range(len(video_frames)):
|
||||
frame = video_frames[frame_index].cpu().numpy()
|
||||
frame = (frame * 255).astype(np.uint8)
|
||||
frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
|
||||
|
||||
anchor = anchor_locations[frame_index]
|
||||
query = query_locations[frame_index]
|
||||
|
||||
# Calculate scaling factors based on the frame diagonal
|
||||
|
||||
|
||||
# Draw all anchor and query points (using only x and y coordinates)
|
||||
for point in anchor:
|
||||
scaled_point = (int(point[0] * scale_factor), int(point[1] * scale_factor))
|
||||
cv2.circle(frame, scaled_point, 2, anchor_color, -1)
|
||||
for point in query:
|
||||
scaled_point = (int(point[0] * scale_factor), int(point[1] * scale_factor))
|
||||
cv2.circle(frame, scaled_point, 2, query_color, -1)
|
||||
|
||||
# Find the closest pair of points
|
||||
if len(anchor) > 0 and len(query) > 0:
|
||||
distances = cdist(anchor.points[:, :2], query.points[:, :2]) # Use only x and y for distance calculation
|
||||
min_idx = np.unravel_index(distances.argmin(), distances.shape)
|
||||
closest_anchor = anchor[min_idx[0]]
|
||||
closest_query = query[min_idx[1]]
|
||||
|
||||
# Draw line between closest points (using only x and y coordinates)
|
||||
cv2.line(frame,
|
||||
(int(closest_anchor[0] * scale_factor), int(closest_anchor[1] * scale_factor)),
|
||||
(int(closest_query[0] * scale_factor), int(closest_query[1] * scale_factor)),
|
||||
line_color, 2)
|
||||
|
||||
# Display coordinates of closest points
|
||||
anchor_text = f"Anchor: ({int(closest_anchor[0] * scale_factor)}, {int(closest_anchor[1] * scale_factor)}"
|
||||
query_text = f"Query: ({int(closest_query[0] * scale_factor)}, {int(closest_query[1] * scale_factor)}"
|
||||
|
||||
if anchor.z is not None and query.z is not None:
|
||||
anchor_text += f", {closest_anchor[2]:.2f})"
|
||||
query_text += f", {closest_query[2]:.2f})"
|
||||
else:
|
||||
anchor_text += ") [2D]"
|
||||
query_text += ") [2D]"
|
||||
|
||||
cv2.putText(frame, anchor_text, (10, 60), cv2.FONT_HERSHEY_SIMPLEX, font_scale, text_color, 1)
|
||||
cv2.putText(frame, query_text, (10, 90), cv2.FONT_HERSHEY_SIMPLEX, font_scale, text_color, 1)
|
||||
|
||||
# Add proximity value to the frame
|
||||
proximity_value = proximity_feature.get_value_at_frame(frame_index)
|
||||
cv2.putText(frame, f"Proximity: {proximity_value:.2f}", (10, 30),
|
||||
cv2.FONT_HERSHEY_SIMPLEX, 1, text_color, 2)
|
||||
|
||||
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
output_frames.append(frame)
|
||||
|
||||
output_tensor = torch.from_numpy(np.stack(output_frames)).float() / 255.0
|
||||
return (output_tensor,)
|
||||
|
||||
def parse_color(self, color_string):
|
||||
return tuple(map(int, color_string.strip("()").split(",")))
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[project]
|
||||
name = "comfyui_ryanonyheinside"
|
||||
description = "Custom nodes introducing particle simulations, optical flow, audio manipulation & reactivity, and temporal masks"
|
||||
version = "1.8.0"
|
||||
version = "1.8.2"
|
||||
license = {file = "LICENSE"}
|
||||
dependencies = ["opencv-python==4.10.0","scipy","torchaudio", "pillow","librosa","pymunk==6.8.1","matplotlib","openunmix","mido"]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user