From 492c20ba4ae1484c9e3bdc11d036fbb4820c52c1 Mon Sep 17 00:00:00 2001 From: RyanOnTheInside <7623207+ryanontheinside@users.noreply.github.com> Date: Sat, 17 May 2025 12:09:27 -0400 Subject: [PATCH 01/10] rudimentary midi player --- __init__.py | 5 + nodes/audio/midi_nodes.py | 149 +++++++++++++++++++++++++- nodes/flex/feature_extractors_midi.py | 2 +- 3 files changed, 154 insertions(+), 2 deletions(-) diff --git a/__init__.py b/__init__.py index 81d00f3..d51a90e 100644 --- a/__init__.py +++ b/__init__.py @@ -328,6 +328,9 @@ from .nodes.doom.doom import Doom_ from .audio_latent_blend import AudioLatentBlend, FlexlatentAudioBlend + +from .nodes.audio.midi_nodes import MIDIToAudio + # Import external integrations from .external_integration import ( HAS_ADVANCED_LIVE_PORTRAIT, @@ -588,6 +591,7 @@ NODE_CLASS_MAPPINGS = { "ImageIntervalSelectPercentage":ImageIntervalSelectPercentage, "ImageIndexSelect": ImageIndexSelect, + "MIDIPlayback": MIDIToAudio, } @@ -627,6 +631,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "Audio_Concatenate": "Audio Concatenate ROTI", "Color_Picker": "Color_Picker ROTI", "Doom_": "DOOM", + "MIDIPlayback": "MIDI Player", } diff --git a/nodes/audio/midi_nodes.py b/nodes/audio/midi_nodes.py index 503fa1d..363be75 100644 --- a/nodes/audio/midi_nodes.py +++ b/nodes/audio/midi_nodes.py @@ -1 +1,148 @@ -#TODO \ No newline at end of file +import torch +import numpy as np +import mido +from .audio_nodes import AudioNodeBase +from ...tooltips import apply_tooltips + +@apply_tooltips +class MIDIToAudio(AudioNodeBase): + """Converts MIDI data to audio format compatible with ComfyUI's audio system.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "midi": ("MIDI",), + "sample_rate": ("INT", { + "default": 44100, + "min": 8000, + "max": 192000, + "step": 1 + }), + "instrument": ("INT", { + "default": 0, # Piano (not used for simple synthesis) + "min": 0, + "max": 127, + "step": 1 + }), + "volume": ("FLOAT", { + "default": 0.5, + "min": 0.0, + "max": 1.0, + "step": 0.01 + }) + } + } + + RETURN_TYPES = ("AUDIO",) + FUNCTION = "convert_midi_to_audio" + CATEGORY = "RyanOnTheInside/Audio/MIDI" + + def convert_midi_to_audio(self, midi, sample_rate, instrument, volume): + try: + # Get tempo from MIDI file (default to 120 BPM if not specified) + tempo = 500000 # Default tempo (microseconds per quarter note) + for track in midi.tracks: + for msg in track: + if msg.type == 'set_tempo': + tempo = msg.tempo + break + + # Calculate total duration in seconds + total_time = 0 + for track in midi.tracks: + track_time = 0 + for msg in track: + if hasattr(msg, 'time'): + track_time += msg.time + total_time = max(total_time, track_time) + + # Convert ticks to seconds using tempo + seconds_per_tick = tempo / (midi.ticks_per_beat * 1000000.0) + total_time = total_time * seconds_per_tick + + # Ensure we have some duration + total_time = max(total_time, 1.0) # At least 1 second + + print(f"MIDI duration: {total_time:.2f} seconds") + + # Create empty audio buffer + audio_length = int(total_time * sample_rate) + audio_buffer = np.zeros(audio_length) + + # Process MIDI messages from all tracks + for track in midi.tracks: + current_time = 0 + active_notes = {} # {note: (start_time, velocity)} + + for msg in track: + if hasattr(msg, 'time'): + current_time += msg.time * seconds_per_tick + + if msg.type == 'note_on' and msg.velocity > 0: + # Start a new note + active_notes[msg.note] = (current_time, msg.velocity) + + elif msg.type == 'note_off' or (msg.type == 'note_on' and msg.velocity == 0): + # End a note + if msg.note in active_notes: + start_time, velocity = active_notes[msg.note] + duration = current_time - start_time + + # Generate note audio using simple sine wave + frequency = 440.0 * (2.0 ** ((msg.note - 69) / 12.0)) # Convert MIDI note to frequency + note_samples = int(duration * sample_rate) + if note_samples > 0: + t = np.linspace(0, duration, note_samples, False) + # Simple envelope to avoid clicks + envelope = np.ones_like(t) + attack = int(0.01 * sample_rate) + release = int(0.01 * sample_rate) + if len(envelope) > attack: + envelope[:attack] = np.linspace(0, 1, attack) + if len(envelope) > release: + envelope[-release:] = np.linspace(1, 0, release) + note_audio = np.sin(2 * np.pi * frequency * t) * envelope * (velocity / 127.0) * volume + + # Add to buffer with proper timing + start_idx = int(start_time * sample_rate) + end_idx = start_idx + len(note_audio) + if end_idx > len(audio_buffer): + audio_buffer = np.pad(audio_buffer, (0, end_idx - len(audio_buffer))) + audio_buffer[start_idx:end_idx] += note_audio + + del active_notes[msg.note] + + # Normalize audio if we have any content + if len(audio_buffer) > 0 and np.max(np.abs(audio_buffer)) > 0: + audio_buffer = audio_buffer / np.max(np.abs(audio_buffer)) + + # Convert to stereo + audio_buffer = np.stack([audio_buffer, audio_buffer]) + + # Convert to torch tensor + audio_tensor = torch.from_numpy(audio_buffer).float() + + print(f"Generated audio length: {len(audio_buffer[0])/sample_rate:.2f} seconds") + + return ({ + "waveform": audio_tensor.unsqueeze(0), # Add batch dimension + "sample_rate": sample_rate + },) + + except Exception as e: + # If anything goes wrong, return empty audio + print(f"Error in MIDIToAudio: {str(e)}") + empty_audio = torch.zeros((1, 2, 1)) # 1 batch, 2 channels, 1 sample + return ({ + "waveform": empty_audio, + "sample_rate": sample_rate + },) + +NODE_CLASS_MAPPINGS = { + "MIDIToAudio": MIDIToAudio +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "MIDIToAudio": "MIDI to Audio" +} \ No newline at end of file diff --git a/nodes/flex/feature_extractors_midi.py b/nodes/flex/feature_extractors_midi.py index 61add3c..fb59e3c 100644 --- a/nodes/flex/feature_extractors_midi.py +++ b/nodes/flex/feature_extractors_midi.py @@ -88,7 +88,7 @@ class MIDILoadAndExtract(FeatureExtractorBase): } @classmethod - def VALIDATE_INPUTS(cls, midi_file, track_selection, notes, extraction_method, frame_rate, video_frames): + def VALIDATE_INPUTS(cls, midi_file, track_selection, notes, extraction_method, frame_rate): midi_path = folder_paths.get_full_path("midi_files", midi_file) if not midi_path or not os.path.isfile(midi_path): return f"MIDI file not found: {midi_file}" From 19ef7b113dc8dc426b7fcb88ff5779060885e993 Mon Sep 17 00:00:00 2001 From: RyanOnTheInside <7623207+ryanontheinside@users.noreply.github.com> Date: Sat, 17 May 2025 13:53:25 -0400 Subject: [PATCH 02/10] add midi loader func --- nodes/audio/midi_nodes.py | 327 +++++++++++++++++++++++++++++++++++++- 1 file changed, 324 insertions(+), 3 deletions(-) diff --git a/nodes/audio/midi_nodes.py b/nodes/audio/midi_nodes.py index 363be75..f705fb5 100644 --- a/nodes/audio/midi_nodes.py +++ b/nodes/audio/midi_nodes.py @@ -1,8 +1,13 @@ import torch import numpy as np import mido +import os +import folder_paths from .audio_nodes import AudioNodeBase from ...tooltips import apply_tooltips +from server import PromptServer +from aiohttp import web +import shutil @apply_tooltips class MIDIToAudio(AudioNodeBase): @@ -139,10 +144,326 @@ class MIDIToAudio(AudioNodeBase): "sample_rate": sample_rate },) +@apply_tooltips +class MIDILoader: + """Loads MIDI files for processing in ComfyUI""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "midi_file": (folder_paths.get_filename_list("midi_files"),), + "track_selection": (["all"],), + "start_time": ("FLOAT", {"default": 0.0, "min": 0.0, "step": 0.1}), + "duration": ("FLOAT", {"default": 0.0, "min": 0.0, "step": 0.1, "description": "Duration in seconds (0 = full duration)"}), + } + } + + RETURN_TYPES = ("MIDI",) + FUNCTION = "load_midi" + CATEGORY = "RyanOnTheInside/Audio/MIDI" + + def load_midi(self, midi_file, track_selection, start_time=0.0, duration=0.0): + try: + midi_path = folder_paths.get_full_path("midi_files", midi_file) + if not midi_path or not os.path.exists(midi_path): + raise FileNotFoundError(f"MIDI file not found: {midi_file}") + + # Load the MIDI file + midi_data = mido.MidiFile(midi_path) + + # Get tempo for time calculations + tempo = 500000 # Default tempo (microseconds per quarter note) + for track in midi_data.tracks: + for msg in track: + if msg.type == 'set_tempo': + tempo = msg.tempo + break + + # Calculate seconds per tick for time conversion + seconds_per_tick = tempo / (midi_data.ticks_per_beat * 1000000.0) + + # Convert seconds to ticks + start_tick = int(start_time / seconds_per_tick) if start_time > 0 else 0 + end_tick = float('inf') + if duration > 0: + end_tick = start_tick + int(duration / seconds_per_tick) + + # Calculate total duration + total_ticks = 0 + for track in midi_data.tracks: + track_ticks = 0 + for msg in track: + if hasattr(msg, 'time'): + track_ticks += msg.time + total_ticks = max(total_ticks, track_ticks) + + total_duration = total_ticks * seconds_per_tick + + # Apply track selection if not "all" + if track_selection != "all": + track_index_str = track_selection.split(':')[0].strip() + if track_index_str.isdigit(): + track_index = int(track_index_str) + # Create a new MIDI file with only the selected track + selected_midi = mido.MidiFile(ticks_per_beat=midi_data.ticks_per_beat) + + # First, add any metadata tracks (typically track 0 in type 1 MIDI files) + if midi_data.type == 1 and track_index > 0: + selected_midi.tracks.append(midi_data.tracks[0]) + + # Add the selected track + if track_index < len(midi_data.tracks): + selected_midi.tracks.append(midi_data.tracks[track_index]) + + midi_data = selected_midi + + # Apply time slicing if needed + if (start_time > 0 or duration > 0) and midi_data.tracks: + # Create a new MIDI file with the selected time range + trimmed_midi = mido.MidiFile(ticks_per_beat=midi_data.ticks_per_beat) + + # Copy data for the trimmed section + # NOTE: This is a simplified approach and may not handle all MIDI events perfectly + for track in midi_data.tracks: + new_track = mido.MidiTrack() + trimmed_midi.tracks.append(new_track) + + # Copy all metadata and control messages + for msg in track: + if not hasattr(msg, 'time') or msg.type in ['time_signature', 'key_signature', 'set_tempo']: + new_track.append(msg.copy()) + + # Extract all note events within our time range + current_tick = 0 + last_tick = 0 + + # First pass: collect all note_on events within our range + active_notes = set() + for msg in track: + if not hasattr(msg, 'time'): + continue + + current_tick += msg.time + + # Track note_on events within our range + if msg.type == 'note_on' and msg.velocity > 0: + if current_tick >= start_tick and current_tick < end_tick: + active_notes.add(msg.note) + + # Second pass: include all relevant events + current_tick = 0 + for msg in track: + if not hasattr(msg, 'time'): + continue + + prev_tick = current_tick + current_tick += msg.time + + # Include message if: + # 1. It's within our time range + # 2. OR it's a note_off for an active note + # 3. OR it's a control message + in_range = current_tick >= start_tick and current_tick < end_tick + is_note_release = msg.type == 'note_off' or (msg.type == 'note_on' and msg.velocity == 0) + is_active_note = hasattr(msg, 'note') and msg.note in active_notes + is_control = msg.type in ['control_change', 'program_change', 'pitchwheel'] + + if in_range or (is_note_release and is_active_note) or is_control: + new_msg = msg.copy() + + # Adjust timing for the first event + if not new_track or current_tick == start_tick: + new_msg.time = 0 + else: + new_msg.time = current_tick - prev_tick + + new_track.append(new_msg) + + # Remove from active notes if it's a note off + if is_note_release and is_active_note: + active_notes.remove(msg.note) + + # If we actually need to trim the audio + if start_time > 0 or duration > 0: + midi_data = trimmed_midi + + return (midi_data,) + + except Exception as e: + raise RuntimeError(f"Error loading MIDI file: {type(e).__name__}: {str(e)}") + + @classmethod + def analyze_midi(cls, midi_path, start_time=0.0, duration=0.0): + """Analyze the MIDI file and return available notes within the time range""" + midi_data = mido.MidiFile(midi_path) + + # Get tempo for time calculations + tempo = 500000 # Default tempo (microseconds per quarter note) + for track in midi_data.tracks: + for msg in track: + if msg.type == 'set_tempo': + tempo = msg.tempo + break + + # Calculate seconds per tick for time conversion + seconds_per_tick = tempo / (midi_data.ticks_per_beat * 1000000.0) + + # Convert seconds to ticks + start_tick = int(start_time / seconds_per_tick) if start_time > 0 else 0 + end_tick = float('inf') + if duration > 0: + end_tick = start_tick + int(duration / seconds_per_tick) + + # Collect track data and note data within time range + tracks = ["all"] + all_notes = set() + track_notes = {} + + for i, track in enumerate(midi_data.tracks): + track_notes[str(i)] = set() + current_tick = 0 + + for msg in track: + if hasattr(msg, 'time'): + current_tick += msg.time + + # Only collect notes in our time range + if current_tick >= start_tick and current_tick < end_tick: + if msg.type == 'note_on' and hasattr(msg, 'velocity') and msg.velocity > 0: + track_notes[str(i)].add(msg.note) + all_notes.add(msg.note) + + # Add track name info + track_name = getattr(track, 'name', None) or f'Track {i}' + if len(track_notes[str(i)]) == 0: + tracks.append(f"{i}: (Empty)") + else: + tracks.append(f"{i}: {track_name}") + + # Calculate total duration + total_ticks = 0 + for track in midi_data.tracks: + track_ticks = 0 + for msg in track: + if hasattr(msg, 'time'): + track_ticks += msg.time + total_ticks = max(total_ticks, track_ticks) + + total_duration = total_ticks * seconds_per_tick + + return { + "tracks": tracks, + "all_notes": ",".join(map(str, sorted(all_notes))), + "track_notes": {k: ",".join(map(str, sorted(v))) for k, v in track_notes.items()}, + "total_duration": total_duration, + "selected_start": start_time, + "selected_duration": duration if duration > 0 else (total_duration - start_time) + } + + @classmethod + def VALIDATE_INPUTS(cls, midi_file, track_selection, start_time=0.0, duration=0.0): + midi_path = folder_paths.get_full_path("midi_files", midi_file) + if not midi_path or not os.path.isfile(midi_path): + return f"MIDI file not found: {midi_file}" + + # Check if the file has a .mid or .midi extension + if not midi_file.lower().endswith(('.mid', '.midi')): + return f"Invalid file type. Expected .mid or .midi file, got: {midi_file}" + + if start_time < 0: + return f"Start time must be non-negative, got: {start_time}" + + if duration < 0: + return f"Duration must be non-negative, got: {duration}" + + return True + +# Server routes for MIDI file handling +routes = PromptServer.instance.routes + +@routes.post('/get_track_notes') +async def get_track_notes(request): + data = await request.json() + midi_file = data.get('midi_file') + start_time = data.get('start_time', 0) + duration = data.get('duration', 0) + + if not midi_file: + return web.json_response({"error": "Missing required parameters"}, status=400) + + midi_path = folder_paths.get_full_path("midi_files", midi_file) + if not midi_path or not os.path.exists(midi_path): + return web.json_response({"error": "MIDI file not found"}, status=404) + + # Analyze MIDI file with time filtering + analysis = MIDILoader.analyze_midi(midi_path, start_time, duration) + return web.json_response(analysis) + +@routes.post('/upload_midi') +async def upload_midi(request): + data = await request.post() + midi_file = data['file'] + + if midi_file and midi_file.filename: + safe_filename = os.path.basename(midi_file.filename) + + midi_dir = folder_paths.get_folder_paths("midi_files")[0] + + if not os.path.exists(midi_dir): + os.makedirs(midi_dir, exist_ok=True) + + midi_path = os.path.join(midi_dir, safe_filename) + + with open(midi_path, 'wb') as f: + shutil.copyfileobj(midi_file.file, f) + + midi_files = folder_paths.get_filename_list("midi_files") + analysis = MIDILoader.analyze_midi(midi_path) + + return web.json_response({ + "status": "success", + "uploaded_file": safe_filename, + "midi_files": midi_files, + "analysis": analysis + }) + else: + return web.json_response({"status": "error", "message": "No file uploaded"}, status=400) + +@routes.post('/refresh_midi_data') +async def refresh_midi_data(request): + data = await request.json() + midi_file = data.get('midi_file') + track_selection = data.get('track_selection') + start_time = data.get('start_time', 0) + duration = data.get('duration', 0) + + if not midi_file: + return web.json_response({"error": "Missing required parameters"}, status=400) + + midi_path = folder_paths.get_full_path("midi_files", midi_file) + if not midi_path or not os.path.exists(midi_path): + return web.json_response({"error": "MIDI file not found"}, status=404) + + # Get basic MIDI analysis with time filtering + analysis = MIDILoader.analyze_midi(midi_path, start_time, duration) + + # Filter notes by track if needed + if track_selection != "all": + track_index = track_selection.split(':')[0] + if track_index.isdigit() and track_index in analysis['track_notes']: + analysis['all_notes'] = analysis['track_notes'][track_index] + else: + analysis['all_notes'] = "" + + return web.json_response(analysis) + NODE_CLASS_MAPPINGS = { - "MIDIToAudio": MIDIToAudio + "MIDIToAudio": MIDIToAudio, + "MIDILoader": MIDILoader } NODE_DISPLAY_NAME_MAPPINGS = { - "MIDIToAudio": "MIDI to Audio" -} \ No newline at end of file + "MIDIToAudio": "MIDI to Audio", + "MIDILoader": "MIDI Loader" +} \ No newline at end of file From 8d107e4c92286d563e4c011e96045c67fedbf866 Mon Sep 17 00:00:00 2001 From: RyanOnTheInside <7623207+ryanontheinside@users.noreply.github.com> Date: Sat, 17 May 2025 13:53:56 -0400 Subject: [PATCH 03/10] moved fucntionalty --- __init__.py | 7 +- nodes/flex/feature_extractors_midi.py | 180 ++++-------- tooltips/categories/flex.py | 4 +- web/js/midi_extensions.js | 402 +++++++++++--------------- web/js/midi_loader_extensions.js | 373 ++++++++++++++++++++++++ 5 files changed, 601 insertions(+), 365 deletions(-) create mode 100644 web/js/midi_loader_extensions.js diff --git a/__init__.py b/__init__.py index d51a90e..186b6bd 100644 --- a/__init__.py +++ b/__init__.py @@ -150,7 +150,7 @@ from .nodes.flex.feature_extractors_audio import( ) from .nodes.flex.feature_extractors_midi import( - MIDILoadAndExtract, + MIDIFeatureExtractor, ) from .nodes.flex.feature_extractors_proximity import( @@ -329,7 +329,7 @@ from .nodes.doom.doom import Doom_ from .audio_latent_blend import AudioLatentBlend, FlexlatentAudioBlend -from .nodes.audio.midi_nodes import MIDIToAudio +from .nodes.audio.midi_nodes import MIDIToAudio, MIDILoader # Import external integrations from .external_integration import ( @@ -483,7 +483,7 @@ NODE_CLASS_MAPPINGS = { "PitchRange": PitchRangeNode, "PitchRangePreset": PitchRangePresetNode, "PitchRangeByNoteNode": PitchRangeByNoteNode, - "MIDILoadAndExtract": MIDILoadAndExtract, + "MIDIFeatureExtractor": MIDIFeatureExtractor, "TimeFeatureNode": TimeFeatureNode, "FloatFeatureNode": FloatFeatureNode, "ManualFeatureNode": ManualFeatureNode, @@ -592,6 +592,7 @@ NODE_CLASS_MAPPINGS = { "ImageIndexSelect": ImageIndexSelect, "MIDIPlayback": MIDIToAudio, + "MIDILoader": MIDILoader, } diff --git a/nodes/flex/feature_extractors_midi.py b/nodes/flex/feature_extractors_midi.py index fb59e3c..7d9c1fb 100644 --- a/nodes/flex/feature_extractors_midi.py +++ b/nodes/flex/feature_extractors_midi.py @@ -1,15 +1,18 @@ import mido import os import folder_paths -from server import PromptServer -from aiohttp import web -import shutil from .features_midi import MIDIFeature from .feature_extractors import FeatureExtractorBase from ...tooltips import apply_tooltips @apply_tooltips -class MIDILoadAndExtract(FeatureExtractorBase): +class MIDIFeatureExtractor(FeatureExtractorBase): + """Extract musical features from MIDI data based on selected notes from a piano interface. + + This node connects to a MIDILoader node to receive MIDI data and extract features + based on selected notes. Features can be used to modulate other parameters in the workflow. + """ + @classmethod def feature_type(cls) -> type[MIDIFeature]: return MIDIFeature @@ -18,27 +21,57 @@ class MIDILoadAndExtract(FeatureExtractorBase): def INPUT_TYPES(cls): parent_inputs = super().INPUT_TYPES()["required"] parent_inputs["extraction_method"] = (MIDIFeature.get_extraction_methods(),) + parent_inputs["frame_count"] = ("INT", {"default": 0, "min": 0}) # Changed default to 0 for auto-calculation return { "required": { **parent_inputs, - "midi_file": (folder_paths.get_filename_list("midi_files"),), - "track_selection": (["all"],), + "midi": ("MIDI",), "chord_only": ("BOOLEAN", {"default": False}), "notes": ("STRING", {"default": ""}), }, } - RETURN_TYPES = ("MIDI", "FEATURE") + RETURN_TYPES = ("MIDI", "FEATURE", "INT") + RETURN_NAMES = ("midi", "feature", "frame_count") FUNCTION = "process_midi" - + CATEGORY = "RyanOnTheInside/Audio/Features" - def process_midi(self, midi_file, track_selection, notes, extraction_method, frame_rate, frame_count, width, height, chord_only=False): + def calculate_target_frame_count(self, midi, frame_rate, frame_count): + """Calculate the target frame count based on MIDI duration and specified frame count""" + if frame_count > 0: + return frame_count + + # Calculate total duration in seconds + total_time = 0 + for track in midi.tracks: + track_time = 0 + for msg in track: + if hasattr(msg, 'time'): + track_time += msg.time + total_time = max(total_time, track_time) + + # Get tempo (default to 120 BPM if not specified) + tempo = 500000 # Default tempo in microseconds per quarter note + for track in midi.tracks: + for msg in track: + if msg.type == 'set_tempo': + tempo = msg.tempo + break + + # Convert ticks to seconds + seconds_per_tick = tempo / (midi.ticks_per_beat * 1000000.0) + duration_seconds = total_time * seconds_per_tick + + # Ensure we have some duration + duration_seconds = max(duration_seconds, 1.0) # At least 1 second + + # Calculate frame count + return max(1, int(duration_seconds * frame_rate)) + + def process_midi(self, midi, notes, extraction_method, frame_rate, frame_count, width, height, chord_only=False): try: - midi_path = folder_paths.get_full_path("midi_files", midi_file) - if not midi_path or not os.path.exists(midi_path): - raise FileNotFoundError(f"MIDI file not found: {midi_file}") - - midi_data = mido.MidiFile(midi_path) + # Calculate appropriate frame count + target_frame_count = self.calculate_target_frame_count(midi, frame_rate, frame_count) selected_notes = [int(n.strip()) for n in notes.split(',') if n.strip().isdigit()] @@ -47,10 +80,10 @@ class MIDILoadAndExtract(FeatureExtractorBase): feature = MIDIFeature( f"midi_{internal_attribute}", - midi_data, + midi, internal_attribute, frame_rate, - frame_count, + target_frame_count, # Use calculated frame count width, height, notes=selected_notes, @@ -59,119 +92,28 @@ class MIDILoadAndExtract(FeatureExtractorBase): feature.extract() - return (midi_data, feature) + return (midi, feature, target_frame_count) except Exception as e: - raise RuntimeError(f"Error processing MIDI file: {type(e).__name__}: {str(e)}") - - @classmethod - def analyze_midi(cls, midi_path): - midi_data = mido.MidiFile(midi_path) - tracks = ["all"] - all_notes = set() - track_notes = {} - for i, track in enumerate(midi_data.tracks): - track_notes[str(i)] = set() - for msg in track: - if msg.type == 'note_on': - track_notes[str(i)].add(msg.note) - all_notes.add(msg.note) - if len(track_notes[str(i)]) == 0: - tracks.append(f"{i}: (Empty)") - else: - tracks.append(f"{i}: {track.name or f'Track {i}'}") - - return { - "tracks": tracks, - "all_notes": ",".join(map(str, sorted(set(all_notes)))), - "track_notes": {k: ",".join(map(str, sorted(v))) for k, v in track_notes.items()} - } + raise RuntimeError(f"Error processing MIDI: {type(e).__name__}: {str(e)}") @classmethod - def VALIDATE_INPUTS(cls, midi_file, track_selection, notes, extraction_method, frame_rate): - midi_path = folder_paths.get_full_path("midi_files", midi_file) - if not midi_path or not os.path.isfile(midi_path): - return f"MIDI file not found: {midi_file}" - - # Check if the file has a .mid or .midi extension - if not midi_file.lower().endswith(('.mid', '.midi')): - return f"Invalid file type. Expected .mid or .midi file, got: {midi_file}" - - if notes != "all": + def VALIDATE_INPUTS(cls, midi, notes, extraction_method, frame_rate, frame_count, width, height, chord_only=False): + if notes: try: note_list = [int(n.strip()) for n in notes.split(',') if n.strip()] if not all(0 <= n <= 127 for n in note_list): return "Invalid note value. All notes must be between 0 and 127." except ValueError: - return "Invalid notes format. Please provide comma-separated integers or 'all'." + return "Invalid notes format. Please provide comma-separated integers." return True + +NODE_CLASS_MAPPINGS = { + "MIDIFeatureExtractor": MIDIFeatureExtractor, +} -routes = PromptServer.instance.routes -@PromptServer.instance.routes.post('/get_track_notes') -async def get_track_notes(request): - data = await request.json() - midi_file = data.get('midi_file') - - if not midi_file: - return web.json_response({"error": "Missing required parameters"}, status=400) - - midi_path = folder_paths.get_full_path("midi_files", midi_file) - if not midi_path or not os.path.exists(midi_path): - return web.json_response({"error": "MIDI file not found"}, status=404) - - analysis = MIDILoadAndExtract.analyze_midi(midi_path) - return web.json_response(analysis) - -@routes.post('/upload_midi') -async def upload_midi(request): - data = await request.post() - midi_file = data['file'] - - if midi_file and midi_file.filename: - safe_filename = os.path.basename(midi_file.filename) - - midi_dir = folder_paths.get_folder_paths("midi_files")[0] - - if not os.path.exists(midi_dir): - os.makedirs(midi_dir, exist_ok=True) - - midi_path = os.path.join(midi_dir, safe_filename) - - with open(midi_path, 'wb') as f: - shutil.copyfileobj(midi_file.file, f) - - midi_files = folder_paths.get_filename_list("midi_files") - analysis = MIDILoadAndExtract.analyze_midi(midi_path) - - return web.json_response({ - "status": "success", - "uploaded_file": safe_filename, - "midi_files": midi_files, - "analysis": analysis - }) - else: - return web.json_response({"status": "error", "message": "No file uploaded"}, status=400) - -@PromptServer.instance.routes.post('/refresh_midi_data') -async def refresh_midi_data(request): - data = await request.json() - midi_file = data.get('midi_file') - track_selection = data.get('track_selection') - - if not midi_file: - return web.json_response({"error": "Missing required parameters"}, status=400) - - midi_path = folder_paths.get_full_path("midi_files", midi_file) - if not midi_path or not os.path.exists(midi_path): - return web.json_response({"error": "MIDI file not found"}, status=404) - - analysis = MIDILoadAndExtract.analyze_midi(midi_path) - - # Filter notes based on track selection - if track_selection != "all": - track_index = track_selection.split(':')[0] - analysis['all_notes'] = analysis['track_notes'].get(track_index, "") - - return web.json_response(analysis) \ No newline at end of file +NODE_DISPLAY_NAME_MAPPINGS = { + "MIDIFeatureExtractor": "MIDI Feature Extractor", +} \ No newline at end of file diff --git a/tooltips/categories/flex.py b/tooltips/categories/flex.py index c3fb856..0ae09c5 100644 --- a/tooltips/categories/flex.py +++ b/tooltips/categories/flex.py @@ -225,7 +225,7 @@ Great for creating music videos, reactive animations, or automated effects.""") }, inherits_from='ProximityFeatureInput') # MIDILoadAndExtract tooltips (inherits from: FeatureExtractorBase) - TooltipManager.register_tooltips("MIDILoadAndExtract", { + TooltipManager.register_tooltips("MIDIFeatureExtractor", { "midi_file": "MIDI file to load from the midi_files directory", "extraction_method": """Choose what MIDI information to analyze: @@ -248,7 +248,7 @@ Great for creating music videos, reactive animations, or automated effects.""") "track_selection": "Track to analyze ('all' or specific track number)", "chord_only": "When true, only considers full chords (default: false)", "notes": "Comma-separated list of MIDI note numbers (default: empty)" - }, inherits_from='FeatureExtractorBase') + }, inherits_from='FeatureExtractorBase', description="Extract features from MIDI files. Analyze note on/off events, velocity, pitch, and more to create dynamic animations and effects. Use the piano to filter the notes that are considered for extraction.") # AudioFeatureExtractor tooltips (inherits from: FeatureExtractorBase) TooltipManager.register_tooltips("AudioFeatureExtractor", { diff --git a/web/js/midi_extensions.js b/web/js/midi_extensions.js index d1a352a..8cfd6e1 100644 --- a/web/js/midi_extensions.js +++ b/web/js/midi_extensions.js @@ -2,14 +2,11 @@ import { app } from "../../scripts/app.js"; import { drawPiano, handlePianoMouseDown } from "./piano.js"; app.registerExtension({ - name: "org.ryanontheinside.midi_load_and_extract", + name: "org.ryanontheinside.midi_feature_extractor", - // async setup() { - // console.log("MIDI Extension: Setup called"); - // }, - async beforeRegisterNodeDef(nodeType, nodeData, app) { - if (nodeData.name === "MIDILoadAndExtract") { + //claude: do not change the node name + if (nodeData.name === "MIDIFeatureExtractor") { // Add the disableWidget method to the nodeType prototype nodeType.prototype.disableWidget = function(widget) { @@ -21,47 +18,6 @@ app.registerExtension({ widget.onMouseUp = () => {}; } }; - - nodeType.prototype.uploadMIDI = function() { - const input = document.createElement("input"); - input.type = "file"; - input.accept = ".mid,.midi"; - input.onchange = (event) => { - const file = event.target.files[0]; - if (file) { - const formData = new FormData(); - formData.append("file", file); - - fetch("/upload_midi", { - method: "POST", - body: formData - }) - .then(response => response.json()) - .then(data => { - if (data.success) { - // Update the midi_file widget with the new file path - const midiFileWidget = this.widgets.find(w => w.name === "midi_file"); - if (midiFileWidget) { - midiFileWidget.value = data.file_path; - // Trigger any necessary updates - if (midiFileWidget.callback) { - midiFileWidget.callback(data.file_path); - } - } - // You might want to call updateTrackSelection here if it's needed - this.updateTrackSelection(); - this.saveState(); // Save state after successful upload - } else { - console.error("Failed to upload MIDI file:", data.error); - } - }) - .catch(error => { - console.error("Error uploading MIDI file:", error); - }); - } - }; - input.click(); - }; nodeType.prototype.onNodeCreated = function() { this.properties = this.properties || {}; @@ -71,14 +27,6 @@ app.registerExtension({ this.scrollRightIndicator = null; this.availableNotes = new Set(); - this.addWidget("button", "Upload MIDI", "upload", () => { - this.uploadMIDI(); - }); - - this.addWidget("button", "Refresh", "refresh", () => { - this.refreshMIDIData(); - }); - const notesWidget = this.widgets.find(widget => widget.name === "notes"); if (notesWidget) { // Disable the widget @@ -87,20 +35,6 @@ app.registerExtension({ notesWidget.hidden = true; } - // Add event listeners for midi_file and track_selection widgets - const midiFileWidget = this.widgets.find(w => w.name === "midi_file"); - if (midiFileWidget) { - midiFileWidget.callback = (value) => { - this.updateTrackSelection(); - this.saveState(); // Save state when MIDI file changes - }; - } - - const trackSelectionWidget = this.widgets.find(w => w.name === "track_selection"); - if (trackSelectionWidget) { - trackSelectionWidget.callback = () => this.onTrackSelectionChange(trackSelectionWidget.value); - } - // Increase the node's height and width const extraHeight = 80; // Piano height + margins const pianoWidth = 1020; // Total width of the piano @@ -116,156 +50,197 @@ app.registerExtension({ // Load saved state this.loadSavedState(); - // Set up a flag to indicate initial load - this.isInitialLoad = true; - - // Instead of calling updateTrackSelection here, we'll set up a callback - // that will be called when the graph is available - this.onGraphAvailable = () => { - this.updateTrackSelection(); - }; + // Check for existing connections after a short delay + setTimeout(() => { + const connectedNodes = this.graph._nodes.filter( + n => n.type === 'MIDILoader' && this.isNodeConnected(n) + ); + + if (connectedNodes.length > 0) { + const loaderNode = connectedNodes[0]; + this.queryMIDILoaderNotes(loaderNode); + this.listenToMIDILoaderChanges(loaderNode); + } + }, 100); // Small delay to ensure links are established }; - nodeType.prototype.loadSavedState = function() { - const savedState = localStorage.getItem(`MIDILoadAndExtract_${this.id}`); - if (savedState) { - this.loadedState = JSON.parse(savedState); - - const midiFileWidget = this.widgets.find(w => w.name === "midi_file"); - if (midiFileWidget && this.loadedState.midiFile) { - midiFileWidget.value = this.loadedState.midiFile; + // Handle connections changing + nodeType.prototype.onConnectionsChange = function(type, index, connected, link_info) { + if (type === 1) { // Input connection (1 is for inputs, 0 is for outputs) + if (connected && link_info) { + // New connection established + const inputNode = this.graph.getNodeById(link_info.origin_id); + if (inputNode && inputNode.type === 'MIDILoader') { + this.queryMIDILoaderNotes(inputNode); + this.listenToMIDILoaderChanges(inputNode); + } + } else { + // Check if we still have any connected MIDILoader nodes + const connectedMidiNodes = this.graph._nodes.filter( + n => n.type === 'MIDILoader' && this.isNodeConnected(n) + ); + + if (connectedMidiNodes.length === 0) { + // All MIDILoader connections removed + this.availableNotes = new Set(); + this.properties.selectedNotes = []; + this.syncNotesWidget(); + this.setDirtyCanvas(true, true); + } } } }; - nodeType.prototype.updateTrackSelection = async function() { - if (this.isUpdatingTrackSelection) { - console.log('Track selection update already in progress, skipping'); - return; + // Check if a node is connected to this node + nodeType.prototype.isNodeConnected = function(node) { + for (const link_id in this.graph.links) { + const link = this.graph.links[link_id]; + if (link.target_id === this.id && link.origin_id === node.id) { + return true; + } } - this.isUpdatingTrackSelection = true; + return false; + }; + // Query the MIDILoader for available notes + nodeType.prototype.queryMIDILoaderNotes = async function(loaderNode) { + if (!loaderNode) return; + + const midiFile = loaderNode.getCurrentMidiFile(); + const trackSelection = loaderNode.widgets.find(w => w.name === "track_selection").value; + + // Get time selection parameters from the MIDILoader + const startTimeWidget = loaderNode.widgets.find(w => w.name === "start_time"); + const durationWidget = loaderNode.widgets.find(w => w.name === "duration"); + + const startTime = startTimeWidget ? startTimeWidget.value : 0; + const duration = durationWidget ? durationWidget.value : 0; + + if (!midiFile) return; + try { - const midiFile = this.getCurrentMidiFile(); - - if (!midiFile) { - console.warn('No MIDI file selected'); - return; - } - - const response = await fetch('/get_track_notes', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ midi_file: midiFile }) - }); - const data = await response.json(); + console.log(`Querying MIDI notes with start_time=${startTime}, duration=${duration}`); - if (data.error) { - console.error('Error fetching track notes:', data.error); - return; - } - - - const trackSelectionWidget = this.widgets.find(w => w.name === "track_selection"); - if (trackSelectionWidget) { - // Update the options - trackSelectionWidget.options.values = data.tracks; - } - - // Update available notes - this.availableNotes = new Set(data.all_notes.split(',').map(Number)); - - // Apply loaded state if this is the initial load - if (this.isInitialLoad && this.loadedState) { - this.applyLoadedState(); - this.isInitialLoad = false; - } - - this.setDirtyCanvas(true, true); - } catch (error) { - console.error('Error updating track selection:', error); - this.availableNotes = new Set(); // Reset to empty Set in case of error - } finally { - this.isUpdatingTrackSelection = false; - } - }; - - nodeType.prototype.refreshMIDIData = async function() { - const midiFile = this.getCurrentMidiFile(); - const trackSelection = this.widgets.find(w => w.name === "track_selection").value; - - if (!midiFile) { - console.warn('No MIDI file selected'); - return; - } - - try { const response = await fetch('/refresh_midi_data', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ midi_file: midiFile, track_selection: trackSelection }) + body: JSON.stringify({ + midi_file: midiFile, + track_selection: trackSelection, + start_time: startTime, + duration: duration + }) }); const data = await response.json(); if (data.error) { - console.error('Error refreshing MIDI data:', data.error); + console.error('Error fetching MIDI notes:', data.error); return; } - - // Update available tracks - const trackSelectionWidget = this.widgets.find(w => w.name === "track_selection"); - if (trackSelectionWidget) { - trackSelectionWidget.options.values = data.tracks; - } - // Update available notes - this.availableNotes = new Set(data.all_notes.split(',').map(Number)); + const noteArray = data.all_notes.split(',').map(Number).filter(n => !isNaN(n)); + console.log(`Received ${noteArray.length} available notes in time range ${startTime}s - ${startTime + duration}s`); + + this.availableNotes = new Set(noteArray); // Filter out selected notes that are no longer available if (this.properties.selectedNotes) { - this.properties.selectedNotes = this.properties.selectedNotes.filter(note => this.availableNotes.has(note)); - } else { - this.properties.selectedNotes = []; - } - this.syncNotesWidget(); - - // Trigger redraw - this.setDirtyCanvas(true, true); - } catch (error) { - console.error('Error refreshing MIDI data:', error); - } - }; - - nodeType.prototype.applyLoadedState = function() { - if (this.loadedState) { - - const trackSelectionWidget = this.widgets.find(w => w.name === "track_selection"); - if (trackSelectionWidget && this.loadedState.trackSelection) { - if (trackSelectionWidget.options.values.includes(this.loadedState.trackSelection)) { - trackSelectionWidget.value = this.loadedState.trackSelection; - } else { - console.log(`Loaded track selection ${this.loadedState.trackSelection} is no longer valid`); + const originalCount = this.properties.selectedNotes.length; + this.properties.selectedNotes = this.properties.selectedNotes.filter( + note => this.availableNotes.has(note) + ); + + if (originalCount !== this.properties.selectedNotes.length) { + console.log(`Filtered out ${originalCount - this.properties.selectedNotes.length} selected notes that aren't in the time range`); } } - if (this.loadedState.selectedNotes) { - this.properties.selectedNotes = this.loadedState.selectedNotes.filter(note => this.availableNotes.has(note)); - } - this.syncNotesWidget(); - delete this.loadedState; - this.saveState(); // Save the applied state + this.setDirtyCanvas(true, true); + } catch (error) { + console.error('Error querying MIDI loader:', error); + } + }; + + // Listen for changes in the MIDILoader node + nodeType.prototype.listenToMIDILoaderChanges = function(loaderNode) { + // Store reference to loader node + this.connectedLoaderNode = loaderNode; + + // Check if we already added a listener to this node + if (loaderNode._feature_extractor_listeners && + loaderNode._feature_extractor_listeners.includes(this.id)) { + return; + } + + // Initialize listener array if needed + if (!loaderNode._feature_extractor_listeners) { + loaderNode._feature_extractor_listeners = []; + } + loaderNode._feature_extractor_listeners.push(this.id); + + // Find widgets in loader node + const trackWidget = loaderNode.widgets.find(w => w.name === "track_selection"); + const startTimeWidget = loaderNode.widgets.find(w => w.name === "start_time"); + const durationWidget = loaderNode.widgets.find(w => w.name === "duration"); + + // Set up callback for track selection changes + if (trackWidget) { + const origTrackCallback = trackWidget.callback; + trackWidget.callback = (value) => { + // Call original callback + if (origTrackCallback) origTrackCallback.call(loaderNode, value); + + // Notify this feature extractor to update notes + this.queryMIDILoaderNotes(loaderNode); + }; + } + + // Set up callback for start time changes + if (startTimeWidget) { + const origCallback = startTimeWidget.callback; + startTimeWidget.callback = (value) => { + // Call original callback + if (origCallback) origCallback.call(loaderNode, value); + + // Notify this feature extractor to update notes + this.queryMIDILoaderNotes(loaderNode); + }; + } + + // Set up callback for duration changes + if (durationWidget) { + const origCallback = durationWidget.callback; + durationWidget.callback = (value) => { + // Call original callback + if (origCallback) origCallback.call(loaderNode, value); + + // Notify this feature extractor to update notes + this.queryMIDILoaderNotes(loaderNode); + }; + } + }; + + nodeType.prototype.loadSavedState = function() { + const savedState = localStorage.getItem(`MIDIFeatureExtractor_${this.id}`); + if (savedState) { + try { + const state = JSON.parse(savedState); + if (state.selectedNotes) { + this.properties.selectedNotes = state.selectedNotes; + } + } catch (e) { + console.error("Error loading saved state:", e); + } } }; nodeType.prototype.saveState = function() { const state = { - midiFile: this.getCurrentMidiFile(), - trackSelection: this.widgets.find(w => w.name === "track_selection")?.value, selectedNotes: this.properties.selectedNotes }; - localStorage.setItem(`MIDILoadAndExtract_${this.id}`, JSON.stringify(state)); + localStorage.setItem(`MIDIFeatureExtractor_${this.id}`, JSON.stringify(state)); }; nodeType.prototype.syncNotesWidget = function() { @@ -283,10 +258,8 @@ app.registerExtension({ // Ensure the node updates its internal state this.onPropertyChanged('selectedNotes'); - // Only call saveState if it's available - if (typeof this.saveState === 'function') { - this.saveState(); - } + // Save state + this.saveState(); }; nodeType.prototype.onPropertyChanged = function(property) { @@ -309,7 +282,6 @@ app.registerExtension({ // Ensure availableNotes exists before drawing if (!this.availableNotes) { - console.warn('availableNotes is undefined, initializing as empty Set'); this.availableNotes = new Set(); } @@ -338,58 +310,6 @@ app.registerExtension({ const [x, y] = pos; return handlePianoMouseDown(this, x, y); }; - - nodeType.prototype.getCurrentMidiFile = function() { - const midiFileWidget = this.widgets.find(w => w.name === "midi_file"); - return midiFileWidget ? midiFileWidget.value : null; - }; - - nodeType.prototype.onTrackSelectionChange = async function(selectedTrack) { - try { - const midiFile = this.getCurrentMidiFile(); - const response = await fetch('/get_track_notes', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ midi_file: midiFile }) - }); - const data = await response.json(); - - if (data.error) { - console.error('Error fetching track notes:', data.error); - return; - } - - const notesInTrack = selectedTrack === "all" - ? data.all_notes - : data.track_notes[selectedTrack.split(':')[0]]; - - this.availableNotes = new Set(notesInTrack.split(',').map(Number)); - - // Filter out selected notes that are no longer available - if (this.properties.selectedNotes) { - this.properties.selectedNotes = this.properties.selectedNotes.filter(note => this.availableNotes.has(note)); - } else { - this.properties.selectedNotes = []; - } - this.syncNotesWidget(); - - // Trigger redraw - this.setDirtyCanvas(true, true); - } catch (error) { - console.error('Error updating piano notes:', error); - } - }; - - // Add this at the end of the function - const onAdded = app.graph.onNodeAdded; - app.graph.onNodeAdded = function(node) { - if (onAdded) { - onAdded.call(this, node); - } - if (node.type === "MIDILoadAndExtract" && node.onGraphLoaded) { - node.onGraphLoaded(); - } - }; } }, }); \ No newline at end of file diff --git a/web/js/midi_loader_extensions.js b/web/js/midi_loader_extensions.js new file mode 100644 index 0000000..7389e70 --- /dev/null +++ b/web/js/midi_loader_extensions.js @@ -0,0 +1,373 @@ +import { app } from "../../scripts/app.js"; +import { drawPiano, handlePianoMouseDown } from "./piano.js"; + +app.registerExtension({ + name: "org.ryanontheinside.midi_loader", + + async beforeRegisterNodeDef(nodeType, nodeData, app) { + if (nodeData.name === "MIDILoader") { + + nodeType.prototype.disableWidget = function(widget) { + if (widget) { + widget.disabled = true; + widget.visibleWidth = 0; + widget.onMouseDown = () => {}; + widget.onMouseMove = () => {}; + widget.onMouseUp = () => {}; + } + }; + + nodeType.prototype.uploadMIDI = function() { + const input = document.createElement("input"); + input.type = "file"; + input.accept = ".mid,.midi"; + input.onchange = (event) => { + const file = event.target.files[0]; + if (file) { + const formData = new FormData(); + formData.append("file", file); + + fetch("/upload_midi", { + method: "POST", + body: formData + }) + .then(response => response.json()) + .then(data => { + if (data.status === "success") { + // Update the midi_file widget with the new file + const midiFileWidget = this.widgets.find(w => w.name === "midi_file"); + if (midiFileWidget) { + midiFileWidget.value = data.uploaded_file; + // Trigger any necessary updates + if (midiFileWidget.callback) { + midiFileWidget.callback(data.uploaded_file); + } + } + this.updateTrackSelection(); + this.saveState(); + } else { + console.error("Failed to upload MIDI file:", data.message); + } + }) + .catch(error => { + console.error("Error uploading MIDI file:", error); + }); + } + }; + input.click(); + }; + + const originalNodeCreated = nodeType.prototype.onNodeCreated; + nodeType.prototype.onNodeCreated = function() { + this.properties = this.properties || {}; + this.pianoScroll = 0; + this.scrollLeftIndicator = null; + this.scrollRightIndicator = null; + this.availableNotes = new Set(); + + this.addWidget("button", "Upload MIDI", "upload", () => { + this.uploadMIDI(); + }); + + this.addWidget("button", "Refresh", "refresh", () => { + this.refreshMIDIData(); + }); + + // Add event listeners for midi_file widget + const midiFileWidget = this.widgets.find(w => w.name === "midi_file"); + if (midiFileWidget) { + midiFileWidget.callback = (value) => { + this.updateTrackSelection(); + this.saveState(); // Save state when MIDI file changes + }; + } + + const trackSelectionWidget = this.widgets.find(w => w.name === "track_selection"); + if (trackSelectionWidget) { + const origTrackCallback = trackSelectionWidget.callback; + trackSelectionWidget.callback = (value) => { + // Call original callback if it exists + if (origTrackCallback) origTrackCallback.call(this, value); + this.saveState(); + + // Notify connected nodes of the change + this.notifyConnectedNodes(); + }; + } + + // Set up callbacks for start time and duration widgets + const startTimeWidget = this.widgets.find(w => w.name === "start_time_seconds"); + if (startTimeWidget) { + const origStartCallback = startTimeWidget.callback; + startTimeWidget.callback = (value) => { + // Call original callback if it exists + if (origStartCallback) origStartCallback.call(this, value); + this.saveState(); + + // Notify connected nodes of the change + this.notifyConnectedNodes(); + }; + } + + const durationWidget = this.widgets.find(w => w.name === "duration_seconds"); + if (durationWidget) { + const origDurCallback = durationWidget.callback; + durationWidget.callback = (value) => { + // Call original callback if it exists + if (origDurCallback) origDurCallback.call(this, value); + this.saveState(); + + // Notify connected nodes of the change + this.notifyConnectedNodes(); + }; + } + + // Increase the node's height and width + const extraHeight = 80; // Piano height + margins + const pianoWidth = 380; // Reduced width for just the loader + this.size[1] += extraHeight; + this.size[0] = Math.max(this.size[0], pianoWidth); + + // Ensure the node recomputes its size + this.setDirtyCanvas(true, true); + + // Load saved state + this.loadSavedState(); + + // Set up a flag to indicate initial load + this.isInitialLoad = true; + + // Set up a callback that will be called when the graph is available + this.onGraphAvailable = () => { + this.updateTrackSelection(); + }; + + // Initialize the feature extractor listeners array + this._feature_extractor_listeners = []; + + // Same for refreshMIDIData method + const originalRefreshMIDIData = this.refreshMIDIData; + this.refreshMIDIData = async function() { + await originalRefreshMIDIData.call(this); + this.notifyConnectedNodes(); + }; + }; + + nodeType.prototype.loadSavedState = function() { + const savedState = localStorage.getItem(`MIDILoader_${this.id}`); + if (savedState) { + this.loadedState = JSON.parse(savedState); + + const midiFileWidget = this.widgets.find(w => w.name === "midi_file"); + if (midiFileWidget && this.loadedState.midiFile) { + midiFileWidget.value = this.loadedState.midiFile; + } + + if (this.loadedState.trackSelection) { + const trackWidget = this.widgets.find(w => w.name === "track_selection"); + if (trackWidget) { + trackWidget.value = this.loadedState.trackSelection; + } + } + + // Restore measure/beat selection + if (this.loadedState.startMeasure !== undefined) { + const widget = this.widgets.find(w => w.name === "start_measure"); + if (widget) widget.value = this.loadedState.startMeasure; + } + + if (this.loadedState.startBeat !== undefined) { + const widget = this.widgets.find(w => w.name === "start_beat"); + if (widget) widget.value = this.loadedState.startBeat; + } + + if (this.loadedState.numMeasures !== undefined) { + const widget = this.widgets.find(w => w.name === "num_measures"); + if (widget) widget.value = this.loadedState.numMeasures; + } + } + }; + + nodeType.prototype.updateTrackSelection = async function() { + if (this.isUpdatingTrackSelection) { + console.log('Track selection update already in progress, skipping'); + return; + } + this.isUpdatingTrackSelection = true; + + try { + const midiFile = this.getCurrentMidiFile(); + + if (!midiFile) { + console.warn('No MIDI file selected'); + return; + } + + // Get start time and duration values + const startTimeWidget = this.widgets.find(w => w.name === "start_time_seconds"); + const durationWidget = this.widgets.find(w => w.name === "duration_seconds"); + const startTime = startTimeWidget ? startTimeWidget.value : 0; + const duration = durationWidget ? durationWidget.value : 0; + + const response = await fetch('/get_track_notes', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + midi_file: midiFile, + start_time_seconds: startTime, + duration_seconds: duration + }) + }); + const data = await response.json(); + + if (data.error) { + console.error('Error fetching track notes:', data.error); + return; + } + + const trackWidget = this.widgets.find(w => w.name === "track_selection"); + if (trackWidget) { + // Update the options + trackWidget.options.values = data.tracks; + } + + // Apply loaded state if this is the initial load + if (this.isInitialLoad && this.loadedState) { + if (this.loadedState.trackSelection) { + const trackWidget = this.widgets.find(w => w.name === "track_selection"); + if (trackWidget && trackWidget.options.values.includes(this.loadedState.trackSelection)) { + trackWidget.value = this.loadedState.trackSelection; + } + } + + // Apply saved start time and duration + if (this.loadedState.startTimeSeconds !== undefined && startTimeWidget) { + startTimeWidget.value = this.loadedState.startTimeSeconds; + } + + if (this.loadedState.durationSeconds !== undefined && durationWidget) { + durationWidget.value = this.loadedState.durationSeconds; + } + + this.isInitialLoad = false; + } + + this.setDirtyCanvas(true, true); + + // Notify connected nodes after track selection update + this.notifyConnectedNodes(); + } catch (error) { + console.error('Error updating track selection:', error); + } finally { + this.isUpdatingTrackSelection = false; + } + }; + + nodeType.prototype.refreshMIDIData = async function() { + const midiFile = this.getCurrentMidiFile(); + const trackWidget = this.widgets.find(w => w.name === "track_selection"); + const trackSelection = trackWidget ? trackWidget.value : "all"; + + // Get start time and duration values + const startTimeWidget = this.widgets.find(w => w.name === "start_time_seconds"); + const durationWidget = this.widgets.find(w => w.name === "duration_seconds"); + const startTime = startTimeWidget ? startTimeWidget.value : 0; + const duration = durationWidget ? durationWidget.value : 0; + + if (!midiFile) { + console.warn('No MIDI file selected'); + return; + } + + try { + const response = await fetch('/refresh_midi_data', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + midi_file: midiFile, + track_selection: trackSelection, + start_time_seconds: startTime, + duration_seconds: duration + }) + }); + const data = await response.json(); + + if (data.error) { + console.error('Error refreshing MIDI data:', data.error); + return; + } + + // Update available tracks + const trackWidget = this.widgets.find(w => w.name === "track_selection"); + if (trackWidget) { + trackWidget.options.values = data.tracks; + } + + // Trigger redraw + this.setDirtyCanvas(true, true); + } catch (error) { + console.error('Error refreshing MIDI data:', error); + } + }; + + nodeType.prototype.saveState = function() { + // Get current widgets' values + const startMeasureWidget = this.widgets.find(w => w.name === "start_measure"); + const startBeatWidget = this.widgets.find(w => w.name === "start_beat"); + const numMeasuresWidget = this.widgets.find(w => w.name === "num_measures"); + + const state = { + midiFile: this.getCurrentMidiFile(), + trackSelection: this.widgets.find(w => w.name === "track_selection")?.value, + startMeasure: startMeasureWidget ? startMeasureWidget.value : 1, + startBeat: startBeatWidget ? startBeatWidget.value : 1, + numMeasures: numMeasuresWidget ? numMeasuresWidget.value : 0 + }; + localStorage.setItem(`MIDILoader_${this.id}`, JSON.stringify(state)); + }; + + nodeType.prototype.getCurrentMidiFile = function() { + const midiFileWidget = this.widgets.find(w => w.name === "midi_file"); + return midiFileWidget ? midiFileWidget.value : null; + }; + + // Add a method to notify connected feature extractors + nodeType.prototype.notifyConnectedNodes = function() { + if (!this._feature_extractor_listeners) return; + + for (const id of this._feature_extractor_listeners) { + const node = this.graph.getNodeById(id); + if (node && node.queryMIDILoaderNotes) { + node.queryMIDILoaderNotes(this); + } + } + }; + + // Handle disconnection to clean up listeners + nodeType.prototype.onConnectionsChange = function(type, index, connected, link_info) { + if (type === 0 && !connected) { // Output disconnected + // Clean up the disconnected listener + if (link_info && this._feature_extractor_listeners) { + const targetNode = this.graph.getNodeById(link_info.target_id); + if (targetNode) { + this._feature_extractor_listeners = this._feature_extractor_listeners.filter( + id => id !== targetNode.id + ); + } + } + } + }; + + // Add this at the end of the function + const onAdded = app.graph.onNodeAdded; + app.graph.onNodeAdded = function(node) { + if (onAdded) { + onAdded.call(this, node); + } + if (node.type === "MIDILoader" && node.onGraphLoaded) { + node.onGraphLoaded(); + } + }; + } + }, +}); \ No newline at end of file From e75dbb7156d34ee0d0dcfba629be0acb7f9c2538 Mon Sep 17 00:00:00 2001 From: RyanOnTheInside <7623207+ryanontheinside@users.noreply.github.com> Date: Sat, 17 May 2025 13:57:29 -0400 Subject: [PATCH 04/10] measures, bars --- nodes/audio/midi_nodes.py | 341 ++++++++++++++++++++++---------------- web/js/midi_extensions.js | 63 ++++--- 2 files changed, 236 insertions(+), 168 deletions(-) diff --git a/nodes/audio/midi_nodes.py b/nodes/audio/midi_nodes.py index f705fb5..f6be572 100644 --- a/nodes/audio/midi_nodes.py +++ b/nodes/audio/midi_nodes.py @@ -3,6 +3,7 @@ import numpy as np import mido import os import folder_paths +import math from .audio_nodes import AudioNodeBase from ...tooltips import apply_tooltips from server import PromptServer @@ -144,9 +145,71 @@ class MIDIToAudio(AudioNodeBase): "sample_rate": sample_rate },) +def calculate_midi_total_measures(midi_data): + """Calculate the total number of measures in the MIDI file""" + # Get time signature (defaults to 4/4) + time_sig_numerator = 4 + time_sig_denominator = 4 + + for track in midi_data.tracks: + for msg in track: + if msg.type == 'time_signature': + time_sig_numerator = msg.numerator + time_sig_denominator = msg.denominator + break + + # Calculate total ticks + total_ticks = 0 + for track in midi_data.tracks: + track_ticks = 0 + for msg in track: + if hasattr(msg, 'time'): + track_ticks += msg.time + total_ticks = max(total_ticks, track_ticks) + + # Calculate ticks per measure + ticks_per_beat = midi_data.ticks_per_beat + ticks_per_measure = ticks_per_beat * 4 * (time_sig_numerator / time_sig_denominator) + + # Calculate total measures + total_measures = math.ceil(total_ticks / ticks_per_measure) + return total_measures, time_sig_numerator, time_sig_denominator + +def convert_measures_to_ticks(midi_data, start_measure, start_beat=1, num_measures=0): + """Convert musical measures to MIDI ticks""" + # Get time signature (defaults to 4/4) + time_sig_numerator = 4 + time_sig_denominator = 4 + + for track in midi_data.tracks: + for msg in track: + if msg.type == 'time_signature': + time_sig_numerator = msg.numerator + time_sig_denominator = msg.denominator + break + + # Calculate ticks per beat and measure + ticks_per_beat = midi_data.ticks_per_beat + beats_per_measure = time_sig_numerator + + # Adjust start measure/beat to 0-indexed for calculation + start_measure_index = start_measure - 1 + start_beat_index = start_beat - 1 + + # Calculate start tick + start_tick = (start_measure_index * beats_per_measure + start_beat_index) * ticks_per_beat + + # Calculate end tick if duration is specified + if num_measures > 0: + end_tick = start_tick + (num_measures * beats_per_measure * ticks_per_beat) + else: + end_tick = float('inf') + + return start_tick, end_tick, time_sig_numerator, time_sig_denominator + @apply_tooltips class MIDILoader: - """Loads MIDI files for processing in ComfyUI""" + """Loads MIDI files for processing in ComfyUI with options for selecting specific measures.""" @classmethod def INPUT_TYPES(cls): @@ -154,8 +217,9 @@ class MIDILoader: "required": { "midi_file": (folder_paths.get_filename_list("midi_files"),), "track_selection": (["all"],), - "start_time": ("FLOAT", {"default": 0.0, "min": 0.0, "step": 0.1}), - "duration": ("FLOAT", {"default": 0.0, "min": 0.0, "step": 0.1, "description": "Duration in seconds (0 = full duration)"}), + "start_measure": ("INT", {"default": 1, "min": 1, "step": 1}), + "start_beat": ("INT", {"default": 1, "min": 1, "step": 1}), + "num_measures": ("INT", {"default": 0, "min": 0, "step": 1, "description": "Number of measures (0 = all remaining measures)"}) } } @@ -163,7 +227,7 @@ class MIDILoader: FUNCTION = "load_midi" CATEGORY = "RyanOnTheInside/Audio/MIDI" - def load_midi(self, midi_file, track_selection, start_time=0.0, duration=0.0): + def load_midi(self, midi_file, track_selection, start_measure=1, start_beat=1, num_measures=0): try: midi_path = folder_paths.get_full_path("midi_files", midi_file) if not midi_path or not os.path.exists(midi_path): @@ -172,34 +236,6 @@ class MIDILoader: # Load the MIDI file midi_data = mido.MidiFile(midi_path) - # Get tempo for time calculations - tempo = 500000 # Default tempo (microseconds per quarter note) - for track in midi_data.tracks: - for msg in track: - if msg.type == 'set_tempo': - tempo = msg.tempo - break - - # Calculate seconds per tick for time conversion - seconds_per_tick = tempo / (midi_data.ticks_per_beat * 1000000.0) - - # Convert seconds to ticks - start_tick = int(start_time / seconds_per_tick) if start_time > 0 else 0 - end_tick = float('inf') - if duration > 0: - end_tick = start_tick + int(duration / seconds_per_tick) - - # Calculate total duration - total_ticks = 0 - for track in midi_data.tracks: - track_ticks = 0 - for msg in track: - if hasattr(msg, 'time'): - track_ticks += msg.time - total_ticks = max(total_ticks, track_ticks) - - total_duration = total_ticks * seconds_per_tick - # Apply track selection if not "all" if track_selection != "all": track_index_str = track_selection.split(':')[0].strip() @@ -218,75 +254,65 @@ class MIDILoader: midi_data = selected_midi - # Apply time slicing if needed - if (start_time > 0 or duration > 0) and midi_data.tracks: - # Create a new MIDI file with the selected time range + # Apply measure slicing if needed + if (start_measure > 1 or start_beat > 1 or num_measures > 0) and midi_data.tracks: + # Convert musical measures to ticks + start_tick, end_tick, _, _ = convert_measures_to_ticks(midi_data, start_measure, start_beat, num_measures) + + # Create a new MIDI file with the selected measures trimmed_midi = mido.MidiFile(ticks_per_beat=midi_data.ticks_per_beat) - # Copy data for the trimmed section - # NOTE: This is a simplified approach and may not handle all MIDI events perfectly for track in midi_data.tracks: new_track = mido.MidiTrack() trimmed_midi.tracks.append(new_track) - # Copy all metadata and control messages - for msg in track: - if not hasattr(msg, 'time') or msg.type in ['time_signature', 'key_signature', 'set_tempo']: - new_track.append(msg.copy()) - - # Extract all note events within our time range - current_tick = 0 - last_tick = 0 - - # First pass: collect all note_on events within our range - active_notes = set() + # Copy all metadata and non-time-based messages for msg in track: if not hasattr(msg, 'time'): - continue - - current_tick += msg.time - - # Track note_on events within our range - if msg.type == 'note_on' and msg.velocity > 0: - if current_tick >= start_tick and current_tick < end_tick: - active_notes.add(msg.note) + new_track.append(msg) - # Second pass: include all relevant events + # Add all messages that fall within our measure range current_tick = 0 + first_msg_added = False + prev_tick = 0 + for msg in track: if not hasattr(msg, 'time'): - continue - - prev_tick = current_tick + continue # Skip metadata (already copied) + current_tick += msg.time - # Include message if: - # 1. It's within our time range - # 2. OR it's a note_off for an active note - # 3. OR it's a control message - in_range = current_tick >= start_tick and current_tick < end_tick - is_note_release = msg.type == 'note_off' or (msg.type == 'note_on' and msg.velocity == 0) - is_active_note = hasattr(msg, 'note') and msg.note in active_notes - is_control = msg.type in ['control_change', 'program_change', 'pitchwheel'] - - if in_range or (is_note_release and is_active_note) or is_control: - new_msg = msg.copy() + # Include messages that are within our measure range + if current_tick >= start_tick and current_tick <= end_tick: + # Create an adjusted message with corrected timing + adjusted_msg = msg.copy() - # Adjust timing for the first event - if not new_track or current_tick == start_tick: - new_msg.time = 0 + # If this is the first message in the trimmed section, + # adjust its time relative to the start point + if not first_msg_added: + adjusted_msg.time = 0 # First event starts immediately + first_msg_added = True + prev_tick = current_tick else: - new_msg.time = current_tick - prev_tick - - new_track.append(new_msg) + # Preserve the relative timing between events + adjusted_msg.time = current_tick - prev_tick + prev_tick = current_tick - # Remove from active notes if it's a note off - if is_note_release and is_active_note: - active_notes.remove(msg.note) + new_track.append(adjusted_msg) + + # For note_off events or note_on with velocity 0 (note release), + # we need to include them if the corresponding note_on was included + elif current_tick > end_tick and first_msg_added: + if (msg.type == 'note_off' or + (msg.type == 'note_on' and hasattr(msg, 'velocity') and msg.velocity == 0)): + + # Add with adjusted timing + adjusted_msg = msg.copy() + adjusted_msg.time = current_tick - prev_tick + prev_tick = current_tick + new_track.append(adjusted_msg) - # If we actually need to trim the audio - if start_time > 0 or duration > 0: - midi_data = trimmed_midi + midi_data = trimmed_midi return (midi_data,) @@ -294,32 +320,22 @@ class MIDILoader: raise RuntimeError(f"Error loading MIDI file: {type(e).__name__}: {str(e)}") @classmethod - def analyze_midi(cls, midi_path, start_time=0.0, duration=0.0): - """Analyze the MIDI file and return available notes within the time range""" + def analyze_midi(cls, midi_path, start_measure=1, start_beat=1, num_measures=0): midi_data = mido.MidiFile(midi_path) - # Get tempo for time calculations - tempo = 500000 # Default tempo (microseconds per quarter note) - for track in midi_data.tracks: - for msg in track: - if msg.type == 'set_tempo': - tempo = msg.tempo - break + # Get the total measures + total_measures, time_sig_num, time_sig_denom = calculate_midi_total_measures(midi_data) - # Calculate seconds per tick for time conversion - seconds_per_tick = tempo / (midi_data.ticks_per_beat * 1000000.0) + # Convert measures to ticks if measure filtering is applied + if start_measure > 1 or start_beat > 1 or num_measures > 0: + start_tick, end_tick, _, _ = convert_measures_to_ticks(midi_data, start_measure, start_beat, num_measures) + else: + start_tick = 0 + end_tick = float('inf') - # Convert seconds to ticks - start_tick = int(start_time / seconds_per_tick) if start_time > 0 else 0 - end_tick = float('inf') - if duration > 0: - end_tick = start_tick + int(duration / seconds_per_tick) - - # Collect track data and note data within time range tracks = ["all"] all_notes = set() track_notes = {} - for i, track in enumerate(midi_data.tracks): track_notes[str(i)] = set() current_tick = 0 @@ -328,41 +344,27 @@ class MIDILoader: if hasattr(msg, 'time'): current_tick += msg.time - # Only collect notes in our time range - if current_tick >= start_tick and current_tick < end_tick: - if msg.type == 'note_on' and hasattr(msg, 'velocity') and msg.velocity > 0: + # Only consider notes within our measure range + if current_tick >= start_tick and current_tick <= end_tick: + if msg.type == 'note_on' and msg.velocity > 0: track_notes[str(i)].add(msg.note) all_notes.add(msg.note) - # Add track name info - track_name = getattr(track, 'name', None) or f'Track {i}' if len(track_notes[str(i)]) == 0: tracks.append(f"{i}: (Empty)") else: - tracks.append(f"{i}: {track_name}") - - # Calculate total duration - total_ticks = 0 - for track in midi_data.tracks: - track_ticks = 0 - for msg in track: - if hasattr(msg, 'time'): - track_ticks += msg.time - total_ticks = max(total_ticks, track_ticks) - - total_duration = total_ticks * seconds_per_tick + tracks.append(f"{i}: {getattr(track, 'name', '') or f'Track {i}'}") return { "tracks": tracks, - "all_notes": ",".join(map(str, sorted(all_notes))), + "all_notes": ",".join(map(str, sorted(set(all_notes)))), "track_notes": {k: ",".join(map(str, sorted(v))) for k, v in track_notes.items()}, - "total_duration": total_duration, - "selected_start": start_time, - "selected_duration": duration if duration > 0 else (total_duration - start_time) + "total_measures": total_measures, + "time_signature": f"{time_sig_num}/{time_sig_denom}" } @classmethod - def VALIDATE_INPUTS(cls, midi_file, track_selection, start_time=0.0, duration=0.0): + def VALIDATE_INPUTS(cls, midi_file, track_selection, start_measure=1, start_beat=1, num_measures=0): midi_path = folder_paths.get_full_path("midi_files", midi_file) if not midi_path or not os.path.isfile(midi_path): return f"MIDI file not found: {midi_file}" @@ -371,11 +373,23 @@ class MIDILoader: if not midi_file.lower().endswith(('.mid', '.midi')): return f"Invalid file type. Expected .mid or .midi file, got: {midi_file}" - if start_time < 0: - return f"Start time must be non-negative, got: {start_time}" + if start_measure < 1: + return f"Start measure must be at least 1, got: {start_measure}" - if duration < 0: - return f"Duration must be non-negative, got: {duration}" + if start_beat < 1: + return f"Start beat must be at least 1, got: {start_beat}" + + if num_measures < 0: + return f"Number of measures cannot be negative, got: {num_measures}" + + # Check if the start measure is valid for this MIDI file + try: + midi_data = mido.MidiFile(midi_path) + total_measures, _, _ = calculate_midi_total_measures(midi_data) + if start_measure > total_measures: + return f"Start measure {start_measure} exceeds the total measures in the file ({total_measures})" + except Exception as e: + return f"Error validating MIDI file: {str(e)}" return True @@ -386,8 +400,9 @@ routes = PromptServer.instance.routes async def get_track_notes(request): data = await request.json() midi_file = data.get('midi_file') - start_time = data.get('start_time', 0) - duration = data.get('duration', 0) + start_measure = data.get('start_measure', 1) + start_beat = data.get('start_beat', 1) + num_measures = data.get('num_measures', 0) if not midi_file: return web.json_response({"error": "Missing required parameters"}, status=400) @@ -396,8 +411,8 @@ async def get_track_notes(request): if not midi_path or not os.path.exists(midi_path): return web.json_response({"error": "MIDI file not found"}, status=404) - # Analyze MIDI file with time filtering - analysis = MIDILoader.analyze_midi(midi_path, start_time, duration) + # Analyze MIDI file + analysis = MIDILoader.analyze_midi(midi_path, start_measure, start_beat, num_measures) return web.json_response(analysis) @routes.post('/upload_midi') @@ -435,8 +450,9 @@ async def refresh_midi_data(request): data = await request.json() midi_file = data.get('midi_file') track_selection = data.get('track_selection') - start_time = data.get('start_time', 0) - duration = data.get('duration', 0) + start_measure = data.get('start_measure', 1) + start_beat = data.get('start_beat', 1) + num_measures = data.get('num_measures', 0) if not midi_file: return web.json_response({"error": "Missing required parameters"}, status=400) @@ -445,18 +461,57 @@ async def refresh_midi_data(request): if not midi_path or not os.path.exists(midi_path): return web.json_response({"error": "MIDI file not found"}, status=404) - # Get basic MIDI analysis with time filtering - analysis = MIDILoader.analyze_midi(midi_path, start_time, duration) + # Load the full MIDI data + midi_data = mido.MidiFile(midi_path) - # Filter notes by track if needed - if track_selection != "all": + # Get total measures information + total_measures, time_sig_num, time_sig_denom = calculate_midi_total_measures(midi_data) + + # Apply track selection and measure filtering + all_notes = set() + + # Convert measures to ticks if measure filtering is applied + if start_measure > 1 or start_beat > 1 or num_measures > 0: + start_tick, end_tick, _, _ = convert_measures_to_ticks(midi_data, start_measure, start_beat, num_measures) + else: + start_tick = 0 + end_tick = float('inf') + + # Process tracks to collect notes within the time range + track_notes = {} + for i, track in enumerate(midi_data.tracks): + track_notes[str(i)] = set() + current_tick = 0 + + for msg in track: + if hasattr(msg, 'time'): + current_tick += msg.time + + # Only consider note_on messages with velocity > 0 (actual notes being played) + if msg.type == 'note_on' and hasattr(msg, 'velocity') and msg.velocity > 0: + # Check if this note is within our measure range + if current_tick >= start_tick and current_tick < end_tick: + note = msg.note + track_notes[str(i)].add(note) + # If all tracks are selected or the current track matches selection, add to all_notes + if track_selection == "all" or track_selection.startswith(f"{i}:"): + all_notes.add(note) + + # If a specific track is selected but we didn't find any notes, it might be because + # our filtering logic is different from above + if track_selection != "all" and not all_notes: track_index = track_selection.split(':')[0] - if track_index.isdigit() and track_index in analysis['track_notes']: - analysis['all_notes'] = analysis['track_notes'][track_index] - else: - analysis['all_notes'] = "" + if track_index.isdigit() and track_index in track_notes: + all_notes = track_notes[track_index] - return web.json_response(analysis) + # Format and return the response + return web.json_response({ + "tracks": ["all"] + [f"{i}: {getattr(track, 'name', '') or f'Track {i}'}" for i, track in enumerate(midi_data.tracks)], + "all_notes": ",".join(map(str, sorted(all_notes))), + "track_notes": {k: ",".join(map(str, sorted(v))) for k, v in track_notes.items()}, + "total_measures": total_measures, + "time_signature": f"{time_sig_num}/{time_sig_denom}" + }) NODE_CLASS_MAPPINGS = { "MIDIToAudio": MIDIToAudio, diff --git a/web/js/midi_extensions.js b/web/js/midi_extensions.js index 8cfd6e1..561b846 100644 --- a/web/js/midi_extensions.js +++ b/web/js/midi_extensions.js @@ -109,17 +109,19 @@ app.registerExtension({ const midiFile = loaderNode.getCurrentMidiFile(); const trackSelection = loaderNode.widgets.find(w => w.name === "track_selection").value; - // Get time selection parameters from the MIDILoader - const startTimeWidget = loaderNode.widgets.find(w => w.name === "start_time"); - const durationWidget = loaderNode.widgets.find(w => w.name === "duration"); + // Get measure selection parameters from the MIDILoader + const startMeasureWidget = loaderNode.widgets.find(w => w.name === "start_measure"); + const startBeatWidget = loaderNode.widgets.find(w => w.name === "start_beat"); + const numMeasuresWidget = loaderNode.widgets.find(w => w.name === "num_measures"); - const startTime = startTimeWidget ? startTimeWidget.value : 0; - const duration = durationWidget ? durationWidget.value : 0; + const startMeasure = startMeasureWidget ? startMeasureWidget.value : 1; + const startBeat = startBeatWidget ? startBeatWidget.value : 1; + const numMeasures = numMeasuresWidget ? numMeasuresWidget.value : 0; if (!midiFile) return; try { - console.log(`Querying MIDI notes with start_time=${startTime}, duration=${duration}`); + console.log(`Querying MIDI notes with startMeasure=${startMeasure}, startBeat=${startBeat}, numMeasures=${numMeasures}`); const response = await fetch('/refresh_midi_data', { method: 'POST', @@ -127,8 +129,9 @@ app.registerExtension({ body: JSON.stringify({ midi_file: midiFile, track_selection: trackSelection, - start_time: startTime, - duration: duration + start_measure: startMeasure, + start_beat: startBeat, + num_measures: numMeasures }) }); const data = await response.json(); @@ -140,7 +143,7 @@ app.registerExtension({ // Update available notes const noteArray = data.all_notes.split(',').map(Number).filter(n => !isNaN(n)); - console.log(`Received ${noteArray.length} available notes in time range ${startTime}s - ${startTime + duration}s`); + console.log(`Received ${noteArray.length} available notes in measures ${startMeasure}-${startMeasure + numMeasures}, time signature: ${data.time_signature}`); this.availableNotes = new Set(noteArray); @@ -152,7 +155,7 @@ app.registerExtension({ ); if (originalCount !== this.properties.selectedNotes.length) { - console.log(`Filtered out ${originalCount - this.properties.selectedNotes.length} selected notes that aren't in the time range`); + console.log(`Filtered out ${originalCount - this.properties.selectedNotes.length} selected notes`); } } @@ -180,43 +183,53 @@ app.registerExtension({ } loaderNode._feature_extractor_listeners.push(this.id); - // Find widgets in loader node + // Find all relevant widgets in loader node const trackWidget = loaderNode.widgets.find(w => w.name === "track_selection"); - const startTimeWidget = loaderNode.widgets.find(w => w.name === "start_time"); - const durationWidget = loaderNode.widgets.find(w => w.name === "duration"); + const startMeasureWidget = loaderNode.widgets.find(w => w.name === "start_measure"); + const startBeatWidget = loaderNode.widgets.find(w => w.name === "start_beat"); + const numMeasuresWidget = loaderNode.widgets.find(w => w.name === "num_measures"); - // Set up callback for track selection changes + // Set up callbacks for all relevant widgets if (trackWidget) { const origTrackCallback = trackWidget.callback; trackWidget.callback = (value) => { // Call original callback if (origTrackCallback) origTrackCallback.call(loaderNode, value); - // Notify this feature extractor to update notes + // Notify this feature extractor this.queryMIDILoaderNotes(loaderNode); }; } - // Set up callback for start time changes - if (startTimeWidget) { - const origCallback = startTimeWidget.callback; - startTimeWidget.callback = (value) => { + if (startMeasureWidget) { + const origCallback = startMeasureWidget.callback; + startMeasureWidget.callback = (value) => { // Call original callback if (origCallback) origCallback.call(loaderNode, value); - // Notify this feature extractor to update notes + // Notify this feature extractor this.queryMIDILoaderNotes(loaderNode); }; } - // Set up callback for duration changes - if (durationWidget) { - const origCallback = durationWidget.callback; - durationWidget.callback = (value) => { + if (startBeatWidget) { + const origCallback = startBeatWidget.callback; + startBeatWidget.callback = (value) => { // Call original callback if (origCallback) origCallback.call(loaderNode, value); - // Notify this feature extractor to update notes + // Notify this feature extractor + this.queryMIDILoaderNotes(loaderNode); + }; + } + + if (numMeasuresWidget) { + const origCallback = numMeasuresWidget.callback; + numMeasuresWidget.callback = (value) => { + // Call original callback + if (origCallback) origCallback.call(loaderNode, value); + + // Notify this feature extractor this.queryMIDILoaderNotes(loaderNode); }; } From d060e57a8db80e631364478a3b228fc632d85f39 Mon Sep 17 00:00:00 2001 From: RyanOnTheInside <7623207+ryanontheinside@users.noreply.github.com> Date: Sat, 17 May 2025 15:35:47 -0400 Subject: [PATCH 05/10] end measure, end beat --- nodes/audio/midi_nodes.py | 77 +++++++++++++++++++++++--------- tooltips/categories/flex.py | 2 +- web/js/midi_extensions.js | 33 ++++++++++---- web/js/midi_loader_extensions.js | 68 ++++++++++++++++------------ 4 files changed, 120 insertions(+), 60 deletions(-) diff --git a/nodes/audio/midi_nodes.py b/nodes/audio/midi_nodes.py index f6be572..29a5bf6 100644 --- a/nodes/audio/midi_nodes.py +++ b/nodes/audio/midi_nodes.py @@ -175,8 +175,8 @@ def calculate_midi_total_measures(midi_data): total_measures = math.ceil(total_ticks / ticks_per_measure) return total_measures, time_sig_numerator, time_sig_denominator -def convert_measures_to_ticks(midi_data, start_measure, start_beat=1, num_measures=0): - """Convert musical measures to MIDI ticks""" +def convert_measures_to_ticks_range(midi_data, start_measure, start_beat, end_measure, end_beat): + """Convert musical measure range to MIDI tick range""" # Get time signature (defaults to 4/4) time_sig_numerator = 4 time_sig_denominator = 4 @@ -199,13 +199,15 @@ def convert_measures_to_ticks(midi_data, start_measure, start_beat=1, num_measur # Calculate start tick start_tick = (start_measure_index * beats_per_measure + start_beat_index) * ticks_per_beat - # Calculate end tick if duration is specified - if num_measures > 0: - end_tick = start_tick + (num_measures * beats_per_measure * ticks_per_beat) + # Calculate end tick + if end_measure > 0: + end_measure_index = end_measure - 1 + end_beat_index = end_beat - 1 + end_tick = (end_measure_index * beats_per_measure + end_beat_index) * ticks_per_beat else: end_tick = float('inf') - return start_tick, end_tick, time_sig_numerator, time_sig_denominator + return start_tick, end_tick @apply_tooltips class MIDILoader: @@ -219,7 +221,8 @@ class MIDILoader: "track_selection": (["all"],), "start_measure": ("INT", {"default": 1, "min": 1, "step": 1}), "start_beat": ("INT", {"default": 1, "min": 1, "step": 1}), - "num_measures": ("INT", {"default": 0, "min": 0, "step": 1, "description": "Number of measures (0 = all remaining measures)"}) + "end_measure": ("INT", {"default": 0, "min": 0, "step": 1, "description": "End measure (0 = all remaining measures)"}), + "end_beat": ("INT", {"default": 1, "min": 1, "step": 1, "description": "End beat"}) } } @@ -227,7 +230,7 @@ class MIDILoader: FUNCTION = "load_midi" CATEGORY = "RyanOnTheInside/Audio/MIDI" - def load_midi(self, midi_file, track_selection, start_measure=1, start_beat=1, num_measures=0): + def load_midi(self, midi_file, track_selection, start_measure=1, start_beat=1, end_measure=0, end_beat=1): try: midi_path = folder_paths.get_full_path("midi_files", midi_file) if not midi_path or not os.path.exists(midi_path): @@ -255,9 +258,16 @@ class MIDILoader: midi_data = selected_midi # Apply measure slicing if needed - if (start_measure > 1 or start_beat > 1 or num_measures > 0) and midi_data.tracks: + if (start_measure > 1 or start_beat > 1 or end_measure > 0) and midi_data.tracks: + # Get total measures in the file for validation + total_measures, _, _ = calculate_midi_total_measures(midi_data) + + # If end measure is 0, use the total measures + if end_measure == 0: + end_measure = total_measures + # Convert musical measures to ticks - start_tick, end_tick, _, _ = convert_measures_to_ticks(midi_data, start_measure, start_beat, num_measures) + start_tick, end_tick = convert_measures_to_ticks_range(midi_data, start_measure, start_beat, end_measure, end_beat) # Create a new MIDI file with the selected measures trimmed_midi = mido.MidiFile(ticks_per_beat=midi_data.ticks_per_beat) @@ -320,15 +330,19 @@ class MIDILoader: raise RuntimeError(f"Error loading MIDI file: {type(e).__name__}: {str(e)}") @classmethod - def analyze_midi(cls, midi_path, start_measure=1, start_beat=1, num_measures=0): + def analyze_midi(cls, midi_path, start_measure=1, start_beat=1, end_measure=0, end_beat=1): midi_data = mido.MidiFile(midi_path) # Get the total measures total_measures, time_sig_num, time_sig_denom = calculate_midi_total_measures(midi_data) + # If end measure is 0, use the total measures + if end_measure == 0: + end_measure = total_measures + # Convert measures to ticks if measure filtering is applied - if start_measure > 1 or start_beat > 1 or num_measures > 0: - start_tick, end_tick, _, _ = convert_measures_to_ticks(midi_data, start_measure, start_beat, num_measures) + if start_measure > 1 or start_beat > 1 or end_measure < total_measures: + start_tick, end_tick = convert_measures_to_ticks_range(midi_data, start_measure, start_beat, end_measure, end_beat) else: start_tick = 0 end_tick = float('inf') @@ -364,7 +378,7 @@ class MIDILoader: } @classmethod - def VALIDATE_INPUTS(cls, midi_file, track_selection, start_measure=1, start_beat=1, num_measures=0): + def VALIDATE_INPUTS(cls, midi_file, track_selection, start_measure=1, start_beat=1, end_measure=0, end_beat=1): midi_path = folder_paths.get_full_path("midi_files", midi_file) if not midi_path or not os.path.isfile(midi_path): return f"MIDI file not found: {midi_file}" @@ -379,15 +393,28 @@ class MIDILoader: if start_beat < 1: return f"Start beat must be at least 1, got: {start_beat}" - if num_measures < 0: - return f"Number of measures cannot be negative, got: {num_measures}" + if end_measure < 0: + return f"End measure cannot be negative, got: {end_measure}" + + if end_beat < 1: + return f"End beat must be at least 1, got: {end_beat}" # Check if the start measure is valid for this MIDI file try: midi_data = mido.MidiFile(midi_path) total_measures, _, _ = calculate_midi_total_measures(midi_data) + if start_measure > total_measures: return f"Start measure {start_measure} exceeds the total measures in the file ({total_measures})" + + if end_measure > 0 and end_measure > total_measures: + return f"End measure {end_measure} exceeds the total measures in the file ({total_measures})" + + if end_measure > 0 and end_measure < start_measure: + return f"End measure {end_measure} cannot be less than start measure {start_measure}" + + if end_measure == start_measure and end_beat < start_beat: + return f"When start and end measures are the same, end beat {end_beat} cannot be less than start beat {start_beat}" except Exception as e: return f"Error validating MIDI file: {str(e)}" @@ -402,7 +429,8 @@ async def get_track_notes(request): midi_file = data.get('midi_file') start_measure = data.get('start_measure', 1) start_beat = data.get('start_beat', 1) - num_measures = data.get('num_measures', 0) + end_measure = data.get('end_measure', 0) + end_beat = data.get('end_beat', 1) if not midi_file: return web.json_response({"error": "Missing required parameters"}, status=400) @@ -412,7 +440,7 @@ async def get_track_notes(request): return web.json_response({"error": "MIDI file not found"}, status=404) # Analyze MIDI file - analysis = MIDILoader.analyze_midi(midi_path, start_measure, start_beat, num_measures) + analysis = MIDILoader.analyze_midi(midi_path, start_measure, start_beat, end_measure, end_beat) return web.json_response(analysis) @routes.post('/upload_midi') @@ -452,7 +480,8 @@ async def refresh_midi_data(request): track_selection = data.get('track_selection') start_measure = data.get('start_measure', 1) start_beat = data.get('start_beat', 1) - num_measures = data.get('num_measures', 0) + end_measure = data.get('end_measure', 0) + end_beat = data.get('end_beat', 1) if not midi_file: return web.json_response({"error": "Missing required parameters"}, status=400) @@ -467,12 +496,16 @@ async def refresh_midi_data(request): # Get total measures information total_measures, time_sig_num, time_sig_denom = calculate_midi_total_measures(midi_data) + # If end measure is 0, use the total measures + if end_measure == 0: + end_measure = total_measures + # Apply track selection and measure filtering all_notes = set() - # Convert measures to ticks if measure filtering is applied - if start_measure > 1 or start_beat > 1 or num_measures > 0: - start_tick, end_tick, _, _ = convert_measures_to_ticks(midi_data, start_measure, start_beat, num_measures) + # Convert measures to ticks + if start_measure > 1 or start_beat > 1 or end_measure < total_measures: + start_tick, end_tick = convert_measures_to_ticks_range(midi_data, start_measure, start_beat, end_measure, end_beat) else: start_tick = 0 end_tick = float('inf') diff --git a/tooltips/categories/flex.py b/tooltips/categories/flex.py index 0ae09c5..e8843ac 100644 --- a/tooltips/categories/flex.py +++ b/tooltips/categories/flex.py @@ -248,7 +248,7 @@ Great for creating music videos, reactive animations, or automated effects.""") "track_selection": "Track to analyze ('all' or specific track number)", "chord_only": "When true, only considers full chords (default: false)", "notes": "Comma-separated list of MIDI note numbers (default: empty)" - }, inherits_from='FeatureExtractorBase', description="Extract features from MIDI files. Analyze note on/off events, velocity, pitch, and more to create dynamic animations and effects. Use the piano to filter the notes that are considered for extraction.") + }, inherits_from='FeatureExtractorBase', description="Extract features from MIDI files. Analyze note on/off events, velocity, pitch, and more to create dynamic animations and effects. Use the piano to filter the notes that are considered for extraction, or leave none selected to use them all. Notes that do not exist in the MIDI file will be disabled.") # AudioFeatureExtractor tooltips (inherits from: FeatureExtractorBase) TooltipManager.register_tooltips("AudioFeatureExtractor", { diff --git a/web/js/midi_extensions.js b/web/js/midi_extensions.js index 561b846..1f4e9c6 100644 --- a/web/js/midi_extensions.js +++ b/web/js/midi_extensions.js @@ -112,16 +112,18 @@ app.registerExtension({ // Get measure selection parameters from the MIDILoader const startMeasureWidget = loaderNode.widgets.find(w => w.name === "start_measure"); const startBeatWidget = loaderNode.widgets.find(w => w.name === "start_beat"); - const numMeasuresWidget = loaderNode.widgets.find(w => w.name === "num_measures"); + const endMeasureWidget = loaderNode.widgets.find(w => w.name === "end_measure"); + const endBeatWidget = loaderNode.widgets.find(w => w.name === "end_beat"); const startMeasure = startMeasureWidget ? startMeasureWidget.value : 1; const startBeat = startBeatWidget ? startBeatWidget.value : 1; - const numMeasures = numMeasuresWidget ? numMeasuresWidget.value : 0; + const endMeasure = endMeasureWidget ? endMeasureWidget.value : 0; + const endBeat = endBeatWidget ? endBeatWidget.value : 1; if (!midiFile) return; try { - console.log(`Querying MIDI notes with startMeasure=${startMeasure}, startBeat=${startBeat}, numMeasures=${numMeasures}`); + console.log(`Querying MIDI notes with startMeasure=${startMeasure}, startBeat=${startBeat}, endMeasure=${endMeasure}, endBeat=${endBeat}`); const response = await fetch('/refresh_midi_data', { method: 'POST', @@ -131,7 +133,8 @@ app.registerExtension({ track_selection: trackSelection, start_measure: startMeasure, start_beat: startBeat, - num_measures: numMeasures + end_measure: endMeasure, + end_beat: endBeat }) }); const data = await response.json(); @@ -143,7 +146,7 @@ app.registerExtension({ // Update available notes const noteArray = data.all_notes.split(',').map(Number).filter(n => !isNaN(n)); - console.log(`Received ${noteArray.length} available notes in measures ${startMeasure}-${startMeasure + numMeasures}, time signature: ${data.time_signature}`); + console.log(`Received ${noteArray.length} available notes from measures ${startMeasure} to ${endMeasure}, time signature: ${data.time_signature}`); this.availableNotes = new Set(noteArray); @@ -187,7 +190,8 @@ app.registerExtension({ const trackWidget = loaderNode.widgets.find(w => w.name === "track_selection"); const startMeasureWidget = loaderNode.widgets.find(w => w.name === "start_measure"); const startBeatWidget = loaderNode.widgets.find(w => w.name === "start_beat"); - const numMeasuresWidget = loaderNode.widgets.find(w => w.name === "num_measures"); + const endMeasureWidget = loaderNode.widgets.find(w => w.name === "end_measure"); + const endBeatWidget = loaderNode.widgets.find(w => w.name === "end_beat"); // Set up callbacks for all relevant widgets if (trackWidget) { @@ -223,9 +227,20 @@ app.registerExtension({ }; } - if (numMeasuresWidget) { - const origCallback = numMeasuresWidget.callback; - numMeasuresWidget.callback = (value) => { + if (endMeasureWidget) { + const origCallback = endMeasureWidget.callback; + endMeasureWidget.callback = (value) => { + // Call original callback + if (origCallback) origCallback.call(loaderNode, value); + + // Notify this feature extractor + this.queryMIDILoaderNotes(loaderNode); + }; + } + + if (endBeatWidget) { + const origCallback = endBeatWidget.callback; + endBeatWidget.callback = (value) => { // Call original callback if (origCallback) origCallback.call(loaderNode, value); diff --git a/web/js/midi_loader_extensions.js b/web/js/midi_loader_extensions.js index 7389e70..b001f2b 100644 --- a/web/js/midi_loader_extensions.js +++ b/web/js/midi_loader_extensions.js @@ -181,9 +181,14 @@ app.registerExtension({ if (widget) widget.value = this.loadedState.startBeat; } - if (this.loadedState.numMeasures !== undefined) { - const widget = this.widgets.find(w => w.name === "num_measures"); - if (widget) widget.value = this.loadedState.numMeasures; + if (this.loadedState.endMeasure !== undefined) { + const widget = this.widgets.find(w => w.name === "end_measure"); + if (widget) widget.value = this.loadedState.endMeasure; + } + + if (this.loadedState.endBeat !== undefined) { + const widget = this.widgets.find(w => w.name === "end_beat"); + if (widget) widget.value = this.loadedState.endBeat; } } }; @@ -203,19 +208,26 @@ app.registerExtension({ return; } - // Get start time and duration values - const startTimeWidget = this.widgets.find(w => w.name === "start_time_seconds"); - const durationWidget = this.widgets.find(w => w.name === "duration_seconds"); - const startTime = startTimeWidget ? startTimeWidget.value : 0; - const duration = durationWidget ? durationWidget.value : 0; + // Get measure selection parameters + const startMeasureWidget = this.widgets.find(w => w.name === "start_measure"); + const startBeatWidget = this.widgets.find(w => w.name === "start_beat"); + const endMeasureWidget = this.widgets.find(w => w.name === "end_measure"); + const endBeatWidget = this.widgets.find(w => w.name === "end_beat"); + + const startMeasure = startMeasureWidget ? startMeasureWidget.value : 1; + const startBeat = startBeatWidget ? startBeatWidget.value : 1; + const endMeasure = endMeasureWidget ? endMeasureWidget.value : 0; + const endBeat = endBeatWidget ? endBeatWidget.value : 1; const response = await fetch('/get_track_notes', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ midi_file: midiFile, - start_time_seconds: startTime, - duration_seconds: duration + start_measure: startMeasure, + start_beat: startBeat, + end_measure: endMeasure, + end_beat: endBeat }) }); const data = await response.json(); @@ -240,15 +252,6 @@ app.registerExtension({ } } - // Apply saved start time and duration - if (this.loadedState.startTimeSeconds !== undefined && startTimeWidget) { - startTimeWidget.value = this.loadedState.startTimeSeconds; - } - - if (this.loadedState.durationSeconds !== undefined && durationWidget) { - durationWidget.value = this.loadedState.durationSeconds; - } - this.isInitialLoad = false; } @@ -268,11 +271,16 @@ app.registerExtension({ const trackWidget = this.widgets.find(w => w.name === "track_selection"); const trackSelection = trackWidget ? trackWidget.value : "all"; - // Get start time and duration values - const startTimeWidget = this.widgets.find(w => w.name === "start_time_seconds"); - const durationWidget = this.widgets.find(w => w.name === "duration_seconds"); - const startTime = startTimeWidget ? startTimeWidget.value : 0; - const duration = durationWidget ? durationWidget.value : 0; + // Get measure selection parameters + const startMeasureWidget = this.widgets.find(w => w.name === "start_measure"); + const startBeatWidget = this.widgets.find(w => w.name === "start_beat"); + const endMeasureWidget = this.widgets.find(w => w.name === "end_measure"); + const endBeatWidget = this.widgets.find(w => w.name === "end_beat"); + + const startMeasure = startMeasureWidget ? startMeasureWidget.value : 1; + const startBeat = startBeatWidget ? startBeatWidget.value : 1; + const endMeasure = endMeasureWidget ? endMeasureWidget.value : 0; + const endBeat = endBeatWidget ? endBeatWidget.value : 1; if (!midiFile) { console.warn('No MIDI file selected'); @@ -286,8 +294,10 @@ app.registerExtension({ body: JSON.stringify({ midi_file: midiFile, track_selection: trackSelection, - start_time_seconds: startTime, - duration_seconds: duration + start_measure: startMeasure, + start_beat: startBeat, + end_measure: endMeasure, + end_beat: endBeat }) }); const data = await response.json(); @@ -314,14 +324,16 @@ app.registerExtension({ // Get current widgets' values const startMeasureWidget = this.widgets.find(w => w.name === "start_measure"); const startBeatWidget = this.widgets.find(w => w.name === "start_beat"); - const numMeasuresWidget = this.widgets.find(w => w.name === "num_measures"); + const endMeasureWidget = this.widgets.find(w => w.name === "end_measure"); + const endBeatWidget = this.widgets.find(w => w.name === "end_beat"); const state = { midiFile: this.getCurrentMidiFile(), trackSelection: this.widgets.find(w => w.name === "track_selection")?.value, startMeasure: startMeasureWidget ? startMeasureWidget.value : 1, startBeat: startBeatWidget ? startBeatWidget.value : 1, - numMeasures: numMeasuresWidget ? numMeasuresWidget.value : 0 + endMeasure: endMeasureWidget ? endMeasureWidget.value : 0, + endBeat: endBeatWidget ? endBeatWidget.value : 1 }; localStorage.setItem(`MIDILoader_${this.id}`, JSON.stringify(state)); }; From 044663c4066ff27aaa284919a7c12266a10d489d Mon Sep 17 00:00:00 2001 From: RyanOnTheInside <7623207+ryanontheinside@users.noreply.github.com> Date: Sun, 18 May 2025 12:02:50 -0400 Subject: [PATCH 06/10] simple midi tutorial --- examples/tutorials/MIDI/MIDI-Tutorial.json | 1660 ++++++++++++++++++++ 1 file changed, 1660 insertions(+) create mode 100644 examples/tutorials/MIDI/MIDI-Tutorial.json diff --git a/examples/tutorials/MIDI/MIDI-Tutorial.json b/examples/tutorials/MIDI/MIDI-Tutorial.json new file mode 100644 index 0000000..82da63d --- /dev/null +++ b/examples/tutorials/MIDI/MIDI-Tutorial.json @@ -0,0 +1,1660 @@ +{ + "id": "5f2341e1-12d2-42c3-b821-0c41d3d8202f", + "revision": 0, + "last_node_id": 99, + "last_link_id": 127, + "nodes": [ + { + "id": 45, + "type": "MIDILoader", + "pos": [ + -1339.8095703125, + 1347.4271240234375 + ], + "size": [ + 381.1168212890625, + 226 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MIDI", + "type": "MIDI", + "links": [ + 64, + 67, + 69, + 78 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "e75dbb7156d34ee0d0dcfba629be0acb7f9c2538", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "sample_midi.mid", + "1: piano", + 1, + 1, + 0, + 1, + "upload", + "refresh" + ] + }, + { + "id": 29, + "type": "PreviewAudio", + "pos": [ + -394.14019775390625, + 728.9194946289062 + ], + "size": [ + 310.4596252441406, + 126.08221435546875 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 65 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.33", + "Node name for S&R": "PreviewAudio", + "widget_ue_connectable": {} + }, + "widgets_values": [] + }, + { + "id": 71, + "type": "PreviewAudio", + "pos": [ + -370, + -170 + ], + "size": [ + 315, + 88 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 90 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.34", + "Node name for S&R": "PreviewAudio", + "widget_ue_connectable": {} + }, + "widgets_values": [] + }, + { + "id": 74, + "type": "PreviewAudio", + "pos": [ + -333.0875549316406, + -1521.66552734375 + ], + "size": [ + 315, + 88 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 92 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.34", + "Node name for S&R": "PreviewAudio", + "widget_ue_connectable": {} + }, + "widgets_values": [] + }, + { + "id": 72, + "type": "MIDILoader", + "pos": [ + -1297.677001953125, + -1001.4724731445312 + ], + "size": [ + 381.1168212890625, + 226 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MIDI", + "type": "MIDI", + "links": [ + 91, + 102, + 103 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "e75dbb7156d34ee0d0dcfba629be0acb7f9c2538", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "sample_midi.mid", + "3: drum", + 1, + 1, + 0, + 1, + "upload", + "refresh" + ] + }, + { + "id": 79, + "type": "MIDIFeatureExtractor", + "pos": [ + -767.5911254882812, + -830.1699829101562 + ], + "size": [ + 1020, + 322 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "midi", + "type": "MIDI", + "link": 102 + } + ], + "outputs": [ + { + "name": "midi", + "type": "MIDI", + "links": null + }, + { + "name": "feature", + "type": "FEATURE", + "links": [ + 107 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": [] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "d060e57a8db80e631364478a3b228fc632d85f39", + "selectedNotes": [ + 36 + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Note On/Off", + 30, + 0, + 512, + 512, + false, + "36" + ] + }, + { + "id": 75, + "type": "MIDIFeatureExtractor", + "pos": [ + -820, + -10 + ], + "size": [ + 1020, + 322 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "midi", + "type": "MIDI", + "link": 93 + } + ], + "outputs": [ + { + "name": "midi", + "type": "MIDI", + "links": null + }, + { + "name": "feature", + "type": "FEATURE", + "links": [ + 99 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": [] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "d060e57a8db80e631364478a3b228fc632d85f39", + "selectedNotes": [], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Velocity", + 30, + 0, + 512, + 512, + false, + "" + ] + }, + { + "id": 88, + "type": "Note", + "pos": [ + -1312.242919921875, + -1373.744873046875 + ], + "size": [ + 373.50885009765625, + 179.8002166748047 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "2. Drums", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "For the drums, we have an example of extracting individual notes, the kick and the snare." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 69, + "type": "MIDILoader", + "pos": [ + -1312.6334228515625, + 97.49837493896484 + ], + "size": [ + 381.1168212890625, + 226 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MIDI", + "type": "MIDI", + "links": [ + 89, + 93 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "e75dbb7156d34ee0d0dcfba629be0acb7f9c2538", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "sample_midi.mid", + "2: bass", + 1, + 1, + 0, + 1, + "upload", + "refresh" + ] + }, + { + "id": 89, + "type": "Note", + "pos": [ + -1315.5673828125, + -162.17872619628906 + ], + "size": [ + 372.4935607910156, + 182.84597778320312 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "2. Bass", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "When no notes are selected, all notes are used" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 91, + "type": "Note", + "pos": [ + -1322.5369873046875, + 903.3281860351562 + ], + "size": [ + 372.4935607910156, + 182.84597778320312 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "3. Piano", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "When 'chord only' is enabled, feature will only be extracted when all selected notes are on simulaneously. This is useful for different reactivity for different chords (think special behavior for tonic chord or something).\n\nAdditionally, this is critical for differentiating a melody from the accompaniment." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 87, + "type": "Note", + "pos": [ + -2116.0517578125, + -1389.1805419921875 + ], + "size": [ + 569.0508422851562, + 480.17010498046875 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "1. Overview", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "This workflow demonstrates how to extract features from a MIDI file. This uses a single midi file with multiple tracks with standard instruments.\n\nThe first and most important step is to get an extremely beautiful and compelling composition, like the one used in this example. A song both happy and sad\n\n\n\nTHE PIANO NODE (midi extractor):\nThe piano keys will be enabled based on the available notes in the selected section of midi. The available notes will be updated based on the connnectd MIDI Loader node. \n\nTHE AUDIO:\nThe midi-to-audio node is is a rudimentary approximation of the midi notes for quick reference only. \n\nNOTE:\nFor brevity, all of the feature extraction is done against simply \"velocity\" and \"note on-off\". The sample midi file here does not include it, but you can use many message type for reactivity.... \n\nFor example, think of a setup where you dont map specific notes, but some expression of all notes on a track. This could be reused for any song. \n\nFeel free to DM me with some midi tracks and I could make an example workflow with them. " + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 64, + "type": "MIDIFeatureExtractor", + "pos": [ + -871.2890625, + 2088.83984375 + ], + "size": [ + 1020, + 322 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "midi", + "type": "MIDI", + "link": 78 + } + ], + "outputs": [ + { + "name": "midi", + "type": "MIDI", + "links": null + }, + { + "name": "feature", + "type": "FEATURE", + "links": [ + 88 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "d060e57a8db80e631364478a3b228fc632d85f39", + "selectedNotes": [ + 41, + 53, + 60, + 68 + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Velocity", + 30, + 0, + 512, + 512, + true, + "41, 53, 60, 68" + ] + }, + { + "id": 43, + "type": "MIDIFeatureExtractor", + "pos": [ + -852.0150756835938, + 1054.3472900390625 + ], + "size": [ + 1020, + 322 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "midi", + "type": "MIDI", + "link": 67 + } + ], + "outputs": [ + { + "name": "midi", + "type": "MIDI", + "links": null + }, + { + "name": "feature", + "type": "FEATURE", + "links": [ + 50 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": [] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "492c20ba4ae1484c9e3bdc11d036fbb4820c52c1", + "selectedNotes": [ + 37, + 49, + 56, + 61, + 65 + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Velocity", + 30, + 0, + 512, + 512, + true, + "37, 49, 56, 61, 65" + ] + }, + { + "id": 56, + "type": "MIDIFeatureExtractor", + "pos": [ + -852.0150756835938, + 1523.5477294921875 + ], + "size": [ + 1020, + 322 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "midi", + "type": "MIDI", + "link": 69 + } + ], + "outputs": [ + { + "name": "midi", + "type": "MIDI", + "links": null + }, + { + "name": "feature", + "type": "FEATURE", + "links": [ + 75 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": [] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "d060e57a8db80e631364478a3b228fc632d85f39", + "selectedNotes": [ + 51, + 58, + 63, + 67 + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Velocity", + 30, + 0, + 512, + 512, + true, + "51, 58, 63, 67" + ] + }, + { + "id": 84, + "type": "EffectVisualizer", + "pos": [ + 920.0111694335938, + -1252.7969970703125 + ], + "size": [ + 315, + 174 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": 108 + }, + { + "name": "feature", + "type": "FEATURE", + "link": 110 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 116 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "492c20ba4ae1484c9e3bdc11d036fbb4820c52c1", + "Node name for S&R": "EffectVisualizer", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "(255,255,255)", + 1.4000000000000001, + "snare", + 10, + 10 + ] + }, + { + "id": 73, + "type": "MIDIToAudio", + "pos": [ + -733.9694213867188, + -1521.224609375 + ], + "size": [ + 315, + 106 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "midi", + "type": "MIDI", + "link": 91 + } + ], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 92, + 123 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "d060e57a8db80e631364478a3b228fc632d85f39", + "Node name for S&R": "MIDIToAudio", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Drums", + 44100, + 0.5 + ] + }, + { + "id": 80, + "type": "MIDIFeatureExtractor", + "pos": [ + -770.4898071289062, + -1256.8670654296875 + ], + "size": [ + 1020, + 322 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "midi", + "type": "MIDI", + "link": 103 + } + ], + "outputs": [ + { + "name": "midi", + "type": "MIDI", + "links": [] + }, + { + "name": "feature", + "type": "FEATURE", + "links": [ + 110 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": [ + 111 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "d060e57a8db80e631364478a3b228fc632d85f39", + "selectedNotes": [ + 38 + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Note On/Off", + 30, + 0, + 512, + 512, + false, + "38" + ] + }, + { + "id": 70, + "type": "MIDIToAudio", + "pos": [ + -790, + -170 + ], + "size": [ + 315, + 106 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "midi", + "type": "MIDI", + "link": 89 + } + ], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 90, + 124 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "d060e57a8db80e631364478a3b228fc632d85f39", + "Node name for S&R": "MIDIToAudio", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Bass", + 44100, + 0.5 + ] + }, + { + "id": 97, + "type": "Audio_Combine", + "pos": [ + 1589.7186279296875, + -94.85237884521484 + ], + "size": [ + 352.79998779296875, + 102 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "audio1", + "type": "AUDIO", + "link": 123 + }, + { + "name": "audio2", + "type": "AUDIO", + "link": 124 + } + ], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 125 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "d060e57a8db80e631364478a3b228fc632d85f39", + "widget_ue_connectable": {}, + "Node name for S&R": "Audio_Combine" + }, + "widgets_values": [ + 0.5, + 0.5 + ] + }, + { + "id": 50, + "type": "MIDIToAudio", + "pos": [ + -821.4395751953125, + 731.225341796875 + ], + "size": [ + 315, + 106 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "midi", + "type": "MIDI", + "link": 64 + } + ], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 65, + 126 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "d060e57a8db80e631364478a3b228fc632d85f39", + "Node name for S&R": "MIDIToAudio", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Piano", + 44100, + 0.5 + ] + }, + { + "id": 99, + "type": "Audio_Combine", + "pos": [ + 1560.019287109375, + 773.7644653320312 + ], + "size": [ + 352.79998779296875, + 102 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "audio1", + "type": "AUDIO", + "link": 125 + }, + { + "name": "audio2", + "type": "AUDIO", + "link": 126 + } + ], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 127 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "d060e57a8db80e631364478a3b228fc632d85f39", + "widget_ue_connectable": {}, + "Node name for S&R": "Audio_Combine" + }, + "widgets_values": [ + 0.5, + 0.5 + ] + }, + { + "id": 68, + "type": "VHS_VideoCombine", + "pos": [ + 2741.55517578125, + -937.7992553710938 + ], + "size": [ + 1749.921875, + 1321.0810546875 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 86 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 127 + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "1.6.1", + "Node name for S&R": "VHS_VideoCombine", + "widget_ue_connectable": {} + }, + "widgets_values": { + "frame_rate": 30, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00022-audio.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4", + "frame_rate": 30, + "workflow": "AnimateDiff_00022.png", + "fullpath": "C:\\_dev\\comfy\\ComfyUI\\temp\\AnimateDiff_00022-audio.mp4" + } + } + } + }, + { + "id": 85, + "type": "EmptyImage", + "pos": [ + 479.27947998046875, + -1257.9613037109375 + ], + "size": [ + 315, + 130 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": 111 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 108 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.33", + "Node name for S&R": "EmptyImage", + "widget_ue_connectable": {} + }, + "widgets_values": [ + 960, + 540, + 1, + 0 + ] + }, + { + "id": 81, + "type": "EffectVisualizer", + "pos": [ + 914.053955078125, + -824.3253173828125 + ], + "size": [ + 315, + 174 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": 116 + }, + { + "name": "feature", + "type": "FEATURE", + "link": 107 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 117 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "492c20ba4ae1484c9e3bdc11d036fbb4820c52c1", + "Node name for S&R": "EffectVisualizer", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "(255,255,255)", + 1.4000000000000001, + "kick", + 10, + 60 + ] + }, + { + "id": 76, + "type": "EffectVisualizer", + "pos": [ + 910, + -10 + ], + "size": [ + 315, + 174 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": 117 + }, + { + "name": "feature", + "type": "FEATURE", + "link": 99 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 118 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "492c20ba4ae1484c9e3bdc11d036fbb4820c52c1", + "Node name for S&R": "EffectVisualizer", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "(255,255,255)", + 1.4000000000000001, + "bass", + 10, + 110 + ] + }, + { + "id": 38, + "type": "EffectVisualizer", + "pos": [ + 906.0049438476562, + 1105.6773681640625 + ], + "size": [ + 315, + 174 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": 118 + }, + { + "name": "feature", + "type": "FEATURE", + "link": 50 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 119 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "492c20ba4ae1484c9e3bdc11d036fbb4820c52c1", + "Node name for S&R": "EffectVisualizer", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "(255,255,255)", + 1.4000000000000001, + "C# ", + 10, + 160 + ] + }, + { + "id": 58, + "type": "EffectVisualizer", + "pos": [ + 933.9990844726562, + 1612.6243896484375 + ], + "size": [ + 315, + 174 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": 119 + }, + { + "name": "feature", + "type": "FEATURE", + "link": 75 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 120 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "492c20ba4ae1484c9e3bdc11d036fbb4820c52c1", + "Node name for S&R": "EffectVisualizer", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "(255,255,255)", + 1.4000000000000001, + "D# ", + 10, + 210 + ] + }, + { + "id": 67, + "type": "EffectVisualizer", + "pos": [ + 912.8372802734375, + 2150.399658203125 + ], + "size": [ + 315, + 174 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": 120 + }, + { + "name": "feature", + "type": "FEATURE", + "link": 88 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 86 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "492c20ba4ae1484c9e3bdc11d036fbb4820c52c1", + "Node name for S&R": "EffectVisualizer", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "(255,255,255)", + 1.4000000000000001, + "F ", + 10, + 270 + ] + } + ], + "links": [ + [ + null, + null, + null, + null, + null, + null + ], + [ + 50, + 43, + 1, + 38, + 1, + "FEATURE" + ], + [ + 64, + 45, + 0, + 50, + 0, + "MIDI" + ], + [ + 65, + 50, + 0, + 29, + 0, + "AUDIO" + ], + [ + 67, + 45, + 0, + 43, + 0, + "MIDI" + ], + [ + 69, + 45, + 0, + 56, + 0, + "MIDI" + ], + [ + 75, + 56, + 1, + 58, + 1, + "FEATURE" + ], + [ + 78, + 45, + 0, + 64, + 0, + "MIDI" + ], + [ + 86, + 67, + 0, + 68, + 0, + "IMAGE" + ], + [ + 88, + 64, + 1, + 67, + 1, + "FEATURE" + ], + [ + 89, + 69, + 0, + 70, + 0, + "MIDI" + ], + [ + 90, + 70, + 0, + 71, + 0, + "AUDIO" + ], + [ + 91, + 72, + 0, + 73, + 0, + "MIDI" + ], + [ + 92, + 73, + 0, + 74, + 0, + "AUDIO" + ], + [ + 93, + 69, + 0, + 75, + 0, + "MIDI" + ], + [ + 99, + 75, + 1, + 76, + 1, + "FEATURE" + ], + [ + 102, + 72, + 0, + 79, + 0, + "MIDI" + ], + [ + 103, + 72, + 0, + 80, + 0, + "MIDI" + ], + [ + 107, + 79, + 1, + 81, + 1, + "FEATURE" + ], + [ + 108, + 85, + 0, + 84, + 0, + "IMAGE" + ], + [ + 110, + 80, + 1, + 84, + 1, + "FEATURE" + ], + [ + 111, + 80, + 2, + 85, + 0, + "INT" + ], + [ + 116, + 84, + 0, + 81, + 0, + "IMAGE" + ], + [ + 117, + 81, + 0, + 76, + 0, + "IMAGE" + ], + [ + 118, + 76, + 0, + 38, + 0, + "IMAGE" + ], + [ + 119, + 38, + 0, + 58, + 0, + "IMAGE" + ], + [ + 120, + 58, + 0, + 67, + 0, + "IMAGE" + ], + [ + 123, + 73, + 0, + 97, + 0, + "AUDIO" + ], + [ + 124, + 70, + 0, + 97, + 1, + "AUDIO" + ], + [ + 125, + 97, + 0, + 99, + 0, + "AUDIO" + ], + [ + 126, + 50, + 0, + 99, + 1, + "AUDIO" + ], + [ + 127, + 99, + 0, + 68, + 1, + "AUDIO" + ] + ], + "groups": [ + { + "id": 1, + "title": "Bass", + "bounding": [ + -1362.0684814453125, + -374.0391845703125, + 3360.12255859375, + 775.947998046875 + ], + "color": "#3f789e", + "font_size": 100, + "flags": {} + }, + { + "id": 2, + "title": "Piano", + "bounding": [ + -1361.111083984375, + 532.2680053710938, + 3372.01708984375, + 2264.61669921875 + ], + "color": "#3f789e", + "font_size": 100, + "flags": {} + }, + { + "id": 3, + "title": "Drums", + "bounding": [ + -1359.57666015625, + -1695.46826171875, + 3364.666259765625, + 1269.9716796875 + ], + "color": "#3f789e", + "font_size": 100, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.1210000000000066, + "offset": [ + 5189.493242450809, + 2982.4761975652627 + ] + }, + "ue_links": [], + "links_added_by_ue": [], + "frontendVersion": "1.17.11", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true, + "reroutes": [ + { + "id": 1, + "pos": [ + 1747.84130859375, + -1475.663818359375 + ], + "linkIds": [ + 123 + ] + } + ], + "linkExtensions": [ + { + "id": 123, + "parentId": 1 + } + ] + }, + "version": 0.4 +} \ No newline at end of file From 31a5144e7a4ee40f1c046806d22d256f90f9d8bb Mon Sep 17 00:00:00 2001 From: RyanOnTheInside <7623207+ryanontheinside@users.noreply.github.com> Date: Sun, 18 May 2025 12:03:00 -0400 Subject: [PATCH 07/10] improve visualizer --- nodes/flex/visualizers.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/nodes/flex/visualizers.py b/nodes/flex/visualizers.py index 36bb383..a25a7b2 100644 --- a/nodes/flex/visualizers.py +++ b/nodes/flex/visualizers.py @@ -20,6 +20,9 @@ class EffectVisualizer(RyanOnTheInside): "feature": ("FEATURE",), "text_color": ("STRING", {"default": "(255,255,255)"}), "font_scale": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 2.0, "step": 0.1}), + "label_prefix": ("STRING", {"default": ""}), + "label_x_offset": ("INT", {"default": 10, "min": 0, "max": 1000, "step": 1}), + "label_y_offset": ("INT", {"default": 10, "min": 0, "max": 1000, "step": 1}), } } @@ -27,7 +30,7 @@ class EffectVisualizer(RyanOnTheInside): FUNCTION = "visualize" CATEGORY = "RyanOnTheInside/FlexFeatures/Utilities/Previews" - def visualize(self, video_frames, feature, text_color, font_scale): + def visualize(self, video_frames, feature, text_color, font_scale, label_prefix, label_x_offset, label_y_offset): text_color = self.parse_color(text_color) output_frames = [] padding = 10 # Padding from the edges @@ -39,10 +42,11 @@ class EffectVisualizer(RyanOnTheInside): # Display feature name and value on the frame feature_value = feature.get_value_at_frame(frame_index) - text = f"{feature.name}: {feature_value:.2f}" + prefix = f"{label_prefix} " if label_prefix else "" + text = f"{prefix}{feature.name}: {feature_value:.2f}" text_size = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, font_scale, 2)[0] - text_x = padding - text_y = padding + text_size[1] + text_x = label_x_offset + text_y = label_y_offset + text_size[1] cv2.putText(frame, text, (text_x, text_y), cv2.FONT_HERSHEY_SIMPLEX, font_scale, text_color, 2) @@ -74,7 +78,8 @@ class ProximityVisualizer(EffectVisualizer): FUNCTION = "visualize_proximity" def visualize_proximity(self, video_frames, anchor_locations, query_locations, feature, - anchor_color, query_color, line_color, text_color, font_scale): + anchor_color, query_color, line_color, text_color, font_scale, + label_prefix, label_x_offset, label_y_offset): anchor_color = self.parse_color(anchor_color) query_color = self.parse_color(query_color) line_color = self.parse_color(line_color) @@ -152,13 +157,16 @@ class PitchVisualizer(EffectVisualizer): "feature": ("FEATURE",), "text_color": ("STRING", {"default": "(255,255,255)"}), "font_scale": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 2.0, "step": 0.1}), + "label_prefix": ("STRING", {"default": ""}), + "label_x_offset": ("INT", {"default": 10, "min": 0, "max": 1000, "step": 1}), + "label_y_offset": ("INT", {"default": 10, "min": 0, "max": 1000, "step": 1}), } } RETURN_TYPES = ("IMAGE",) FUNCTION = "visualize_pitch" - def visualize_pitch(self, video_frames, feature, text_color, font_scale): + def visualize_pitch(self, video_frames, feature, text_color, font_scale, label_prefix, label_x_offset, label_y_offset): text_color = self.parse_color(text_color) output_frames = [] padding = 10 # Padding from the edges @@ -191,8 +199,8 @@ class PitchVisualizer(EffectVisualizer): for i, text in enumerate(texts): text_size = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, font_scale, 2)[0] - text_x = padding - text_y = padding + (i + 1) * (text_size[1] + 10) # Add some vertical spacing + text_x = label_x_offset + text_y = label_y_offset + (i + 1) * (text_size[1] + 10) # Add some vertical spacing cv2.putText(frame, text, (text_x, text_y), cv2.FONT_HERSHEY_SIMPLEX, font_scale, text_color, 2) From 2b82d27243060dc314232e6762aa8049af4a9433 Mon Sep 17 00:00:00 2001 From: RyanOnTheInside <7623207+ryanontheinside@users.noreply.github.com> Date: Sun, 18 May 2025 12:03:15 -0400 Subject: [PATCH 08/10] improved fake instruments --- nodes/audio/midi_nodes.py | 245 +++++++++++++++++++++++++++++++------- 1 file changed, 201 insertions(+), 44 deletions(-) diff --git a/nodes/audio/midi_nodes.py b/nodes/audio/midi_nodes.py index 29a5bf6..5b2b373 100644 --- a/nodes/audio/midi_nodes.py +++ b/nodes/audio/midi_nodes.py @@ -19,18 +19,13 @@ class MIDIToAudio(AudioNodeBase): return { "required": { "midi": ("MIDI",), + "instrument_type": (["Piano", "Synth", "Bass", "Drums"],), "sample_rate": ("INT", { "default": 44100, "min": 8000, "max": 192000, "step": 1 }), - "instrument": ("INT", { - "default": 0, # Piano (not used for simple synthesis) - "min": 0, - "max": 127, - "step": 1 - }), "volume": ("FLOAT", { "default": 0.5, "min": 0.0, @@ -43,8 +38,112 @@ class MIDIToAudio(AudioNodeBase): RETURN_TYPES = ("AUDIO",) FUNCTION = "convert_midi_to_audio" CATEGORY = "RyanOnTheInside/Audio/MIDI" + DESCRIPTION = "This will produce rudimentary approximations of MIDI notes for quick reference and nothing more. May or may not get the notes right, especially for drums." - def convert_midi_to_audio(self, midi, sample_rate, instrument, volume): + def generate_instrument_waveform(self, frequency, t, instrument_type, envelope, note_number=60): + """Generate waveform based on the selected instrument type""" + if instrument_type == "Piano": + # Piano-like sound with some harmonics + waveform = 0.5 * np.sin(2 * np.pi * frequency * t) # Fundamental + waveform += 0.2 * np.sin(2 * np.pi * frequency * 2 * t) # 1st harmonic (octave) + waveform += 0.1 * np.sin(2 * np.pi * frequency * 3 * t) # 2nd harmonic + waveform += 0.05 * np.sin(2 * np.pi * frequency * 4 * t) # 3rd harmonic + # Add faster decay for piano-like sound + decay = np.exp(-t * 3) + return waveform * envelope * decay + + elif instrument_type == "Bass": + # Bass with more low frequencies + waveform = 0.6 * np.sin(2 * np.pi * frequency * t) # Fundamental + waveform += 0.3 * np.sin(2 * np.pi * frequency * 2 * t) # 1st harmonic + # Add some distortion for bass character + waveform = np.tanh(waveform * 1.5) * 0.7 + # Slower decay for bass + decay = np.exp(-t * 2) + return waveform * envelope * decay + + elif instrument_type == "Drums": + # Use standard MIDI drum mappings (channel 10) + # Define custom synthesis for each drum sound based on note number + + # Bass/Kick Drums + if note_number in [35, 36]: # Bass Drum 2, Bass Drum 1 + # Low frequency sine with very fast decay + waveform = np.sin(2 * np.pi * 60 * t) # Fixed low frequency for kick + waveform += 0.2 * np.sin(2 * np.pi * 90 * t) # Add some mid tone + decay = np.exp(-t * 20) # Very quick decay + return waveform * envelope * decay + + # Snare Drums + elif note_number in [38, 40]: # Acoustic Snare, Electric Snare + # Mix of sine wave and noise + waveform = 0.3 * np.sin(2 * np.pi * 150 * t) # Mid frequency tone + noise = np.random.rand(len(t)) * 2 - 1 # White noise + decay = np.exp(-t * 15) # Fast decay + return (waveform + 0.7 * noise) * envelope * decay + + # Hi-Hats + elif note_number in [42, 44, 46]: # Closed, Pedal, Open Hi-Hats + # Mostly noise with different decay times + noise = np.random.rand(len(t)) * 2 - 1 + # Different decay times based on hi-hat type + if note_number == 42: # Closed Hi-Hat + decay = np.exp(-t * 30) # Very short + elif note_number == 44: # Pedal Hi-Hat + decay = np.exp(-t * 25) # Short + else: # Open Hi-Hat + decay = np.exp(-t * 10) # Longer + + # Add some high frequency sine for metallic character + waveform = noise + 0.1 * np.sin(2 * np.pi * 800 * t) + + return waveform * envelope * decay + + # Toms + elif note_number in [41, 43, 45, 47, 48, 50]: # Various Toms + # Pitched sine waves with medium decay + if note_number in [41, 43]: # Floor Toms + tom_freq = 80 # Low frequency + elif note_number in [45, 47]: # Mid Toms + tom_freq = 120 # Mid frequency + else: # High Toms + tom_freq = 180 # Higher frequency + + waveform = np.sin(2 * np.pi * tom_freq * t) + decay = np.exp(-t * 12) + return waveform * envelope * decay + + # Cymbals + elif note_number in [49, 51, 52, 53, 55, 57]: # Crash and Ride Cymbals + # Complex noise with slow decay + noise = np.random.rand(len(t)) * 2 - 1 + # Add some high frequencies for metallic sound + for i in range(3, 10): + noise += 0.1 / i * np.sin(2 * np.pi * 500 * i * t) + + # Longer decay for cymbals + decay = np.exp(-t * 4) + return noise * envelope * decay + + # Other percussion (default case) + else: + # Generic percussion sound based on frequency + noise = np.random.rand(len(t)) * 2 - 1 + waveform = 0.5 * np.sin(2 * np.pi * frequency * t) + 0.5 * noise + decay = np.exp(-t * 10) + return waveform * envelope * decay + + else: # "Synth" (default) + # Simple synth with multiple waveforms + waveform = 0.4 * np.sin(2 * np.pi * frequency * t) # Sine + # Add square wave component + square = 0.3 * np.sign(np.sin(2 * np.pi * frequency * t)) + # Add sawtooth component + sawtooth = 0.3 * ((2 * (frequency * t - np.floor(0.5 + frequency * t))) % 2) + + return (waveform + square + sawtooth) * envelope + + def convert_midi_to_audio(self, midi, instrument_type, sample_rate, volume): try: # Get tempo from MIDI file (default to 120 BPM if not specified) tempo = 500000 # Default tempo (microseconds per quarter note) @@ -56,19 +155,35 @@ class MIDIToAudio(AudioNodeBase): # Calculate total duration in seconds total_time = 0 + last_event_time = 0 + has_notes = False + for track in midi.tracks: track_time = 0 + notes_in_track = False + for msg in track: if hasattr(msg, 'time'): track_time += msg.time + if msg.type == 'note_on' and hasattr(msg, 'velocity') and msg.velocity > 0: + notes_in_track = True + has_notes = True + last_event_time = track_time + elif (msg.type == 'note_off' or (msg.type == 'note_on' and hasattr(msg, 'velocity') and msg.velocity == 0)): + last_event_time = track_time + total_time = max(total_time, track_time) + # If we found notes, use the last note event time plus some padding + if has_notes: + total_time = last_event_time + 240 # Add a small buffer (240 ticks) for note releases + # Convert ticks to seconds using tempo seconds_per_tick = tempo / (midi.ticks_per_beat * 1000000.0) total_time = total_time * seconds_per_tick # Ensure we have some duration - total_time = max(total_time, 1.0) # At least 1 second + total_time = max(total_time, 0.5) # At least 0.5 second print(f"MIDI duration: {total_time:.2f} seconds") @@ -95,7 +210,7 @@ class MIDIToAudio(AudioNodeBase): start_time, velocity = active_notes[msg.note] duration = current_time - start_time - # Generate note audio using simple sine wave + # Generate note audio using instrument-specific synthesis frequency = 440.0 * (2.0 ** ((msg.note - 69) / 12.0)) # Convert MIDI note to frequency note_samples = int(duration * sample_rate) if note_samples > 0: @@ -108,7 +223,11 @@ class MIDIToAudio(AudioNodeBase): envelope[:attack] = np.linspace(0, 1, attack) if len(envelope) > release: envelope[-release:] = np.linspace(1, 0, release) - note_audio = np.sin(2 * np.pi * frequency * t) * envelope * (velocity / 127.0) * volume + + # Generate waveform based on instrument type + note_audio = self.generate_instrument_waveform( + frequency, t, instrument_type, envelope, msg.note + ) * (velocity / 127.0) * volume # Add to buffer with proper timing start_idx = int(start_time * sample_rate) @@ -221,8 +340,8 @@ class MIDILoader: "track_selection": (["all"],), "start_measure": ("INT", {"default": 1, "min": 1, "step": 1}), "start_beat": ("INT", {"default": 1, "min": 1, "step": 1}), - "end_measure": ("INT", {"default": 0, "min": 0, "step": 1, "description": "End measure (0 = all remaining measures)"}), - "end_beat": ("INT", {"default": 1, "min": 1, "step": 1, "description": "End beat"}) + "end_measure": ("INT", {"default": 0, "min": 0, "step": 1, "tooltip": "End measure (0 = all remaining measures)"}), + "end_beat": ("INT", {"default": 1, "min": 1, "step": 1, "tootip": "End beat"}) } } @@ -276,51 +395,73 @@ class MIDILoader: new_track = mido.MidiTrack() trimmed_midi.tracks.append(new_track) - # Copy all metadata and non-time-based messages + # Copy important metadata messages for msg in track: if not hasattr(msg, 'time'): - new_track.append(msg) + if msg.type in ['track_name', 'time_signature', 'key_signature', 'set_tempo']: + new_track.append(msg.copy()) - # Add all messages that fall within our measure range + # First pass: collect all note_on events within range and their corresponding note_off events + note_events = [] + active_notes = {} # {note_num: tick_time_started} current_tick = 0 - first_msg_added = False - prev_tick = 0 + last_event_tick = 0 # To track the last event we need to include for msg in track: if not hasattr(msg, 'time'): - continue # Skip metadata (already copied) + continue current_tick += msg.time - # Include messages that are within our measure range - if current_tick >= start_tick and current_tick <= end_tick: - # Create an adjusted message with corrected timing + if msg.type == 'note_on' and msg.velocity > 0: + # Note being turned on + if current_tick >= start_tick and current_tick <= end_tick: + active_notes[msg.note] = current_tick + note_events.append((current_tick, msg.copy())) + last_event_tick = max(last_event_tick, current_tick) + + elif (msg.type == 'note_off' or (msg.type == 'note_on' and msg.velocity == 0)): + # Note being turned off + if msg.note in active_notes: + # This is a note that started within our range + note_events.append((current_tick, msg.copy())) + del active_notes[msg.note] + last_event_tick = max(last_event_tick, current_tick) + + # Also include other MIDI events within the time range + elif current_tick >= start_tick and current_tick <= end_tick: + if msg.type in ['set_tempo', 'control_change', 'program_change', 'pitchwheel']: + note_events.append((current_tick, msg.copy())) + last_event_tick = max(last_event_tick, current_tick) + + # Sort events by tick time + note_events.sort(key=lambda x: x[0]) + + # Second pass: add events to the new track with adjusted timing + if note_events: + prev_tick = start_tick + total_ticks = last_event_tick - start_tick + + for tick, msg in note_events: + # Adjust timing relative to previous event adjusted_msg = msg.copy() - - # If this is the first message in the trimmed section, - # adjust its time relative to the start point - if not first_msg_added: - adjusted_msg.time = 0 # First event starts immediately - first_msg_added = True - prev_tick = current_tick - else: - # Preserve the relative timing between events - adjusted_msg.time = current_tick - prev_tick - prev_tick = current_tick + adjusted_msg.time = tick - prev_tick + prev_tick = tick new_track.append(adjusted_msg) + + # Add end of track marker with correct timing + end_track = mido.MetaMessage('end_of_track') + + # If there are any active notes at the end boundary, make sure we have enough time + if active_notes: + remaining_ticks = end_tick - last_event_tick + if remaining_ticks > 0: + end_track.time = remaining_ticks + else: + end_track.time = 0 - # For note_off events or note_on with velocity 0 (note release), - # we need to include them if the corresponding note_on was included - elif current_tick > end_tick and first_msg_added: - if (msg.type == 'note_off' or - (msg.type == 'note_on' and hasattr(msg, 'velocity') and msg.velocity == 0)): - - # Add with adjusted timing - adjusted_msg = msg.copy() - adjusted_msg.time = current_tick - prev_tick - prev_tick = current_tick - new_track.append(adjusted_msg) + new_track.append(end_track) midi_data = trimmed_midi @@ -354,6 +495,9 @@ class MIDILoader: track_notes[str(i)] = set() current_tick = 0 + # Keep track of notes that start within range + active_notes = set() + for msg in track: if hasattr(msg, 'time'): current_tick += msg.time @@ -363,6 +507,10 @@ class MIDILoader: if msg.type == 'note_on' and msg.velocity > 0: track_notes[str(i)].add(msg.note) all_notes.add(msg.note) + active_notes.add(msg.note) + elif msg.type == 'note_off' or (msg.type == 'note_on' and msg.velocity == 0): + if msg.note in active_notes: + active_notes.remove(msg.note) if len(track_notes[str(i)]) == 0: tracks.append(f"{i}: (Empty)") @@ -516,6 +664,9 @@ async def refresh_midi_data(request): track_notes[str(i)] = set() current_tick = 0 + # Keep track of notes that start within range + active_notes = set() + for msg in track: if hasattr(msg, 'time'): current_tick += msg.time @@ -526,12 +677,18 @@ async def refresh_midi_data(request): if current_tick >= start_tick and current_tick < end_tick: note = msg.note track_notes[str(i)].add(note) + active_notes.add(note) # If all tracks are selected or the current track matches selection, add to all_notes if track_selection == "all" or track_selection.startswith(f"{i}:"): all_notes.add(note) + + # Remove notes from active set when they're turned off, but don't remove from track_notes + elif msg.type == 'note_off' or (msg.type == 'note_on' and hasattr(msg, 'velocity') and msg.velocity == 0): + if msg.note in active_notes: + active_notes.remove(msg.note) # If a specific track is selected but we didn't find any notes, it might be because - # our filtering logic is different from above + # track selection was changed after note filtering if track_selection != "all" and not all_notes: track_index = track_selection.split(':')[0] if track_index.isdigit() and track_index in track_notes: From 6143b478e707c55a12014c5d67c46ec947bb1451 Mon Sep 17 00:00:00 2001 From: RyanOnTheInside <7623207+ryanontheinside@users.noreply.github.com> Date: Sun, 18 May 2025 12:03:27 -0400 Subject: [PATCH 09/10] feature interpolate multi spike --- __init__.py | 9 +-- nodes/flex/feature_modulation.py | 112 +++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 4 deletions(-) diff --git a/__init__.py b/__init__.py index 186b6bd..f9a4148 100644 --- a/__init__.py +++ b/__init__.py @@ -298,6 +298,7 @@ from .nodes.flex.feature_modulation import ( FeatureRenormalize, FeatureInterpolator, FeaturePeakDetector, + FeatureInterpolateMulti, ) from .nodes.audio.flex_audio import ( @@ -377,6 +378,7 @@ NODE_CLASS_MAPPINGS = { "FlexlatentAudioBlend": FlexlatentAudioBlend, "FeatureInterpolator": FeatureInterpolator, "FeaturePeakDetector": FeaturePeakDetector, + "FeatureInterpolateMulti": FeatureInterpolateMulti, "Doom_": Doom_, "WhisperToPromptTravel": WhisperToPromptTravel, @@ -591,7 +593,7 @@ NODE_CLASS_MAPPINGS = { "ImageIntervalSelectPercentage":ImageIntervalSelectPercentage, "ImageIndexSelect": ImageIndexSelect, - "MIDIPlayback": MIDIToAudio, + "MIDIToAudio": MIDIToAudio, "MIDILoader": MIDILoader, } @@ -602,11 +604,11 @@ EXTENSION_WEB_DIRS = ["./web/extensions"] NODE_DISPLAY_NAME_MAPPINGS = { "AudioSeparatorSimple": "Audio Separator", "ProximityVisualizer": "Preview Proximity", - "EffectVisualizer": "Preview Effect", + "EffectVisualizer": "Preview FeatureEffect", "PitchVisualizer": "Preview Pitch", "FlexVideoSpeed": "**BETA** Flex Video Speed", "FlexVideoFrameBlend": "**BETA**Flex Video Frame Blend", - "AudioFeatureVisualizer": "Audio Feature Visualizer ***BETA***" , + "AudioFeatureVisualizer": "Audio Feature Visualizer ***BETA***", "MIDILoadAndExtract": "MIDI Load & Feature Extract", "PitchRangeByNoteNode": "Pitch Range By Note", @@ -632,7 +634,6 @@ NODE_DISPLAY_NAME_MAPPINGS = { "Audio_Concatenate": "Audio Concatenate ROTI", "Color_Picker": "Color_Picker ROTI", "Doom_": "DOOM", - "MIDIPlayback": "MIDI Player", } diff --git a/nodes/flex/feature_modulation.py b/nodes/flex/feature_modulation.py index 2444f2c..6e6aa9e 100644 --- a/nodes/flex/feature_modulation.py +++ b/nodes/flex/feature_modulation.py @@ -779,3 +779,115 @@ class FeaturePeakDetector(FeatureModulationBase): processed_feature = self.create_processed_feature(feature, peak_signal, name_prefix, invert_output) return (processed_feature,) +@apply_tooltips +class FeatureInterpolateMulti(FeatureModulationBase): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "feature1": ("FEATURE",), + "feature2": ("FEATURE",), + "feature3": ("FEATURE",), + "min_aggregate_value": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}), + "transition_frames": ("INT", {"default": 30, "min": 1, "max": 100, "step": 1}), + **super().INPUT_TYPES()["required"], + } + } + + RETURN_TYPES = ("FEATURE", "FEATURE", "FEATURE") + RETURN_NAMES = ("FEATURE1", "FEATURE2", "FEATURE3") + FUNCTION = "multi_interpolate" + + def multi_interpolate(self, feature1, feature2, feature3, min_aggregate_value, transition_frames, invert_output): + # Ensure all features have same length + frame_count = min(feature1.frame_count, feature2.frame_count, feature3.frame_count) + + # Extract values + values1 = np.array([feature1.get_value_at_frame(i) for i in range(frame_count)]) + values2 = np.array([feature2.get_value_at_frame(i) for i in range(frame_count)]) + values3 = np.array([feature3.get_value_at_frame(i) for i in range(frame_count)]) + + # Stack features + stacked = np.stack([values1, values2, values3], axis=1) # [frame, 3] + + # Create output array + output = np.zeros_like(stacked) + + # Find transition points (where signals change significantly) + threshold = 0.05 # Threshold to detect changes + transitions = [] + + for i in range(1, frame_count): + # Check for significant changes in any feature + changes = np.abs(stacked[i] - stacked[i-1]) + if np.any(changes > threshold): + transitions.append(i) + + # Process each segment between transitions + segment_start = 0 + for transition in transitions + [frame_count]: + segment_end = transition + + # Look ahead and behind to determine transition window + transition_start = max(segment_start - transition_frames // 2, 0) + transition_end = min(segment_end + transition_frames // 2, frame_count) + + # For the main part of the segment (no transition), use normalized values + if segment_end > segment_start: + # Get the average feature values in this segment + segment_values = np.mean(stacked[segment_start:segment_end], axis=0) + + # Normalize to ensure sum equals min_aggregate_value + sum_segment = np.sum(segment_values) + if sum_segment > 0: + normalized_segment = (segment_values / sum_segment) * min_aggregate_value + else: + # If all zero, distribute evenly + normalized_segment = np.ones(3) * (min_aggregate_value / 3) + + # Apply to the non-transition part of the segment + for i in range(segment_start, segment_end): + output[i] = normalized_segment + + segment_start = segment_end + + # Create linear transitions between segments + for i in range(len(transitions)): + trans_idx = transitions[i] + + # Create transition window + window_start = max(trans_idx - transition_frames, 0) + window_end = min(trans_idx + transition_frames, frame_count) + + if window_end > window_start: + # Get stable values before and after transition + before_value = output[max(window_start - 1, 0)] + after_value = output[min(window_end, frame_count - 1)] + + # Linear interpolation across the window + for j in range(window_start, window_end): + alpha = (j - window_start) / (window_end - window_start) + interpolated = (1 - alpha) * before_value + alpha * after_value + + # Ensure sum is maintained at min_aggregate_value + sum_interp = np.sum(interpolated) + if sum_interp > 0: + output[j] = (interpolated / sum_interp) * min_aggregate_value + else: + output[j] = np.ones(3) * (min_aggregate_value / 3) + + # Final check: ensure all frames sum exactly to min_aggregate_value + for i in range(frame_count): + total = np.sum(output[i]) + if abs(total - min_aggregate_value) > 1e-6 and total > 0: + output[i] = (output[i] / total) * min_aggregate_value + elif total == 0: + output[i] = np.ones(3) * (min_aggregate_value / 3) + + # Split back out + out1 = self.create_processed_feature(feature1, output[:, 0], "InterpolatedMulti", invert_output) + out2 = self.create_processed_feature(feature2, output[:, 1], "InterpolatedMulti", invert_output) + out3 = self.create_processed_feature(feature3, output[:, 2], "InterpolatedMulti", invert_output) + + return (out1, out2, out3) + From f31f54f7935915e0112d5a79f6803920ec416858 Mon Sep 17 00:00:00 2001 From: RyanOnTheInside <7623207+ryanontheinside@users.noreply.github.com> Date: Sat, 24 May 2025 10:09:49 -0400 Subject: [PATCH 10/10] midi tutorial --- examples/tutorials/MIDI/MIDI-Tutorial.json | 1832 +++++++++++------ .../MIDI/ace_step_audio_separation.json | 1195 +++++++++++ nodes/audio/midi_nodes.py | 95 +- nodes/flex/feature_modulation.py | 2 +- 4 files changed, 2440 insertions(+), 684 deletions(-) create mode 100644 examples/tutorials/MIDI/ace_step_audio_separation.json diff --git a/examples/tutorials/MIDI/MIDI-Tutorial.json b/examples/tutorials/MIDI/MIDI-Tutorial.json index 82da63d..0208f88 100644 --- a/examples/tutorials/MIDI/MIDI-Tutorial.json +++ b/examples/tutorials/MIDI/MIDI-Tutorial.json @@ -1,65 +1,22 @@ { "id": "5f2341e1-12d2-42c3-b821-0c41d3d8202f", "revision": 0, - "last_node_id": 99, - "last_link_id": 127, + "last_node_id": 116, + "last_link_id": 150, "nodes": [ - { - "id": 45, - "type": "MIDILoader", - "pos": [ - -1339.8095703125, - 1347.4271240234375 - ], - "size": [ - 381.1168212890625, - 226 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MIDI", - "type": "MIDI", - "links": [ - 64, - 67, - 69, - 78 - ] - } - ], - "properties": { - "cnr_id": "comfyui_ryanontheinside", - "ver": "e75dbb7156d34ee0d0dcfba629be0acb7f9c2538", - "widget_ue_connectable": {} - }, - "widgets_values": [ - "sample_midi.mid", - "1: piano", - 1, - 1, - 0, - 1, - "upload", - "refresh" - ] - }, { "id": 29, "type": "PreviewAudio", "pos": [ - -394.14019775390625, - 728.9194946289062 + -378.11572265625, + 589.7911987304688 ], "size": [ 310.4596252441406, 126.08221435546875 ], "flags": {}, - "order": 16, + "order": 17, "mode": 0, "inputs": [ { @@ -81,15 +38,15 @@ "id": 71, "type": "PreviewAudio", "pos": [ - -370, - -170 + -367.02716064453125, + -193.7830352783203 ], "size": [ 315, 88 ], "flags": {}, - "order": 19, + "order": 20, "mode": 0, "inputs": [ { @@ -119,7 +76,7 @@ 88 ], "flags": {}, - "order": 17, + "order": 23, "mode": 0, "inputs": [ { @@ -137,242 +94,19 @@ }, "widgets_values": [] }, - { - "id": 72, - "type": "MIDILoader", - "pos": [ - -1297.677001953125, - -1001.4724731445312 - ], - "size": [ - 381.1168212890625, - 226 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MIDI", - "type": "MIDI", - "links": [ - 91, - 102, - 103 - ] - } - ], - "properties": { - "cnr_id": "comfyui_ryanontheinside", - "ver": "e75dbb7156d34ee0d0dcfba629be0acb7f9c2538", - "widget_ue_connectable": {} - }, - "widgets_values": [ - "sample_midi.mid", - "3: drum", - 1, - 1, - 0, - 1, - "upload", - "refresh" - ] - }, - { - "id": 79, - "type": "MIDIFeatureExtractor", - "pos": [ - -767.5911254882812, - -830.1699829101562 - ], - "size": [ - 1020, - 322 - ], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "midi", - "type": "MIDI", - "link": 102 - } - ], - "outputs": [ - { - "name": "midi", - "type": "MIDI", - "links": null - }, - { - "name": "feature", - "type": "FEATURE", - "links": [ - 107 - ] - }, - { - "name": "frame_count", - "type": "INT", - "links": [] - } - ], - "properties": { - "cnr_id": "comfyui_ryanontheinside", - "ver": "d060e57a8db80e631364478a3b228fc632d85f39", - "selectedNotes": [ - 36 - ], - "widget_ue_connectable": {} - }, - "widgets_values": [ - "Note On/Off", - 30, - 0, - 512, - 512, - false, - "36" - ] - }, - { - "id": 75, - "type": "MIDIFeatureExtractor", - "pos": [ - -820, - -10 - ], - "size": [ - 1020, - 322 - ], - "flags": {}, - "order": 15, - "mode": 0, - "inputs": [ - { - "name": "midi", - "type": "MIDI", - "link": 93 - } - ], - "outputs": [ - { - "name": "midi", - "type": "MIDI", - "links": null - }, - { - "name": "feature", - "type": "FEATURE", - "links": [ - 99 - ] - }, - { - "name": "frame_count", - "type": "INT", - "links": [] - } - ], - "properties": { - "cnr_id": "comfyui_ryanontheinside", - "ver": "d060e57a8db80e631364478a3b228fc632d85f39", - "selectedNotes": [], - "widget_ue_connectable": {} - }, - "widgets_values": [ - "Velocity", - 30, - 0, - 512, - 512, - false, - "" - ] - }, - { - "id": 88, - "type": "Note", - "pos": [ - -1312.242919921875, - -1373.744873046875 - ], - "size": [ - 373.50885009765625, - 179.8002166748047 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "2. Drums", - "properties": { - "widget_ue_connectable": {} - }, - "widgets_values": [ - "For the drums, we have an example of extracting individual notes, the kick and the snare." - ], - "color": "#432", - "bgcolor": "#653" - }, - { - "id": 69, - "type": "MIDILoader", - "pos": [ - -1312.6334228515625, - 97.49837493896484 - ], - "size": [ - 381.1168212890625, - 226 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MIDI", - "type": "MIDI", - "links": [ - 89, - 93 - ] - } - ], - "properties": { - "cnr_id": "comfyui_ryanontheinside", - "ver": "e75dbb7156d34ee0d0dcfba629be0acb7f9c2538", - "widget_ue_connectable": {} - }, - "widgets_values": [ - "sample_midi.mid", - "2: bass", - 1, - 1, - 0, - 1, - "upload", - "refresh" - ] - }, { "id": 89, "type": "Note", "pos": [ - -1315.5673828125, - -162.17872619628906 + -1312.59423828125, + -185.96177673339844 ], "size": [ 372.4935607910156, 182.84597778320312 ], "flags": {}, - "order": 4, + "order": 0, "mode": 0, "inputs": [], "outputs": [], @@ -390,15 +124,15 @@ "id": 91, "type": "Note", "pos": [ - -1322.5369873046875, - 903.3281860351562 + -1306.512451171875, + 764.1998901367188 ], "size": [ 372.4935607910156, 182.84597778320312 ], "flags": {}, - "order": 5, + "order": 1, "mode": 0, "inputs": [], "outputs": [], @@ -424,7 +158,7 @@ 480.17010498046875 ], "flags": {}, - "order": 6, + "order": 2, "mode": 0, "inputs": [], "outputs": [], @@ -438,190 +172,6 @@ "color": "#432", "bgcolor": "#653" }, - { - "id": 64, - "type": "MIDIFeatureExtractor", - "pos": [ - -871.2890625, - 2088.83984375 - ], - "size": [ - 1020, - 322 - ], - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "midi", - "type": "MIDI", - "link": 78 - } - ], - "outputs": [ - { - "name": "midi", - "type": "MIDI", - "links": null - }, - { - "name": "feature", - "type": "FEATURE", - "links": [ - 88 - ] - }, - { - "name": "frame_count", - "type": "INT", - "links": null - } - ], - "properties": { - "cnr_id": "comfyui_ryanontheinside", - "ver": "d060e57a8db80e631364478a3b228fc632d85f39", - "selectedNotes": [ - 41, - 53, - 60, - 68 - ], - "widget_ue_connectable": {} - }, - "widgets_values": [ - "Velocity", - 30, - 0, - 512, - 512, - true, - "41, 53, 60, 68" - ] - }, - { - "id": 43, - "type": "MIDIFeatureExtractor", - "pos": [ - -852.0150756835938, - 1054.3472900390625 - ], - "size": [ - 1020, - 322 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "midi", - "type": "MIDI", - "link": 67 - } - ], - "outputs": [ - { - "name": "midi", - "type": "MIDI", - "links": null - }, - { - "name": "feature", - "type": "FEATURE", - "links": [ - 50 - ] - }, - { - "name": "frame_count", - "type": "INT", - "links": [] - } - ], - "properties": { - "cnr_id": "comfyui_ryanontheinside", - "ver": "492c20ba4ae1484c9e3bdc11d036fbb4820c52c1", - "selectedNotes": [ - 37, - 49, - 56, - 61, - 65 - ], - "widget_ue_connectable": {} - }, - "widgets_values": [ - "Velocity", - 30, - 0, - 512, - 512, - true, - "37, 49, 56, 61, 65" - ] - }, - { - "id": 56, - "type": "MIDIFeatureExtractor", - "pos": [ - -852.0150756835938, - 1523.5477294921875 - ], - "size": [ - 1020, - 322 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "midi", - "type": "MIDI", - "link": 69 - } - ], - "outputs": [ - { - "name": "midi", - "type": "MIDI", - "links": null - }, - { - "name": "feature", - "type": "FEATURE", - "links": [ - 75 - ] - }, - { - "name": "frame_count", - "type": "INT", - "links": [] - } - ], - "properties": { - "cnr_id": "comfyui_ryanontheinside", - "ver": "d060e57a8db80e631364478a3b228fc632d85f39", - "selectedNotes": [ - 51, - 58, - 63, - 67 - ], - "widget_ue_connectable": {} - }, - "widgets_values": [ - "Velocity", - 30, - 0, - 512, - 512, - true, - "51, 58, 63, 67" - ] - }, { "id": 84, "type": "EffectVisualizer", @@ -634,7 +184,7 @@ 174 ], "flags": {}, - "order": 21, + "order": 30, "mode": 0, "inputs": [ { @@ -683,7 +233,7 @@ 106 ], "flags": {}, - "order": 11, + "order": 14, "mode": 0, "inputs": [ { @@ -714,79 +264,19 @@ 0.5 ] }, - { - "id": 80, - "type": "MIDIFeatureExtractor", - "pos": [ - -770.4898071289062, - -1256.8670654296875 - ], - "size": [ - 1020, - 322 - ], - "flags": {}, - "order": 13, - "mode": 0, - "inputs": [ - { - "name": "midi", - "type": "MIDI", - "link": 103 - } - ], - "outputs": [ - { - "name": "midi", - "type": "MIDI", - "links": [] - }, - { - "name": "feature", - "type": "FEATURE", - "links": [ - 110 - ] - }, - { - "name": "frame_count", - "type": "INT", - "links": [ - 111 - ] - } - ], - "properties": { - "cnr_id": "comfyui_ryanontheinside", - "ver": "d060e57a8db80e631364478a3b228fc632d85f39", - "selectedNotes": [ - 38 - ], - "widget_ue_connectable": {} - }, - "widgets_values": [ - "Note On/Off", - 30, - 0, - 512, - 512, - false, - "38" - ] - }, { "id": 70, "type": "MIDIToAudio", "pos": [ - -790, - -170 + -787.0272216796875, + -193.7830352783203 ], "size": [ 315, 106 ], "flags": {}, - "order": 14, + "order": 12, "mode": 0, "inputs": [ { @@ -821,15 +311,15 @@ "id": 97, "type": "Audio_Combine", "pos": [ - 1589.7186279296875, - -94.85237884521484 + 1592.6917724609375, + -118.63542175292969 ], "size": [ 352.79998779296875, 102 ], "flags": {}, - "order": 20, + "order": 24, "mode": 0, "inputs": [ { @@ -855,8 +345,8 @@ "properties": { "cnr_id": "comfyui_ryanontheinside", "ver": "d060e57a8db80e631364478a3b228fc632d85f39", - "widget_ue_connectable": {}, - "Node name for S&R": "Audio_Combine" + "Node name for S&R": "Audio_Combine", + "widget_ue_connectable": {} }, "widgets_values": [ 0.5, @@ -867,15 +357,15 @@ "id": 50, "type": "MIDIToAudio", "pos": [ - -821.4395751953125, - 731.225341796875 + -805.415283203125, + 592.0970458984375 ], "size": [ 315, 106 ], "flags": {}, - "order": 7, + "order": 8, "mode": 0, "inputs": [ { @@ -910,15 +400,15 @@ "id": 99, "type": "Audio_Combine", "pos": [ - 1560.019287109375, - 773.7644653320312 + 1576.0438232421875, + 634.6361694335938 ], "size": [ 352.79998779296875, 102 ], "flags": {}, - "order": 22, + "order": 29, "mode": 0, "inputs": [ { @@ -937,99 +427,21 @@ "name": "AUDIO", "type": "AUDIO", "links": [ - 127 + 138 ] } ], "properties": { "cnr_id": "comfyui_ryanontheinside", "ver": "d060e57a8db80e631364478a3b228fc632d85f39", - "widget_ue_connectable": {}, - "Node name for S&R": "Audio_Combine" + "Node name for S&R": "Audio_Combine", + "widget_ue_connectable": {} }, "widgets_values": [ 0.5, 0.5 ] }, - { - "id": 68, - "type": "VHS_VideoCombine", - "pos": [ - 2741.55517578125, - -937.7992553710938 - ], - "size": [ - 1749.921875, - 1321.0810546875 - ], - "flags": {}, - "order": 28, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 86 - }, - { - "name": "audio", - "shape": 7, - "type": "AUDIO", - "link": 127 - }, - { - "name": "meta_batch", - "shape": 7, - "type": "VHS_BatchManager", - "link": null - }, - { - "name": "vae", - "shape": 7, - "type": "VAE", - "link": null - } - ], - "outputs": [ - { - "name": "Filenames", - "type": "VHS_FILENAMES", - "links": null - } - ], - "properties": { - "cnr_id": "comfyui-videohelpersuite", - "ver": "1.6.1", - "Node name for S&R": "VHS_VideoCombine", - "widget_ue_connectable": {} - }, - "widgets_values": { - "frame_rate": 30, - "loop_count": 0, - "filename_prefix": "AnimateDiff", - "format": "video/h264-mp4", - "pix_fmt": "yuv420p", - "crf": 19, - "save_metadata": true, - "trim_to_audio": false, - "pingpong": false, - "save_output": false, - "videopreview": { - "hidden": false, - "paused": false, - "params": { - "filename": "AnimateDiff_00022-audio.mp4", - "subfolder": "", - "type": "temp", - "format": "video/h264-mp4", - "frame_rate": 30, - "workflow": "AnimateDiff_00022.png", - "fullpath": "C:\\_dev\\comfy\\ComfyUI\\temp\\AnimateDiff_00022-audio.mp4" - } - } - } - }, { "id": 85, "type": "EmptyImage", @@ -1042,7 +454,7 @@ 130 ], "flags": {}, - "order": 18, + "order": 27, "mode": 0, "inputs": [ { @@ -1088,7 +500,7 @@ 174 ], "flags": {}, - "order": 23, + "order": 33, "mode": 0, "inputs": [ { @@ -1129,15 +541,15 @@ "id": 76, "type": "EffectVisualizer", "pos": [ - 910, - -10 + 912.9727783203125, + -33.782958984375 ], "size": [ 315, 174 ], "flags": {}, - "order": 24, + "order": 35, "mode": 0, "inputs": [ { @@ -1178,15 +590,15 @@ "id": 38, "type": "EffectVisualizer", "pos": [ - 906.0049438476562, - 1105.6773681640625 + 922.029296875, + 966.5489501953125 ], "size": [ 315, 174 ], "flags": {}, - "order": 25, + "order": 36, "mode": 0, "inputs": [ { @@ -1227,15 +639,15 @@ "id": 58, "type": "EffectVisualizer", "pos": [ - 933.9990844726562, - 1612.6243896484375 + 950.0234375, + 1473.495849609375 ], "size": [ 315, 174 ], "flags": {}, - "order": 26, + "order": 37, "mode": 0, "inputs": [ { @@ -1272,19 +684,85 @@ 210 ] }, + { + "id": 100, + "type": "Display Int (rgthree)", + "pos": [ + 343.03497314453125, + 168.9556427001953 + ], + "size": [ + 264.5999755859375, + 76 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "dir": 3, + "name": "input", + "type": "INT", + "link": 128 + } + ], + "outputs": [], + "properties": { + "cnr_id": "rgthree-comfy", + "ver": "6c5f7c95ed8487fe08e42bd0341cae6dc9c0f0ad", + "widget_ue_connectable": {}, + "Node name for S&R": "Display Int (rgthree)" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 101, + "type": "Display Int (rgthree)", + "pos": [ + 329.8796691894531, + -677.2709350585938 + ], + "size": [ + 264.5999755859375, + 76 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "dir": 3, + "name": "input", + "type": "INT", + "link": 129 + } + ], + "outputs": [], + "properties": { + "cnr_id": "rgthree-comfy", + "ver": "6c5f7c95ed8487fe08e42bd0341cae6dc9c0f0ad", + "widget_ue_connectable": {}, + "Node name for S&R": "Display Int (rgthree)" + }, + "widgets_values": [ + "" + ] + }, { "id": 67, "type": "EffectVisualizer", "pos": [ - 912.8372802734375, - 2150.399658203125 + 899.8955078125, + 2003.3714599609375 ], "size": [ 315, 174 ], "flags": {}, - "order": 27, + "order": 38, "mode": 0, "inputs": [ { @@ -1320,6 +798,974 @@ 10, 270 ] + }, + { + "id": 88, + "type": "Note", + "pos": [ + -1312.242919921875, + -1373.744873046875 + ], + "size": [ + 373.50885009765625, + 179.8002166748047 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "2. Drums", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "For the drums, we have an example of extracting individual notes, the kick and the snare." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 64, + "type": "MIDIFeatureExtractor", + "pos": [ + -848.0704345703125, + 1863.379638671875 + ], + "size": [ + 1020, + 322 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "midi", + "type": "MIDI", + "link": 78 + } + ], + "outputs": [ + { + "name": "midi", + "type": "MIDI", + "links": null + }, + { + "name": "feature", + "type": "FEATURE", + "links": [ + 88 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "d060e57a8db80e631364478a3b228fc632d85f39", + "selectedNotes": [ + 41, + 53, + 60, + 68 + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Velocity", + 30, + 0, + 512, + 512, + false, + "41, 53, 60, 68" + ] + }, + { + "id": 68, + "type": "VHS_VideoCombine", + "pos": [ + 2494.611328125, + -1668.78662109375 + ], + "size": [ + 1749.921875, + 1321.0810546875 + ], + "flags": {}, + "order": 39, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 86 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 139 + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "1.6.1", + "Node name for S&R": "VHS_VideoCombine", + "widget_ue_connectable": {} + }, + "widgets_values": { + "frame_rate": 30, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00055-audio.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4", + "frame_rate": 30, + "workflow": "AnimateDiff_00055.png", + "fullpath": "C:\\_dev\\comfy\\ComfyUI\\temp\\AnimateDiff_00055-audio.mp4" + } + } + } + }, + { + "id": 79, + "type": "MIDIFeatureExtractor", + "pos": [ + -818.8341064453125, + -827.5294189453125 + ], + "size": [ + 1020, + 322 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "midi", + "type": "MIDI", + "link": 102 + } + ], + "outputs": [ + { + "name": "midi", + "type": "MIDI", + "links": null + }, + { + "name": "feature", + "type": "FEATURE", + "links": [ + 107 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": [ + 129 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "d060e57a8db80e631364478a3b228fc632d85f39", + "selectedNotes": [ + 36 + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Note On/Off", + 30, + 0, + 512, + 512, + false, + "36" + ] + }, + { + "id": 109, + "type": "PreviewFeature", + "pos": [ + 333.9266052246094, + -1549.719482421875 + ], + "size": [ + 315, + 246 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 142 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "6143b478e707c55a12014c5d67c46ec947bb1451", + "widget_ue_connectable": {}, + "Node name for S&R": "PreviewFeature" + }, + "widgets_values": [] + }, + { + "id": 80, + "type": "MIDIFeatureExtractor", + "pos": [ + -815.8054809570312, + -1259.177490234375 + ], + "size": [ + 1020, + 322 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "midi", + "type": "MIDI", + "link": 103 + } + ], + "outputs": [ + { + "name": "midi", + "type": "MIDI", + "links": [] + }, + { + "name": "feature", + "type": "FEATURE", + "links": [ + 110, + 142 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": [ + 111 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "d060e57a8db80e631364478a3b228fc632d85f39", + "selectedNotes": [ + 36 + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Note On/Off", + 30, + 0, + 512, + 512, + false, + "36" + ] + }, + { + "id": 75, + "type": "MIDIFeatureExtractor", + "pos": [ + -801.7716064453125, + -39.12238311767578 + ], + "size": [ + 1020, + 322 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "midi", + "type": "MIDI", + "link": 93 + } + ], + "outputs": [ + { + "name": "midi", + "type": "MIDI", + "links": null + }, + { + "name": "feature", + "type": "FEATURE", + "links": [ + 99, + 143 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": [ + 128 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "d060e57a8db80e631364478a3b228fc632d85f39", + "selectedNotes": [], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Velocity", + 30, + 0, + 512, + 512, + false, + "" + ] + }, + { + "id": 110, + "type": "PreviewFeature", + "pos": [ + 416.0970764160156, + -180.49119567871094 + ], + "size": [ + 315, + 246 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 143 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "6143b478e707c55a12014c5d67c46ec947bb1451", + "widget_ue_connectable": {}, + "Node name for S&R": "PreviewFeature" + }, + "widgets_values": [] + }, + { + "id": 111, + "type": "PreviewFeature", + "pos": [ + 350.5138244628906, + 884.7586059570312 + ], + "size": [ + 315, + 246 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 144 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "6143b478e707c55a12014c5d67c46ec947bb1451", + "widget_ue_connectable": {}, + "Node name for S&R": "PreviewFeature" + }, + "widgets_values": [] + }, + { + "id": 56, + "type": "MIDIFeatureExtractor", + "pos": [ + -772.334228515625, + 1379.951904296875 + ], + "size": [ + 1020, + 322 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "midi", + "type": "MIDI", + "link": 69 + } + ], + "outputs": [ + { + "name": "midi", + "type": "MIDI", + "links": null + }, + { + "name": "feature", + "type": "FEATURE", + "links": [ + 75 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": [] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "d060e57a8db80e631364478a3b228fc632d85f39", + "selectedNotes": [ + 51, + 58, + 67 + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Velocity", + 30, + 0, + 512, + 512, + false, + "51, 58, 67" + ] + }, + { + "id": 43, + "type": "MIDIFeatureExtractor", + "pos": [ + -797.3942260742188, + 927.8047485351562 + ], + "size": [ + 1020, + 322 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "midi", + "type": "MIDI", + "link": 67 + } + ], + "outputs": [ + { + "name": "midi", + "type": "MIDI", + "links": null + }, + { + "name": "feature", + "type": "FEATURE", + "links": [ + 50, + 144, + 145 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": [] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "492c20ba4ae1484c9e3bdc11d036fbb4820c52c1", + "selectedNotes": [ + 37, + 49, + 56, + 61, + 65 + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Velocity", + 30, + 0, + 512, + 512, + true, + "37, 49, 56, 61, 65" + ] + }, + { + "id": 113, + "type": "VHS_VideoCombine", + "pos": [ + 4731.84814453125, + 876.3460083007812 + ], + "size": [ + 1749.921875, + 2077.921875 + ], + "flags": {}, + "order": 34, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 149 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 150 + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "1.6.1", + "widget_ue_connectable": {}, + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 30, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00057-audio.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4", + "frame_rate": 30, + "workflow": "AnimateDiff_00057.png", + "fullpath": "C:\\_dev\\comfy\\ComfyUI\\temp\\AnimateDiff_00057-audio.mp4" + } + } + } + }, + { + "id": 112, + "type": "Reroute", + "pos": [ + 2915.017578125, + 1197.1243896484375 + ], + "size": [ + 75, + 26 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "", + "type": "*", + "link": 145 + } + ], + "outputs": [ + { + "name": "", + "type": "FEATURE", + "links": [ + 146 + ] + } + ], + "properties": { + "showOutputText": false, + "horizontal": false + } + }, + { + "id": 115, + "type": "QuickShapeMask", + "pos": [ + 2789.93212890625, + 898.5335083007812 + ], + "size": [ + 315, + 226 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 147 + ] + } + ], + "properties": { + "aux_id": "ryanontheinside/ComfyUI_RealtimeNodes", + "ver": "91b155a115c2e34f95b9e8e4180ef4d36fe62c87", + "widget_ue_connectable": {}, + "Node name for S&R": "QuickShapeMask" + }, + "widgets_values": [ + "circle", + 64, + 64, + 256, + 256, + 512, + 512, + 1 + ] + }, + { + "id": 114, + "type": "FlexMaskMorph", + "pos": [ + 3305.267578125, + 1196.164306640625 + ], + "size": [ + 315, + 318 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "name": "masks", + "type": "MASK", + "link": 147 + }, + { + "name": "opt_feature", + "shape": 7, + "type": "FEATURE", + "link": 146 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 148 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "6143b478e707c55a12014c5d67c46ec947bb1451", + "widget_ue_connectable": {}, + "Node name for S&R": "FlexMaskMorph" + }, + "widgets_values": [ + 1, + 0, + "max_kernel_size", + "relative", + false, + 0, + 0, + 1, + "dilate", + 5, + 10 + ] + }, + { + "id": 116, + "type": "MaskToImage", + "pos": [ + 3722.48046875, + 1218.4617919921875 + ], + "size": [ + 264.5999755859375, + 26 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 148 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 149 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.34", + "widget_ue_connectable": {}, + "Node name for S&R": "MaskToImage" + } + }, + { + "id": 108, + "type": "Reroute", + "pos": [ + 2350.337890625, + 734.9080200195312 + ], + "size": [ + 75, + 26 + ], + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "name": "", + "type": "*", + "link": 138 + } + ], + "outputs": [ + { + "name": "", + "type": "AUDIO", + "links": [ + 139, + 150 + ] + } + ], + "properties": { + "showOutputText": false, + "horizontal": false + } + }, + { + "id": 45, + "type": "MIDILoader", + "pos": [ + -1323.78515625, + 1073.167724609375 + ], + "size": [ + 381.1168212890625, + 226 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MIDI", + "type": "MIDI", + "links": [ + 64, + 67, + 69, + 78 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "e75dbb7156d34ee0d0dcfba629be0acb7f9c2538", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "sample_midi.mid", + "1: piano", + 1, + 1, + 0, + 1, + "upload", + "refresh" + ] + }, + { + "id": 69, + "type": "MIDILoader", + "pos": [ + -1309.6602783203125, + 73.71533966064453 + ], + "size": [ + 381.1168212890625, + 226 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MIDI", + "type": "MIDI", + "links": [ + 89, + 93 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "e75dbb7156d34ee0d0dcfba629be0acb7f9c2538", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "sample_midi.mid", + "2: bass", + 1, + 1, + 0, + 1, + "upload", + "refresh" + ] + }, + { + "id": 72, + "type": "MIDILoader", + "pos": [ + -1293.427978515625, + -1096.0692138671875 + ], + "size": [ + 381.1168212890625, + 226 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MIDI", + "type": "MIDI", + "links": [ + 91, + 102, + 103 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "e75dbb7156d34ee0d0dcfba629be0acb7f9c2538", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "sample_midi.mid", + "3: drum", + 1, + 1, + 0, + 1, + "upload", + "refresh" + ] } ], "links": [ @@ -1572,12 +2018,108 @@ "AUDIO" ], [ - 127, + 128, + 75, + 2, + 100, + 0, + "INT" + ], + [ + 129, + 79, + 2, + 101, + 0, + "INT" + ], + [ + 138, 99, 0, + 108, + 0, + "*" + ], + [ + 139, + 108, + 0, 68, 1, "AUDIO" + ], + [ + 142, + 80, + 1, + 109, + 0, + "FEATURE" + ], + [ + 143, + 75, + 1, + 110, + 0, + "FEATURE" + ], + [ + 144, + 43, + 1, + 111, + 0, + "FEATURE" + ], + [ + 145, + 43, + 1, + 112, + 0, + "*" + ], + [ + 146, + 112, + 0, + 114, + 1, + "FEATURE" + ], + [ + 147, + 115, + 0, + 114, + 0, + "MASK" + ], + [ + 148, + 114, + 0, + 116, + 0, + "MASK" + ], + [ + 149, + 116, + 0, + 113, + 0, + "IMAGE" + ], + [ + 150, + 108, + 0, + 113, + 1, + "AUDIO" ] ], "groups": [ @@ -1585,8 +2127,8 @@ "id": 1, "title": "Bass", "bounding": [ - -1362.0684814453125, - -374.0391845703125, + -1359.0953369140625, + -397.8218078613281, 3360.12255859375, 775.947998046875 ], @@ -1598,8 +2140,8 @@ "id": 2, "title": "Piano", "bounding": [ - -1361.111083984375, - 532.2680053710938, + -1345.0865478515625, + 393.14019775390625, 3372.01708984375, 2264.61669921875 ], @@ -1624,10 +2166,10 @@ "config": {}, "extra": { "ds": { - "scale": 0.1210000000000066, + "scale": 0.13310000000000022, "offset": [ - 5189.493242450809, - 2982.4761975652627 + 4000.071857761043, + 2264.0340374020057 ] }, "ue_links": [], diff --git a/examples/tutorials/MIDI/ace_step_audio_separation.json b/examples/tutorials/MIDI/ace_step_audio_separation.json new file mode 100644 index 0000000..efa75b0 --- /dev/null +++ b/examples/tutorials/MIDI/ace_step_audio_separation.json @@ -0,0 +1,1195 @@ +{ + "id": "88ac5dad-efd7-40bb-84fe-fbaefdee1fa9", + "revision": 0, + "last_node_id": 75, + "last_link_id": 139, + "nodes": [ + { + "id": 44, + "type": "ConditioningZeroOut", + "pos": [ + 790, + 610 + ], + "size": [ + 211.60000610351562, + 26 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "conditioning", + "type": "CONDITIONING", + "link": 108 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 109 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.32", + "Node name for S&R": "ConditioningZeroOut", + "widget_ue_connectable": {} + }, + "widgets_values": [] + }, + { + "id": 17, + "type": "EmptyAceStepLatentAudio", + "pos": [ + 710, + 690 + ], + "size": [ + 270, + 82 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 23 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.32", + "Node name for S&R": "EmptyAceStepLatentAudio", + "widget_ue_connectable": {} + }, + "widgets_values": [ + 120, + 1 + ] + }, + { + "id": 55, + "type": "ModelSamplingSD3", + "pos": [ + 709.7966918945312, + -211.7812957763672 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 120 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 117 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.32", + "Node name for S&R": "ModelSamplingSD3", + "widget_ue_connectable": {} + }, + "widgets_values": [ + 5.000000000000001 + ] + }, + { + "id": 65, + "type": "PreviewAudio", + "pos": [ + 2700, + 230 + ], + "size": [ + 297.4637756347656, + 88 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 129 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.34", + "widget_ue_connectable": {}, + "Node name for S&R": "PreviewAudio" + }, + "widgets_values": [] + }, + { + "id": 70, + "type": "MaskToImage", + "pos": [ + 4670, + -200 + ], + "size": [ + 264.5999755859375, + 26 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 135 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 136 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.34", + "widget_ue_connectable": {}, + "Node name for S&R": "MaskToImage" + } + }, + { + "id": 73, + "type": "VHS_VideoCombine", + "pos": [ + 4950, + -100 + ], + "size": [ + 214.7587890625, + 542.7587890625 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 136 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 137 + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "1.6.1", + "widget_ue_connectable": {}, + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 30, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_01246-audio.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 30, + "workflow": "AnimateDiff_01246.png", + "fullpath": "C:\\_dev\\comfy\\ComfyUI\\output\\AnimateDiff_01246-audio.mp4" + } + } + } + }, + { + "id": 40, + "type": "CheckpointLoaderSimple", + "pos": [ + 167.00906372070312, + 80.0209732055664 + ], + "size": [ + 375, + 98 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 120 + ] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 80 + ] + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 83 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.32", + "Node name for S&R": "CheckpointLoaderSimple", + "models": [ + { + "name": "ace_step_v1_3.5b.safetensors", + "url": "https://huggingface.co/Comfy-Org/ACE-Step_ComfyUI_repackaged/resolve/main/all_in_one/ace_step_v1_3.5b.safetensors?download=true", + "directory": "checkpoints" + } + ], + "widget_ue_connectable": {} + }, + "widgets_values": [ + "ace_step_v1_3.5b.safetensors" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 56, + "type": "LatentOperationTonemapReinhard", + "pos": [ + 691.5786743164062, + -72.72840118408203 + ], + "size": [ + 314.7124938964844, + 58 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT_OPERATION", + "type": "LATENT_OPERATION", + "links": [ + 118 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.34", + "Node name for S&R": "LatentOperationTonemapReinhard", + "widget_ue_connectable": {} + }, + "widgets_values": [ + 1.0000000000000002 + ] + }, + { + "id": 54, + "type": "LatentApplyOperationCFG", + "pos": [ + 1082.0316162109375, + -92.0929946899414 + ], + "size": [ + 204.87850952148438, + 46 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 117 + }, + { + "name": "operation", + "type": "LATENT_OPERATION", + "link": 118 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 119 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.34", + "Node name for S&R": "LatentApplyOperationCFG", + "widget_ue_connectable": {} + }, + "widgets_values": [] + }, + { + "id": 14, + "type": "TextEncodeAceStepAudio", + "pos": [ + 606.9805908203125, + 66.50801086425781 + ], + "size": [ + 413.60076904296875, + 447.163330078125 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 80 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 108, + 110 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.32", + "Node name for S&R": "TextEncodeAceStepAudio", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "lofi beat, trip hop", + "", + 1.0000000000000002 + ] + }, + { + "id": 3, + "type": "KSampler", + "pos": [ + 1060.9764404296875, + 77.41416931152344 + ], + "size": [ + 315, + 262 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 119 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 110 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 109 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 23 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [ + 101 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.32", + "Node name for S&R": "KSampler", + "widget_ue_connectable": {} + }, + "widgets_values": [ + 882554388057422, + "fixed", + 50, + 5, + "res_multistep", + "simple", + 1 + ] + }, + { + "id": 18, + "type": "VAEDecodeAudio", + "pos": [ + 1434.304443359375, + 126.98146057128906 + ], + "size": [ + 150.93612670898438, + 46 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 101 + }, + { + "name": "vae", + "type": "VAE", + "link": 83 + } + ], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 122, + 124 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.32", + "Node name for S&R": "VAEDecodeAudio", + "widget_ue_connectable": {} + }, + "widgets_values": [] + }, + { + "id": 58, + "type": "Knob", + "pos": [ + 1660.9822998046875, + 11.48207950592041 + ], + "size": [ + 315, + 122 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 122 + } + ], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 123, + 130 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "6143b478e707c55a12014c5d67c46ec947bb1451", + "widget_ue_connectable": {}, + "Node name for S&R": "Knob" + }, + "widgets_values": [ + 0.4900000000000001, + 0.020000000000000004 + ] + }, + { + "id": 60, + "type": "PreviewAudio", + "pos": [ + 1674.4638671875, + 288.9170837402344 + ], + "size": [ + 315, + 88 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 124 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.34", + "widget_ue_connectable": {}, + "Node name for S&R": "PreviewAudio" + }, + "widgets_values": [] + }, + { + "id": 59, + "type": "PreviewAudio", + "pos": [ + 1689.748291015625, + -283.91436767578125 + ], + "size": [ + 315, + 88 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 123 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.34", + "widget_ue_connectable": {}, + "Node name for S&R": "PreviewAudio" + }, + "widgets_values": [] + }, + { + "id": 61, + "type": "DownloadOpenUnmixModel", + "pos": [ + 2264.053955078125, + -94.053955078125 + ], + "size": [ + 294, + 58 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "OPEN_UNMIX_MODEL", + "type": "OPEN_UNMIX_MODEL", + "links": [ + 125 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "6143b478e707c55a12014c5d67c46ec947bb1451", + "widget_ue_connectable": {}, + "Node name for S&R": "DownloadOpenUnmixModel" + }, + "widgets_values": [ + "umxl" + ] + }, + { + "id": 64, + "type": "PreviewAudio", + "pos": [ + 2733.503662109375, + -10.205208778381348 + ], + "size": [ + 291.92596435546875, + 88 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 128 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.34", + "widget_ue_connectable": {}, + "Node name for S&R": "PreviewAudio" + }, + "widgets_values": [] + }, + { + "id": 63, + "type": "PreviewAudio", + "pos": [ + 2744.18701171875, + -170.14122009277344 + ], + "size": [ + 309.4623107910156, + 89.84590911865234 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 127 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.34", + "widget_ue_connectable": {}, + "Node name for S&R": "PreviewAudio" + }, + "widgets_values": [] + }, + { + "id": 57, + "type": "AudioSeparatorSimple", + "pos": [ + 2257.29736328125, + 21.89219093322754 + ], + "size": [ + 315, + 106 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "OPEN_UNMIX_MODEL", + "link": 125 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 130 + } + ], + "outputs": [ + { + "name": "audio", + "type": "AUDIO", + "links": [ + 137 + ] + }, + { + "name": "drums_audio", + "type": "AUDIO", + "links": [ + 127, + 132 + ] + }, + { + "name": "vocals_audio", + "type": "AUDIO", + "links": null + }, + { + "name": "bass_audio", + "type": "AUDIO", + "links": [ + 128 + ] + }, + { + "name": "other_audio", + "type": "AUDIO", + "links": [ + 129 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "6143b478e707c55a12014c5d67c46ec947bb1451", + "widget_ue_connectable": {}, + "Node name for S&R": "AudioSeparatorSimple" + } + }, + { + "id": 72, + "type": "QuickShapeMask", + "pos": [ + 3657.35791015625, + -491.16790771484375 + ], + "size": [ + 315, + 226 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 134 + ] + } + ], + "properties": { + "aux_id": "ryanontheinside/ComfyUI_RealtimeNodes", + "ver": "91b155a115c2e34f95b9e8e4180ef4d36fe62c87", + "widget_ue_connectable": {}, + "Node name for S&R": "QuickShapeMask" + }, + "widgets_values": [ + "circle", + 64, + 64, + 256, + 256, + 512, + 512, + 1 + ] + }, + { + "id": 71, + "type": "FlexMaskMorph", + "pos": [ + 4160.71142578125, + -210.05096435546875 + ], + "size": [ + 315, + 318 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "masks", + "type": "MASK", + "link": 134 + }, + { + "name": "opt_feature", + "shape": 7, + "type": "FEATURE", + "link": 133 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 135 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "6143b478e707c55a12014c5d67c46ec947bb1451", + "widget_ue_connectable": {}, + "Node name for S&R": "FlexMaskMorph" + }, + "widgets_values": [ + 1, + 0, + "max_kernel_size", + "relative", + false, + 0, + 0, + 1, + "dilate", + 5, + 10 + ] + }, + { + "id": 69, + "type": "AudioFeatureExtractor", + "pos": [ + 3180.29443359375, + -188.12203979492188 + ], + "size": [ + 415.8000183105469, + 174 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 132 + } + ], + "outputs": [ + { + "name": "feature", + "type": "FEATURE", + "links": [ + 133 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "6143b478e707c55a12014c5d67c46ec947bb1451", + "widget_ue_connectable": {}, + "Node name for S&R": "AudioFeatureExtractor" + }, + "widgets_values": [ + "amplitude_envelope", + 30, + 0, + 512, + 512 + ] + } + ], + "links": [ + [ + 23, + 17, + 0, + 3, + 3, + "LATENT" + ], + [ + 80, + 40, + 1, + 14, + 0, + "CLIP" + ], + [ + 83, + 40, + 2, + 18, + 1, + "VAE" + ], + [ + 101, + 3, + 0, + 18, + 0, + "LATENT" + ], + [ + 108, + 14, + 0, + 44, + 0, + "CONDITIONING" + ], + [ + 109, + 44, + 0, + 3, + 2, + "CONDITIONING" + ], + [ + 110, + 14, + 0, + 3, + 1, + "CONDITIONING" + ], + [ + 117, + 55, + 0, + 54, + 0, + "MODEL" + ], + [ + 118, + 56, + 0, + 54, + 1, + "LATENT_OPERATION" + ], + [ + 119, + 54, + 0, + 3, + 0, + "MODEL" + ], + [ + 120, + 40, + 0, + 55, + 0, + "MODEL" + ], + [ + 122, + 18, + 0, + 58, + 0, + "AUDIO" + ], + [ + 123, + 58, + 0, + 59, + 0, + "AUDIO" + ], + [ + 124, + 18, + 0, + 60, + 0, + "AUDIO" + ], + [ + 125, + 61, + 0, + 57, + 0, + "OPEN_UNMIX_MODEL" + ], + [ + 127, + 57, + 1, + 63, + 0, + "AUDIO" + ], + [ + 128, + 57, + 3, + 64, + 0, + "AUDIO" + ], + [ + 129, + 57, + 4, + 65, + 0, + "AUDIO" + ], + [ + 130, + 58, + 0, + 57, + 1, + "AUDIO" + ], + [ + 132, + 57, + 1, + 69, + 0, + "AUDIO" + ], + [ + 133, + 69, + 0, + 71, + 1, + "FEATURE" + ], + [ + 134, + 72, + 0, + 71, + 0, + "MASK" + ], + [ + 135, + 71, + 0, + 70, + 0, + "MASK" + ], + [ + 136, + 70, + 0, + 73, + 0, + "IMAGE" + ], + [ + 137, + 57, + 0, + 73, + 1, + "AUDIO" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load model here", + "bounding": [ + 169.5068359375, + 14.167394638061523, + 395, + 181.60000610351562 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.34522712143931317, + "offset": [ + -7.99354499643681, + 940.753420261988 + ] + }, + "frontendVersion": "1.17.11", + "ue_links": [], + "links_added_by_ue": [], + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/nodes/audio/midi_nodes.py b/nodes/audio/midi_nodes.py index 5b2b373..e6029d1 100644 --- a/nodes/audio/midi_nodes.py +++ b/nodes/audio/midi_nodes.py @@ -388,6 +388,9 @@ class MIDILoader: # Convert musical measures to ticks start_tick, end_tick = convert_measures_to_ticks_range(midi_data, start_measure, start_beat, end_measure, end_beat) + # The total duration in ticks is EXACTLY end_tick - start_tick + total_tick_duration = end_tick - start_tick + # Create a new MIDI file with the selected measures trimmed_midi = mido.MidiFile(ticks_per_beat=midi_data.ticks_per_beat) @@ -395,73 +398,89 @@ class MIDILoader: new_track = mido.MidiTrack() trimmed_midi.tracks.append(new_track) - # Copy important metadata messages + # Copy important metadata messages first for msg in track: if not hasattr(msg, 'time'): if msg.type in ['track_name', 'time_signature', 'key_signature', 'set_tempo']: new_track.append(msg.copy()) - # First pass: collect all note_on events within range and their corresponding note_off events + # Collect events within the time range note_events = [] active_notes = {} # {note_num: tick_time_started} + notes_active_before_range = {} # Track notes that started before our range current_tick = 0 - last_event_tick = 0 # To track the last event we need to include + # First pass: collect all note on/off events, tracking notes that cross boundaries for msg in track: if not hasattr(msg, 'time'): continue current_tick += msg.time + # Handle note_on events if msg.type == 'note_on' and msg.velocity > 0: - # Note being turned on - if current_tick >= start_tick and current_tick <= end_tick: + # Note starting before our range + if current_tick < start_tick: + notes_active_before_range[msg.note] = current_tick + # Note starting within our range + elif current_tick <= end_tick: active_notes[msg.note] = current_tick note_events.append((current_tick, msg.copy())) - last_event_tick = max(last_event_tick, current_tick) - elif (msg.type == 'note_off' or (msg.type == 'note_on' and msg.velocity == 0)): - # Note being turned off - if msg.note in active_notes: - # This is a note that started within our range - note_events.append((current_tick, msg.copy())) + # Handle note_off events + elif msg.type == 'note_off' or (msg.type == 'note_on' and msg.velocity == 0): + # Handle notes that started before our range but end within it + if msg.note in notes_active_before_range: + if current_tick >= start_tick and current_tick <= end_tick: + # Create a new note_on at the start boundary + new_note_on = mido.Message('note_on', note=msg.note, velocity=64, time=0) + note_events.append((start_tick, new_note_on)) + # Add the note_off event + note_events.append((current_tick, msg.copy())) + del notes_active_before_range[msg.note] + + # Handle notes that started within our range + elif msg.note in active_notes: + # If note ends within our range, add the note_off event + if current_tick <= end_tick: + note_events.append((current_tick, msg.copy())) + # If note would end outside our range, add a note_off at the end boundary + else: + new_note_off = mido.Message('note_off', note=msg.note, velocity=0, time=0) + note_events.append((end_tick, new_note_off)) del active_notes[msg.note] - last_event_tick = max(last_event_tick, current_tick) - # Also include other MIDI events within the time range + # Include other MIDI events within the time range elif current_tick >= start_tick and current_tick <= end_tick: if msg.type in ['set_tempo', 'control_change', 'program_change', 'pitchwheel']: note_events.append((current_tick, msg.copy())) - last_event_tick = max(last_event_tick, current_tick) + + # Add note_off events at end boundary for any notes still active at the end + for note in active_notes: + new_note_off = mido.Message('note_off', note=note, velocity=0, time=0) + note_events.append((end_tick, new_note_off)) # Sort events by tick time note_events.sort(key=lambda x: x[0]) - # Second pass: add events to the new track with adjusted timing - if note_events: - prev_tick = start_tick - total_ticks = last_event_tick - start_tick + # Add events to the new track with adjusted timing + prev_tick = start_tick + + for tick, msg in note_events: + # Adjust timing relative to previous event + adjusted_msg = msg.copy() + adjusted_msg.time = tick - prev_tick + prev_tick = tick - for tick, msg in note_events: - # Adjust timing relative to previous event - adjusted_msg = msg.copy() - adjusted_msg.time = tick - prev_tick - prev_tick = tick - - new_track.append(adjusted_msg) - - # Add end of track marker with correct timing - end_track = mido.MetaMessage('end_of_track') - - # If there are any active notes at the end boundary, make sure we have enough time - if active_notes: - remaining_ticks = end_tick - last_event_tick - if remaining_ticks > 0: - end_track.time = remaining_ticks - else: - end_track.time = 0 - - new_track.append(end_track) + new_track.append(adjusted_msg) + + # Calculate how many ticks remain until the exact end_tick + remaining_ticks = start_tick + total_tick_duration - prev_tick + + # Add end of track marker at exactly the right position + end_track = mido.MetaMessage('end_of_track') + end_track.time = max(0, remaining_ticks) + new_track.append(end_track) midi_data = trimmed_midi diff --git a/nodes/flex/feature_modulation.py b/nodes/flex/feature_modulation.py index 6e6aa9e..550ce63 100644 --- a/nodes/flex/feature_modulation.py +++ b/nodes/flex/feature_modulation.py @@ -789,7 +789,7 @@ class FeatureInterpolateMulti(FeatureModulationBase): "feature2": ("FEATURE",), "feature3": ("FEATURE",), "min_aggregate_value": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}), - "transition_frames": ("INT", {"default": 30, "min": 1, "max": 100, "step": 1}), + "transition_frames": ("INT", {"default": 5, "min": 1, "max": 100, "step": 1}), **super().INPUT_TYPES()["required"], } }