From 7705fe14139e58602cc2135da96ef10236cfdd79 Mon Sep 17 00:00:00 2001 From: Fillip Date: Tue, 4 Aug 2026 21:55:37 -0700 Subject: [PATCH] Make beat grid directly editable --- README.md | 2 +- nodes/audio/FL_Audio_Beat_Prompt_Schedule.py | 23 ++- nodes/audio/audio_timeline.py | 41 +++- pyproject.toml | 2 +- tests/test_audio_beat_prompt_schedule.py | 22 +++ tests/test_audio_timeline.py | 77 ++++++++ .../audio/FL_Audio_Beat_Prompt_Schedule.js | 175 +++++++++++++++--- 7 files changed, 309 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 9ea1a56..2b695c9 100644 --- a/README.md +++ b/README.md @@ -325,7 +325,7 @@ Fill-Nodes is a versatile collection of custom nodes for ComfyUI that extends fu |------|-------------| | `FL_Audio_BPM_Analyzer` | Analyzes audio using librosa to detect BPM and beat positions, with options for two BPM calculation methods (beat_intervals or onset_strength), beat offset adjustment, and automatic beat filling to cover the entire audio duration. Outputs beat positions as JSON with visualization. | | `FL_Audio_Beat_Prompt_Envelope` | Turns exact detected beats into attack/hold/release prompt-mask pulses for compatible FL diffusion video nodes. Supports beat stride, phase, response curve, and on-beat dominance above normal conditioning weight. | -| `FL_Audio_Beat_Prompt_Schedule` | Integrated audio and prompt sequencer with native upload, transport, source trimming, immediate waveform display, queue-free beat/onset/drum analysis, optional cached stem analysis, and draggable frame-native prompt clips. Outputs the cropped master audio plus beat and drum JSON for the rest of the FL audio ecosystem. | +| `FL_Audio_Beat_Prompt_Schedule` | Integrated audio and prompt sequencer with native upload, transport, source trimming, immediate waveform display, queue-free beat/onset/drum analysis, optional cached stem analysis, a directly draggable beat grid with every-beat and subdivision density controls, and frame-native prompt clips. Outputs the cropped master audio plus beat and drum JSON for the rest of the FL audio ecosystem. | | `FL_Audio_Beat_Visualizer` | Generates video frames that visualize beat patterns by either alternating between black/white on beat changes or cycling through provided images, with configurable frame dimensions and starting color. | | `FL_AudioFrameCalculator` | Given an AUDIO and a target FPS, returns the integer frame count needed to cover the audio's duration (`ceil(duration * fps)`). Useful upstream of video-generation nodes that need an exact frame budget. | | `FL_Audio_Crop` | Crops audio waveforms to specified start and end times using MM:SS or seconds format, with automatic clamping to valid audio boundaries. | diff --git a/nodes/audio/FL_Audio_Beat_Prompt_Schedule.py b/nodes/audio/FL_Audio_Beat_Prompt_Schedule.py index e0e191c..75ad2fc 100644 --- a/nodes/audio/FL_Audio_Beat_Prompt_Schedule.py +++ b/nodes/audio/FL_Audio_Beat_Prompt_Schedule.py @@ -84,6 +84,8 @@ def _load_beat_data(beat_positions): base_detected_beat_times = [] return { "bpm": bpm, + "grid_bpm": data.get("grid_bpm", bpm), + "beat_grid_density": data.get("beat_grid_density", "every_beat"), "beat_times": beat_times, "base_beat_times": base_beat_times, "audio_duration": duration, @@ -498,6 +500,16 @@ class FL_Audio_Beat_Prompt_Schedule(io.ComfyNode): "available after separation finishes." ), ), + io.Combo.Input( + "beat_grid_density", + display_name="beat grid density", + options=["every_2_beats", "every_beat", "half_beat"], + default="every_beat", + tooltip=( + "Backing value for the sequencer's Grid control. Every beat uses the " + "detected tempo; half-beat adds subdivisions." + ), + ), ], outputs=[ FLPromptSchedule.Output( @@ -548,6 +560,7 @@ class FL_Audio_Beat_Prompt_Schedule(io.ComfyNode): half_time=False, beat_offset_ms=0, analysis_source="mix", + beat_grid_density="every_beat", ): internal_analysis = None cropped_audio = None @@ -561,11 +574,17 @@ class FL_Audio_Beat_Prompt_Schedule(io.ComfyNode): half_time, beat_offset_ms, analysis_source, + beat_grid_density, ) if beat_positions: beat_payload = _parse_beat_payload(beat_positions) _load_beat_data(beat_payload) - beat_payload = apply_beat_offset(beat_payload, fps, beat_offset_ms) + beat_payload = apply_beat_offset( + beat_payload, + fps, + beat_offset_ms, + beat_grid_density, + ) beat_positions = json.dumps(beat_payload, separators=(",", ":")) beat_data = _load_beat_data(beat_payload) if internal_analysis is not None: @@ -626,6 +645,8 @@ class FL_Audio_Beat_Prompt_Schedule(io.ComfyNode): } ui_payload = { "bpm": beat_data["bpm"], + "grid_bpm": beat_data["grid_bpm"], + "beat_grid_density": beat_data["beat_grid_density"], "beat_times": beat_times, "base_beat_times": beat_data["base_beat_times"], "detected_beat_times": ( diff --git a/nodes/audio/audio_timeline.py b/nodes/audio/audio_timeline.py index dcdc229..a9d1bf0 100644 --- a/nodes/audio/audio_timeline.py +++ b/nodes/audio/audio_timeline.py @@ -202,11 +202,13 @@ def _detect_drums(waveform, sample_rate, onset_frames, onset_times): } -def apply_beat_offset(analysis, fps, beat_offset_ms=0): +def apply_beat_offset(analysis, fps, beat_offset_ms=0, beat_grid_density="every_beat"): if not math.isfinite(fps) or fps <= 0: raise ValueError("FPS must be greater than zero.") if not math.isfinite(beat_offset_ms): raise ValueError("Beat offset must be a finite number.") + if beat_grid_density not in {"every_2_beats", "every_beat", "half_beat"}: + raise ValueError(f"Unknown beat grid density: {beat_grid_density}") duration = float(analysis["audio_duration"]) offset = beat_offset_ms / 1000.0 @@ -216,6 +218,20 @@ def apply_beat_offset(analysis, fps, beat_offset_ms=0): analysis.get("detected_beat_times", []), ) + if beat_grid_density == "every_2_beats": + grid_beat_times = base_beat_times[::2] + grid_bpm_scale = 0.5 + elif beat_grid_density == "half_beat": + grid_beat_times = [] + for index, beat_time in enumerate(base_beat_times): + grid_beat_times.append(beat_time) + if index + 1 < len(base_beat_times): + grid_beat_times.append((beat_time + base_beat_times[index + 1]) / 2.0) + grid_bpm_scale = 2.0 + else: + grid_beat_times = base_beat_times + grid_bpm_scale = 1.0 + def shifted(values): if not values: return [] @@ -225,7 +241,7 @@ def apply_beat_offset(analysis, fps, beat_offset_ms=0): result = dict(analysis) result["base_beat_times"] = list(base_beat_times) result["base_detected_beat_times"] = list(base_detected_beat_times) - result["beat_times"] = shifted(base_beat_times) + result["beat_times"] = shifted(grid_beat_times) result["detected_beat_times"] = shifted(base_detected_beat_times) result["beat_frames"] = [round(value * fps) for value in result["beat_times"]] result["detected_beat_frames"] = [ @@ -233,10 +249,19 @@ def apply_beat_offset(analysis, fps, beat_offset_ms=0): ] result["num_beats"] = len(result["beat_times"]) result["beat_offset_ms"] = int(round(beat_offset_ms)) + result["beat_grid_density"] = beat_grid_density + result["grid_bpm"] = float(result.get("bpm", 0.0)) * grid_bpm_scale return result -def analyze_audio(audio, fps, bpm_method="beat_intervals", half_time=False, beat_offset_ms=0): +def analyze_audio( + audio, + fps, + bpm_method="beat_intervals", + half_time=False, + beat_offset_ms=0, + beat_grid_density="every_beat", +): if bpm_method not in {"beat_intervals", "onset_strength"}: raise ValueError(f"Unknown BPM method: {bpm_method}") waveform = mono_numpy(audio) @@ -300,7 +325,7 @@ def analyze_audio(audio, fps, bpm_method="beat_intervals", half_time=False, beat "drum_times": drum_times, "waveform_preview": waveform_preview(waveform, sample_rate), } - return apply_beat_offset(analysis, fps, beat_offset_ms) + return apply_beat_offset(analysis, fps, beat_offset_ms, beat_grid_density) def analysis_cache_key(path, fps, trim_start_frame, length_frames, bpm_method, half_time, analysis_source): @@ -333,6 +358,7 @@ def analyze_audio_file( half_time=False, beat_offset_ms=0, analysis_source="mix", + beat_grid_density="every_beat", ): path = resolve_audio_path(filename) _, master_audio = load_audio_file(filename) @@ -370,4 +396,9 @@ def analyze_audio_file( temporary_path = cache_path.with_suffix(".tmp") temporary_path.write_text(json.dumps(analysis, separators=(",", ":")), encoding="utf-8") temporary_path.replace(cache_path) - return apply_beat_offset(analysis, fps, beat_offset_ms), cropped_audio + return apply_beat_offset( + analysis, + fps, + beat_offset_ms, + beat_grid_density, + ), cropped_audio diff --git a/pyproject.toml b/pyproject.toml index d9b2948..4a04298 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui_fill-nodes" description = "Fill-Nodes is a versatile collection of custom nodes for ComfyUI that extends functionality across multiple domains. Features include advanced image processing (pixelation, slicing, masking), visual effects generation (glitch, halftone, pixel art), comprehensive file handling (PDF creation/extraction, Google Drive integration), AI model interfaces (GPT, DALL-E, Hugging Face), utility nodes for workflow enhancement, and specialized tools for video processing, captioning, and batch operations. The pack provides both practical workflow solutions and creative tools within a unified node collection." -version = "2.17.0" +version = "2.18.0" license = {file = "LICENSE"} dependencies = ["librosa", "sounddevice", "glitch_this", "PyOpenGL", "glfw", "scipy>=1.13.1", "requests", "aiohttp", "moviepy", "matplotlib", "reportlab", "openai", "PyPDF2", "pdf2image", "PyMuPDF", "reportlab", "PyPDF2", "ollama", "kornia", "opencv-python", "gdown", "open_clip_torch", "google-genai"] diff --git a/tests/test_audio_beat_prompt_schedule.py b/tests/test_audio_beat_prompt_schedule.py index 102c66a..4b4ef60 100644 --- a/tests/test_audio_beat_prompt_schedule.py +++ b/tests/test_audio_beat_prompt_schedule.py @@ -274,6 +274,7 @@ class BeatPromptScheduleTests(unittest.TestCase): "half_time", "beat_offset_ms", "analysis_source", + "beat_grid_density", ], ) self.assertEqual( @@ -373,6 +374,27 @@ class BeatPromptScheduleTests(unittest.TestCase): self.assertEqual(payload["beat_times"], [0.2, 0.7, 1.3, 2.0]) self.assertEqual(payload["beat_offset_ms"], 100) + def test_scheduler_density_controls_external_grid_and_output(self): + output = schedule.FL_Audio_Beat_Prompt_Schedule.execute( + beat_positions=beat_json(), + timeline="[0 - 24]\nCamera pulse.", + default_fade_in=0.0, + default_fade_out=0.0, + curve="linear", + fps=24.0, + sequence_duration=24, + beat_grid_density="every_2_beats", + ) + + effective = json.loads(output.result[5]) + payload = output.ui["fl_prompt_sequencer"][0] + self.assertEqual(effective["beat_times"], [0.1, 1.2]) + self.assertEqual(effective["base_beat_times"], [0.1, 0.6, 1.2, 1.9]) + self.assertEqual(effective["beat_grid_density"], "every_2_beats") + self.assertEqual(effective["grid_bpm"], 60.0) + self.assertEqual(payload["beat_times"], [0.1, 1.2]) + self.assertEqual(payload["beat_grid_density"], "every_2_beats") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_audio_timeline.py b/tests/test_audio_timeline.py index 6e8841a..c22a2e2 100644 --- a/tests/test_audio_timeline.py +++ b/tests/test_audio_timeline.py @@ -53,6 +53,51 @@ class AudioTimelineTests(unittest.TestCase): self.assertEqual(shifted["beat_times"], [0.0, 0.6000000000000001]) self.assertEqual(shifted["base_beat_times"], [0.0, 0.1, 0.8]) + def test_beat_grid_density_uses_native_beats_as_its_source(self): + analysis = { + "bpm": 120.0, + "audio_duration": 2.0, + "beat_times": [0.0, 0.5, 1.0, 1.5], + "detected_beat_times": [0.05, 0.55], + } + + every_two = timeline.apply_beat_offset( + analysis, + fps=24.0, + beat_grid_density="every_2_beats", + ) + every_beat = timeline.apply_beat_offset( + every_two, + fps=24.0, + beat_grid_density="every_beat", + ) + subdivisions = timeline.apply_beat_offset( + analysis, + fps=24.0, + beat_offset_ms=100, + beat_grid_density="half_beat", + ) + + self.assertEqual(every_two["beat_times"], [0.0, 1.0]) + self.assertEqual(every_two["grid_bpm"], 60.0) + self.assertEqual(every_beat["beat_times"], [0.0, 0.5, 1.0, 1.5]) + self.assertEqual(every_beat["grid_bpm"], 120.0) + self.assertEqual( + subdivisions["beat_times"], + [0.1, 0.35, 0.6, 0.85, 1.1, 1.35, 1.6], + ) + self.assertEqual(subdivisions["detected_beat_times"], [0.15000000000000002, 0.65]) + self.assertEqual(subdivisions["grid_bpm"], 240.0) + self.assertEqual(subdivisions["beat_grid_density"], "half_beat") + + def test_unknown_beat_grid_density_is_rejected(self): + with self.assertRaisesRegex(ValueError, "Unknown beat grid density"): + timeline.apply_beat_offset( + {"audio_duration": 1.0, "beat_times": [0.0]}, + fps=24.0, + beat_grid_density="bars", + ) + def test_crop_uses_video_frames_for_sample_boundaries(self): audio = { "waveform": torch.arange(0, 96000, dtype=torch.float32).reshape(1, 1, -1), @@ -160,6 +205,38 @@ class AudioTimelineTests(unittest.TestCase): self.assertEqual(shifted["beat_times"], [0.2, 0.7]) self.assertEqual(first["cache_key"], shifted["cache_key"]) + def test_density_changes_reuse_the_base_analysis_cache(self): + master = {"waveform": torch.ones(1, 1, 48000), "sample_rate": 48000} + analysis = { + "bpm": 120.0, + "beat_times": [0.0, 0.5, 1.0], + "detected_beat_times": [], + "onset_times": [], + "audio_duration": 1.0, + "waveform_preview": {"version": 1, "duration": 1.0, "scale": 32767, "peaks": [0, 1]}, + "drum_times": {}, + } + with tempfile.TemporaryDirectory() as directory: + cache_path = pathlib.Path(directory) / "analysis.json" + with ( + mock.patch.object(timeline, "resolve_audio_path", return_value=pathlib.Path("song.wav")), + mock.patch.object(timeline, "load_audio_file", return_value=(pathlib.Path("song.wav"), master)), + mock.patch.object(timeline, "analysis_cache_key", return_value="key"), + mock.patch.object(timeline, "_cache_path", return_value=cache_path), + mock.patch.object(timeline, "analyze_audio", return_value=analysis) as analyze, + ): + every_beat, _ = timeline.analyze_audio_file("song.wav", fps=24.0) + every_two, _ = timeline.analyze_audio_file( + "song.wav", + fps=24.0, + beat_grid_density="every_2_beats", + ) + + self.assertEqual(analyze.call_count, 1) + self.assertEqual(every_beat["beat_times"], [0.0, 0.5, 1.0]) + self.assertEqual(every_two["beat_times"], [0.0, 1.0]) + self.assertEqual(every_beat["cache_key"], every_two["cache_key"]) + if __name__ == "__main__": unittest.main() diff --git a/web/nodes/audio/FL_Audio_Beat_Prompt_Schedule.js b/web/nodes/audio/FL_Audio_Beat_Prompt_Schedule.js index ce18601..9828000 100644 --- a/web/nodes/audio/FL_Audio_Beat_Prompt_Schedule.js +++ b/web/nodes/audio/FL_Audio_Beat_Prompt_Schedule.js @@ -5,11 +5,18 @@ const STYLE_ID = "fl-beat-prompt-sequencer-styles"; const INSTANCES = new Map(); const HEADER_RE = /^\s*\[\s*([0-9]+(?:\.[0-9]+)?)\s*-\s*([0-9]+(?:\.[0-9]+)?)(?:\s*\|\s*(.*?))?\s*\]\s*$/; const EPSILON = 1e-6; -const FORMAT_VERSION = 4; +const FORMAT_VERSION = 5; const MIN_NODE_WIDTH = 680; const MIN_NODE_HEIGHT = 1100; const TIMELINE_LEFT = 42; const TIMELINE_RIGHT = 12; +const BEAT_LANE_TOP = 31; +const BEAT_LANE_BOTTOM = 58; +const GRID_DENSITY_LABELS = { + every_2_beats: "Every 2 beats", + every_beat: "Every beat", + half_beat: "Half-beat", +}; const STYLES = ` .flbps-root { @@ -510,6 +517,11 @@ class BeatPromptSequencer { return clamp(Math.round(finiteNumber(this.widgets.beatOffset?.value, 0)), -1000, 1000); } + beatGridDensity() { + const value = this.widgets.beatGridDensity?.value; + return value in GRID_DENSITY_LABELS ? value : "every_beat"; + } + configuredFrameCount() { return Math.max(0, Math.round(finiteNumber(this.widgets.sequenceDuration?.value, 0))); } @@ -553,6 +565,14 @@ class BeatPromptSequencer { +