nodes/filliptm-ComfyUI_Fill-Nodes
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix audio crop beat alignment

Browse Source
This commit is contained in:
Fillip
2026-08-08 01:40:52 -07:00
parent fcbf94d507
commit ca7709acbf
7 changed files with 958 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nodes/audio/FL_Audio_Beat_Prompt_Schedule.py
+3
View File
@@ -857,6 +857,9 @@ class FL_Audio_Beat_Prompt_Schedule(io.ComfyNode):
if internal_analysis is not None:
ui_payload["audio_file"] = internal_analysis.get("audio_file", audio_file)
ui_payload["cache_key"] = internal_analysis.get("cache_key", "")
source_analysis = internal_analysis.get("source_analysis")
if isinstance(source_analysis, dict) and "beat_times" in source_analysis:
ui_payload["source_analysis"] = source_analysis
waveform = (
internal_analysis["waveform_preview"]
if internal_analysis is not None
nodes/audio/audio_timeline.py
+174 -57
View File
@@ -14,8 +14,9 @@ from .audio_separation import load_cached_stem
from .beat_this_detector import MODEL_FPS, MODEL_SHA256, analyze_beats as analyze_beat_this
ANALYSIS_VERSION = 6
ANALYSIS_VERSION = 7
DETECTOR_VERSION = f"beat-this-final0-{MODEL_SHA256[:12]}"
SOURCE_ANALYSIS_VERSION = 1
_WAVEFORM_BUCKETS_PER_SECOND = 60
_MAX_WAVEFORM_BUCKETS = 8192
_WAVEFORM_SCALE = 32767
@@ -114,6 +115,27 @@ def waveform_preview(waveform, sample_rate):
}
def crop_waveform_preview(preview, start, duration):
if not isinstance(preview, dict) or preview.get("version") != 1 or duration <= 0:
return None
peaks = preview.get("peaks")
source_duration = preview.get("duration")
if not isinstance(peaks, list) or len(peaks) < 2 or len(peaks) % 2 or not source_duration:
return None
bucket_count = len(peaks) // 2
start_bucket = min(bucket_count - 1, max(0, math.floor(start / source_duration * bucket_count)))
end_bucket = min(
bucket_count,
max(start_bucket + 1, math.ceil((start + duration) / source_duration * bucket_count)),
)
return {
"version": 1,
"duration": float(duration),
"scale": preview["scale"],
"peaks": peaks[start_bucket * 2:end_bucket * 2],
}
def _detect_drums(waveform, sample_rate, onset_frames, onset_times):
stft_mag = np.abs(librosa.stft(waveform))
frequencies = librosa.fft_frequencies(sr=sample_rate)
@@ -275,15 +297,7 @@ def apply_beat_offset(analysis, fps, beat_offset_ms=0, beat_grid_density="every_
return result
def analyze_audio(
audio,
fps,
half_time=False,
beat_offset_ms=0,
beat_grid_density="every_beat",
detect_beats=True,
beat_audio=None,
):
def analyze_audio(audio, detect_beats=True, beat_audio=None):
waveform = mono_numpy(audio)
sample_rate = int(audio["sample_rate"])
duration = len(waveform) / sample_rate
@@ -302,7 +316,6 @@ def analyze_audio(
"version": ANALYSIS_VERSION,
"detector_version": DETECTOR_VERSION,
"onset_times": onsets,
"onset_frames": [round(value * fps) for value in onsets],
"sample_rate": sample_rate,
"audio_duration": float(duration),
"drum_times": drum_times,
@@ -320,20 +333,8 @@ def analyze_audio(
downbeat_confidences = np.asarray(
detected["downbeat_confidences"], dtype=np.float64
)
if half_time:
retained = np.arange(0, len(beat_times), 2)
retained_indices = set(retained.tolist())
downbeat_retained = []
for index, downbeat in enumerate(downbeat_times):
nearest = int(np.argmin(np.abs(beat_times - downbeat)))
if nearest in retained_indices:
downbeat_retained.append(index)
beat_times = beat_times[retained]
beat_confidences = beat_confidences[retained]
downbeat_times = downbeat_times[downbeat_retained]
downbeat_confidences = downbeat_confidences[downbeat_retained]
if len(beat_times) < 2:
raise ValueError("Beat This must detect at least two beats in the selected audio range.")
raise ValueError("Beat This must detect at least two beats in the source audio.")
interval = float(np.median(np.diff(beat_times)))
analysis.update({
@@ -348,32 +349,141 @@ def analyze_audio(
"detected_beat_confidences": beat_confidences.tolist(),
"base_detected_downbeat_confidences": downbeat_confidences.tolist(),
"detected_downbeat_confidences": downbeat_confidences.tolist(),
"beat_frames": [round(value * fps) for value in beat_times],
"downbeat_frames": [round(value * fps) for value in downbeat_times],
"detected_beat_frames": [round(value * fps) for value in beat_times],
"num_beats": len(beat_times),
"num_downbeats": len(downbeat_times),
"detector": detected["detector"],
})
return apply_beat_offset(analysis, fps, beat_offset_ms, beat_grid_density)
return analysis
def apply_half_time(analysis, half_time):
result = dict(analysis)
if not half_time or "beat_times" not in result:
return result
beat_times = list(result["beat_times"])
beat_confidences = list(result.get("detected_beat_confidences", []))
retained = list(range(0, len(beat_times), 2))
if len(retained) < 2:
raise ValueError("Beat This must detect at least two beats after half-time filtering.")
retained_indices = set(retained)
downbeat_times = list(result.get("downbeat_times", []))
downbeat_confidences = list(result.get("detected_downbeat_confidences", []))
retained_downbeats = []
retained_downbeat_confidences = []
for index, downbeat in enumerate(downbeat_times):
nearest = min(range(len(beat_times)), key=lambda position: abs(beat_times[position] - downbeat))
if nearest in retained_indices:
retained_downbeats.append(downbeat)
if index < len(downbeat_confidences):
retained_downbeat_confidences.append(downbeat_confidences[index])
retained_beats = [beat_times[index] for index in retained]
retained_beat_confidences = [
beat_confidences[index] for index in retained if index < len(beat_confidences)
]
interval = float(np.median(np.diff(retained_beats)))
result.update({
"bpm": 60.0 / interval,
"base_grid_interval_seconds": interval,
"beat_times": retained_beats,
"downbeat_times": retained_downbeats,
"detected_beat_times": retained_beats,
"detected_downbeat_times": retained_downbeats,
"base_detected_beat_confidences": retained_beat_confidences,
"detected_beat_confidences": retained_beat_confidences,
"base_detected_downbeat_confidences": retained_downbeat_confidences,
"detected_downbeat_confidences": retained_downbeat_confidences,
"num_beats": len(retained_beats),
"num_downbeats": len(retained_downbeats),
})
return result
def _crop_times(values, start, end):
return [float(value - start) for value in values if start <= value < end]
def _crop_times_with_values(times, values, start, end):
cropped_times = []
cropped_values = []
for index, value in enumerate(times):
if start <= value < end:
cropped_times.append(float(value - start))
if index < len(values):
cropped_values.append(values[index])
return cropped_times, cropped_values
def project_analysis(analysis, crop, fps):
start = float(crop["source_start"])
duration = float(crop["audio_duration"])
end = start + duration
result = dict(analysis)
for time_key, confidence_key in (
("base_detected_beat_times", "base_detected_beat_confidences"),
("detected_beat_times", "detected_beat_confidences"),
("base_detected_downbeat_times", "base_detected_downbeat_confidences"),
("detected_downbeat_times", "detected_downbeat_confidences"),
):
times, confidences = _crop_times_with_values(
analysis.get(time_key, []),
analysis.get(confidence_key, []),
start,
end,
)
result[time_key] = times
result[confidence_key] = confidences
for key in (
"base_beat_times",
"beat_times",
"base_downbeat_times",
"downbeat_times",
"onset_times",
):
if key in analysis:
result[key] = _crop_times(analysis[key], start, end)
drums = dict(analysis.get("drum_times", {}))
for key in ("kick_times", "snare_times", "hihat_times"):
drums[key] = _crop_times(drums.get(key, []), start, end)
drums.update({
"duration": duration,
"total_kicks": len(drums["kick_times"]),
"total_snares": len(drums["snare_times"]),
"total_hihats": len(drums["hihat_times"]),
})
result.update(crop)
result.update({
"beat_frames": [round(value * fps) for value in result.get("beat_times", [])],
"downbeat_frames": [round(value * fps) for value in result.get("downbeat_times", [])],
"detected_beat_frames": [
round(value * fps) for value in result.get("detected_beat_times", [])
],
"onset_frames": [round(value * fps) for value in result.get("onset_times", [])],
"drum_times": drums,
"num_beats": len(result.get("beat_times", [])),
"num_downbeats": len(result.get("downbeat_times", [])),
"waveform_preview": crop_waveform_preview(
analysis.get("waveform_preview"),
start,
duration,
),
})
return result
def analysis_cache_key(
path,
fps,
trim_start_frame,
length_frames,
half_time,
analysis_source,
detect_beats=True,
):
values = {
"analysis_version": ANALYSIS_VERSION,
"audio_sha256": audio_file_hash(path),
"fps": float(fps),
"trim_start_frame": int(trim_start_frame),
"length_frames": int(length_frames),
"half_time": bool(half_time),
"analysis_source": analysis_source,
"detect_beats": bool(detect_beats),
"detector_version": DETECTOR_VERSION,
@@ -426,50 +536,57 @@ def analyze_audio_file(
path = resolve_audio_path(filename)
_, master_audio = load_audio_file(filename)
cropped_audio, crop = crop_audio(master_audio, fps, trim_start_frame, length_frames)
if analysis_source == "mix":
analysis_audio = cropped_audio
else:
stem_audio = load_cached_stem(filename, analysis_source)
analysis_audio, _ = crop_audio(stem_audio, fps, trim_start_frame, length_frames)
cache_key = analysis_cache_key(
path,
fps,
trim_start_frame,
length_frames,
half_time,
analysis_source,
detect_beats,
)
cache_path = _cache_path(cache_key)
cache_hit = cache_path.is_file()
if cache_hit:
analysis = json.loads(cache_path.read_text(encoding="utf-8"))
source_analysis = json.loads(cache_path.read_text(encoding="utf-8"))
else:
analysis = analyze_audio(
analysis_audio,
fps,
half_time,
detect_beats=detect_beats,
beat_audio=cropped_audio if analysis_source != "mix" and detect_beats else None,
analysis_audio = (
master_audio
if analysis_source == "mix"
else load_cached_stem(filename, analysis_source)
)
analysis.update(crop)
analysis.update({
source_analysis = analyze_audio(
analysis_audio,
detect_beats=detect_beats,
beat_audio=master_audio if analysis_source != "mix" and detect_beats else None,
)
source_analysis.update({
"audio_file": filename,
"analysis_source": analysis_source,
"beat_analysis_source": "mix" if detect_beats else None,
"cache_key": cache_key,
"audio_duration": crop["source_duration"],
"source_duration": crop["source_duration"],
"source_start": 0.0,
})
temporary_path = cache_path.with_suffix(".tmp")
temporary_path.write_text(json.dumps(analysis, separators=(",", ":")), encoding="utf-8")
temporary_path.write_text(
json.dumps(source_analysis, separators=(",", ":")),
encoding="utf-8",
)
temporary_path.replace(cache_path)
if detect_beats:
analysis = apply_beat_offset(
analysis,
apply_half_time(source_analysis, half_time),
fps,
beat_offset_ms,
beat_grid_density,
)
else:
analysis = dict(analysis)
analysis = dict(source_analysis)
analysis = project_analysis(analysis, crop, fps)
analysis["source_analysis"] = {
**source_analysis,
"type": "fl_audio_source_analysis",
"version": SOURCE_ANALYSIS_VERSION,
"analysis_version": ANALYSIS_VERSION,
"analysis_cache_hit": cache_hit,
}
analysis["analysis_cache_hit"] = cache_hit
return analysis, cropped_audio
tests/test_audio_beat_prompt_schedule.py
+7
View File
@@ -498,6 +498,11 @@ class BeatPromptScheduleTests(unittest.TestCase):
"hihat_times": [],
"duration": 1.0,
},
"source_analysis": {
"type": "fl_audio_source_analysis",
"version": 1,
"beat_times": [0.5, 1.0, 1.5],
},
}
with mock.patch.object(schedule, "analyze_audio_file", return_value=(analysis, audio)):
output = schedule.FL_Audio_Beat_Prompt_Schedule.execute(
@@ -524,6 +529,8 @@ class BeatPromptScheduleTests(unittest.TestCase):
self.assertEqual(payload["beat_analysis_source"], "mix")
self.assertEqual(payload["drum_times"]["kick_times"], [0.05])
self.assertEqual(payload["source_start"], 0.5)
self.assertIs(payload["source_analysis"], analysis["source_analysis"])
self.assertNotIn("source_analysis", output.result[0])
def test_cached_analysis_restores_a_missing_audio_widget_value(self):
audio = {"waveform": torch.zeros(1, 1, 24000), "sample_rate": 24000}
tests/test_audio_timeline.py
+157 -10
View File
@@ -42,7 +42,7 @@ class AudioTimelineTests(unittest.TestCase):
mock.patch.object(timeline, "_detect_drums", return_value={}),
mock.patch.object(timeline, "waveform_preview", return_value={}),
):
analysis = timeline.analyze_audio(audio, fps=24.0)
analysis = timeline.analyze_audio(audio)
self.assertAlmostEqual(analysis["bpm"], 120.0)
self.assertEqual(analysis["bpm_source"], "beat_this_intervals_median")
@@ -64,7 +64,7 @@ class AudioTimelineTests(unittest.TestCase):
mock.patch.object(timeline, "_detect_drums", return_value={}),
mock.patch.object(timeline, "waveform_preview", return_value={}),
):
analysis = timeline.analyze_audio(audio, fps=24.0, detect_beats=False)
analysis = timeline.analyze_audio(audio, detect_beats=False)
detector.assert_not_called()
self.assertNotIn("beat_times", analysis)
@@ -88,7 +88,7 @@ class AudioTimelineTests(unittest.TestCase):
mock.patch.object(timeline, "_detect_drums", return_value={}),
mock.patch.object(timeline, "waveform_preview", return_value={}),
):
timeline.analyze_audio(stem, fps=24.0, beat_audio=mix)
timeline.analyze_audio(stem, beat_audio=mix)
self.assertEqual(beat_this.call_args.args[1], 200)
timeline.np.testing.assert_array_equal(beat_this.call_args.args[0], timeline.np.ones(400))
@@ -284,7 +284,7 @@ class AudioTimelineTests(unittest.TestCase):
with (
mock.patch.object(timeline, "resolve_audio_path", return_value=pathlib.Path("song.wav")),
mock.patch.object(timeline, "load_audio_file", return_value=(pathlib.Path("song.wav"), master)),
mock.patch.object(timeline, "load_cached_stem", return_value=stem),
mock.patch.object(timeline, "load_cached_stem", return_value=stem) as load_stem,
mock.patch.object(timeline, "analysis_cache_key", return_value="key"),
mock.patch.object(timeline, "_cache_path", return_value=cache_path),
mock.patch.object(timeline, "analyze_audio", return_value=analysis) as analyze,
@@ -295,10 +295,20 @@ class AudioTimelineTests(unittest.TestCase):
length_frames=24,
analysis_source="drums",
)
_, second_crop = timeline.analyze_audio_file(
"song.wav",
fps=24.0,
trim_start_frame=12,
length_frames=12,
analysis_source="drums",
)
self.assertEqual(cropped["waveform"].mean(), 1.0)
self.assertEqual(second_crop["waveform"].mean(), 1.0)
self.assertEqual(analyze.call_args.args[0]["waveform"].mean(), 2.0)
self.assertEqual(analyze.call_args.kwargs["beat_audio"]["waveform"].mean(), 1.0)
self.assertEqual(analyze.call_count, 1)
self.assertEqual(load_stem.call_count, 1)
def test_offset_changes_reuse_the_base_analysis_cache(self):
master = {"waveform": torch.ones(1, 1, 48000), "sample_rate": 48000}
@@ -366,14 +376,151 @@ class AudioTimelineTests(unittest.TestCase):
self.assertEqual(every_two["beat_times"], [0.0, 0.5])
self.assertEqual(every_beat["cache_key"], every_two["cache_key"])
def test_different_crops_reuse_full_source_analysis_and_rebase_events(self):
master = {"waveform": torch.ones(1, 1, 96000), "sample_rate": 24000}
source_analysis = {
"bpm": 60.0,
"base_grid_interval_seconds": 1.0,
"beat_times": [0.25, 1.25, 2.25, 3.25],
"downbeat_times": [0.25, 2.25],
"detected_beat_times": [0.25, 1.25, 2.25, 3.25],
"detected_downbeat_times": [0.25, 2.25],
"detected_beat_confidences": [0.9, 0.8, 0.7, 0.6],
"detected_downbeat_confidences": [0.95, 0.75],
"onset_times": [0.5, 1.5, 2.5, 3.5],
"audio_duration": 4.0,
"waveform_preview": {
"version": 1,
"duration": 4.0,
"scale": 32767,
"peaks": list(range(16)),
},
"drum_times": {
"kick_times": [1.25, 3.25],
"snare_times": [2.25],
"hihat_times": [],
},
}
with tempfile.TemporaryDirectory() as directory:
cache_path = pathlib.Path(directory) / "analysis.json"
with (
mock.patch.object(timeline, "resolve_audio_path", return_value=pathlib.Path("song.wav")),
mock.patch.object(timeline, "load_audio_file", return_value=(pathlib.Path("song.wav"), master)),
mock.patch.object(timeline, "analysis_cache_key", return_value="key"),
mock.patch.object(timeline, "_cache_path", return_value=cache_path),
mock.patch.object(timeline, "analyze_audio", return_value=source_analysis) as analyze,
):
first, first_audio = timeline.analyze_audio_file(
"song.wav", fps=24.0, trim_start_frame=24, length_frames=48
)
second, second_audio = timeline.analyze_audio_file(
"song.wav", fps=24.0, trim_start_frame=48, length_frames=48
)
self.assertEqual(analyze.call_count, 1)
self.assertEqual(analyze.call_args.args[0]["waveform"].shape[-1], 96000)
self.assertEqual(first_audio["waveform"].shape[-1], 48000)
self.assertEqual(second_audio["waveform"].shape[-1], 48000)
self.assertEqual(first["beat_times"], [0.25, 1.25])
self.assertEqual(second["beat_times"], [0.25, 1.25])
self.assertEqual(first["detected_beat_confidences"], [0.8, 0.7])
self.assertEqual(first["onset_times"], [0.5, 1.5])
self.assertEqual(first["drum_times"]["kick_times"], [0.25])
self.assertEqual(first["source_start"], 1.0)
self.assertEqual(second["source_start"], 2.0)
self.assertEqual(first["source_analysis"]["beat_times"], [0.25, 1.25, 2.25, 3.25])
self.assertFalse(first["analysis_cache_hit"])
self.assertTrue(second["analysis_cache_hit"])
def test_fps_half_time_and_density_project_from_one_source_cache(self):
master = {"waveform": torch.ones(1, 1, 96000), "sample_rate": 24000}
source_analysis = {
"bpm": 120.0,
"base_grid_interval_seconds": 0.5,
"beat_times": [0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5],
"downbeat_times": [0.0, 2.0],
"detected_beat_times": [0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5],
"detected_downbeat_times": [0.0, 2.0],
"detected_beat_confidences": [0.9] * 8,
"detected_downbeat_confidences": [0.95, 0.85],
"onset_times": [],
"audio_duration": 4.0,
"waveform_preview": None,
"drum_times": {},
}
with tempfile.TemporaryDirectory() as directory:
cache_path = pathlib.Path(directory) / "analysis.json"
with (
mock.patch.object(timeline, "resolve_audio_path", return_value=pathlib.Path("song.wav")),
mock.patch.object(timeline, "load_audio_file", return_value=(pathlib.Path("song.wav"), master)),
mock.patch.object(timeline, "analysis_cache_key", return_value="key"),
mock.patch.object(timeline, "_cache_path", return_value=cache_path),
mock.patch.object(timeline, "analyze_audio", return_value=source_analysis) as analyze,
):
regular, _ = timeline.analyze_audio_file(
"song.wav", fps=24.0, trim_start_frame=12, length_frames=48
)
half_time, _ = timeline.analyze_audio_file(
"song.wav",
fps=48.0,
trim_start_frame=24,
length_frames=96,
half_time=True,
)
every_two, _ = timeline.analyze_audio_file(
"song.wav",
fps=24.0,
trim_start_frame=12,
length_frames=48,
beat_grid_density="every_2_beats",
)
self.assertEqual(analyze.call_count, 1)
self.assertEqual(regular["beat_times"], [0.0, 0.5, 1.0, 1.5])
self.assertEqual(half_time["beat_times"], [0.5, 1.5])
self.assertEqual(every_two["beat_times"], [0.5, 1.5])
self.assertEqual(regular["cache_key"], half_time["cache_key"])
self.assertEqual(regular["cache_key"], every_two["cache_key"])
def test_projection_uses_half_open_boundaries_and_keeps_confidence_pairs(self):
analysis = {
"beat_times": [1.0, 1.5, 2.0],
"downbeat_times": [1.0, 2.0],
"base_beat_times": [1.0, 1.5, 2.0],
"base_downbeat_times": [1.0, 2.0],
"detected_beat_times": [1.0, 1.5, 2.0],
"base_detected_beat_times": [1.0, 1.5, 2.0],
"detected_beat_confidences": [0.9, 0.8, 0.7],
"base_detected_beat_confidences": [0.9, 0.8, 0.7],
"detected_downbeat_times": [1.0, 2.0],
"base_detected_downbeat_times": [1.0, 2.0],
"detected_downbeat_confidences": [0.95, 0.75],
"base_detected_downbeat_confidences": [0.95, 0.75],
"onset_times": [1.0, 2.0],
"drum_times": {"kick_times": [1.0, 2.0]},
"waveform_preview": None,
}
crop = {
"source_duration": 3.0,
"source_start": 1.0,
"trim_start_frame": 24,
"length_frames": 24,
"audio_duration": 1.0,
"sample_rate": 48000,
}
projected = timeline.project_analysis(analysis, crop, fps=24.0)
self.assertEqual(projected["beat_times"], [0.0, 0.5])
self.assertEqual(projected["detected_beat_confidences"], [0.9, 0.8])
self.assertEqual(projected["downbeat_times"], [0.0])
self.assertEqual(projected["onset_times"], [0.0])
self.assertEqual(projected["drum_times"]["kick_times"], [0.0])
def test_beat_detection_mode_has_a_separate_versioned_cache_key(self):
with mock.patch.object(timeline, "audio_file_hash", return_value="audio"):
beats = timeline.analysis_cache_key(
pathlib.Path("song.wav"), 24.0, 0, 0, False, "mix", True
)
auxiliary = timeline.analysis_cache_key(
pathlib.Path("song.wav"), 24.0, 0, 0, False, "mix", False
)
beats = timeline.analysis_cache_key(pathlib.Path("song.wav"), "mix", True)
auxiliary = timeline.analysis_cache_key(pathlib.Path("song.wav"), "mix", False)
self.assertNotEqual(beats, auxiliary)
tests/test_audio_timeline_coordinates.mjs
+61
View File
@@ -0,0 +1,61 @@
import assert from "node:assert/strict";
import { readFile } from "node:fs/promises";
import test from "node:test";
const moduleSource = await readFile(
new URL("../web/nodes/audio/audio_timeline_coordinates.js", import.meta.url),
"utf8",
);
const {
cropTimes,
cropTimesWithValues,
sourceTimes,
sourceTimeAtLocalFrame,
sourceTimeToLocalFrame,
waveformBinRange,
} = await import(`data:text/javascript;base64,${Buffer.from(moduleSource).toString("base64")}`);
test("crop projection uses half-open bounds and preserves paired values", () => {
assert.deepEqual(cropTimes([1, 1.5, 2], 1, 2), [0, 0.5]);
assert.deepEqual(
cropTimesWithValues([1, 1.5, 2], [0.9, 0.8, 0.7], 1, 2),
[[0, 0.5], [0.9, 0.8]],
);
});
test("waveform and analyzed beats share the same crop origin", () => {
const fps = 24;
const sourceTime = 10.25;
const firstCrop = 2;
const secondCrop = 3.5;
const firstFrame = sourceTimeToLocalFrame(sourceTime, firstCrop, fps);
const secondFrame = sourceTimeToLocalFrame(sourceTime, secondCrop, fps);
assert.equal(firstFrame - secondFrame, (secondCrop - firstCrop) * fps);
assert.equal(sourceTimeAtLocalFrame(firstFrame, firstCrop, fps), sourceTime);
assert.equal(sourceTimeAtLocalFrame(secondFrame, secondCrop, fps), sourceTime);
const firstBins = waveformBinRange(firstFrame, firstFrame + 1, firstCrop, fps, 0, 20, 2000);
const secondBins = waveformBinRange(secondFrame, secondFrame + 1, secondCrop, fps, 0, 20, 2000);
assert.deepEqual(firstBins, secondBins);
});
test("beat offset moves only the working grid", () => {
const fps = 24;
const cropStart = 2;
const detectedSourceTime = 10.25;
const detectedFrame = sourceTimeToLocalFrame(detectedSourceTime, cropStart, fps);
const waveformSourceTime = sourceTimeAtLocalFrame(detectedFrame, cropStart, fps);
const shiftedGridFrame = sourceTimeToLocalFrame(detectedSourceTime + 0.125, cropStart, fps);
assert.equal(waveformSourceTime, detectedSourceTime);
assert.equal(shiftedGridFrame - detectedFrame, 3);
});
test("legacy crop-local markers are promoted without destructive reprojection", () => {
const source = sourceTimes([0.25, 1.25, 2.25], 4);
assert.deepEqual(cropTimes(source, 4, 6), [0.25, 1.25]);
assert.deepEqual(cropTimes(source, 5, 7), [0.25, 1.25]);
assert.deepEqual(cropTimes(source, 4, 6), [0.25, 1.25]);
});
web/nodes/audio/FL_Audio_Beat_Prompt_Schedule.js
+505 -71
View File
@@ -1,13 +1,21 @@
import { app } from "../../../../scripts/app.js";
import { api } from "../../../../scripts/api.js";
import {
cropTimes,
cropTimesWithValues,
sourceTimes,
sourceTimeToLocalFrame,
waveformBinRange,
} from "./audio_timeline_coordinates.js";
const STYLE_ID = "fl-beat-prompt-sequencer-styles";
const INSTANCES = new Map();
const HEADER_RE = /^\s*\[\s*([0-9]+(?:\.[0-9]+)?)\s*-\s*([0-9]+(?:\.[0-9]+)?)(?:\s*\|\s*(.*?))?\s*\]\s*$/;
const HEADER_START_RE = /^\s*\[\s*[0-9]+(?:\.[0-9]+)?\s*-/;
const EPSILON = 1e-6;
const FORMAT_VERSION = 13;
const COMPATIBLE_FORMAT_VERSIONS = new Set([6, 7, 8, 9, 10, 11, 12, FORMAT_VERSION]);
const FORMAT_VERSION = 15;
const COMPATIBLE_FORMAT_VERSIONS = new Set([6, 7, 8, 9, 10, 11, 12, 13, 14, FORMAT_VERSION]);
const SOURCE_ANALYSIS_VERSION = 1;
const LEGACY_BPM_METHODS = new Set(["beat_intervals", "onset_strength"]);
const COMPACT_NODE_WIDTH = 380;
const MEDIA_FILE_RE = /\.(?:aac|aiff?|flac|m4a|mka|mkv|mov|mp3|mp4|oga|ogg|opus|wav|webm|wma)$/i;
@@ -597,7 +605,7 @@ function setWidgetValue(widget, value) {
}
function restoreCachedAudioWidgets(widgets, saved) {
const beatData = saved?.beatData;
const beatData = saved?.sourceAnalysis || saved?.beatData;
if (!beatData) return;
const audioFile = String(beatData.audioFile || "");
const cacheKey = String(beatData.cacheKey || "");
@@ -894,6 +902,165 @@ function cropWaveformPreview(preview, startSeconds, duration) {
};
}
function analysisArray(value, snake, camel = snake) {
const values = value?.[snake] ?? value?.[camel];
return Array.isArray(values) ? values.map((entry) => finiteNumber(entry)) : [];
}
function sourceAnalysisValue(value) {
if (!value || value.type !== "fl_audio_source_analysis" ||
finiteNumber(value.version) !== SOURCE_ANALYSIS_VERSION) {
return null;
}
const duration = finiteNumber(
value.source_duration ?? value.sourceDuration ?? value.audio_duration ?? value.audioDuration,
);
if (!(duration > 0)) return null;
return {
type: "fl_audio_source_analysis",
version: SOURCE_ANALYSIS_VERSION,
analysisVersion: finiteNumber(value.analysis_version ?? value.analysisVersion),
bpm: finiteNumber(value.bpm),
baseGridIntervalSeconds: finiteNumber(
value.base_grid_interval_seconds ?? value.baseGridIntervalSeconds,
),
beatTimes: analysisArray(value, "beat_times", "beatTimes"),
downbeatTimes: analysisArray(value, "downbeat_times", "downbeatTimes"),
detectedBeatTimes: analysisArray(value, "detected_beat_times", "detectedBeatTimes"),
detectedDownbeatTimes: analysisArray(value, "detected_downbeat_times", "detectedDownbeatTimes"),
detectedBeatConfidences: analysisArray(value, "detected_beat_confidences", "detectedBeatConfidences"),
detectedDownbeatConfidences: analysisArray(
value,
"detected_downbeat_confidences",
"detectedDownbeatConfidences",
),
onsetTimes: analysisArray(value, "onset_times", "onsetTimes"),
drumTimes: value.drum_times || value.drumTimes || {},
duration,
supportsHalfTime: value.supports_half_time == null && value.supportsHalfTime == null
? true
: Boolean(value.supports_half_time ?? value.supportsHalfTime),
waveformPreview: normalizeWaveformPreview(value.waveform_preview || value.waveformPreview),
waveformPreviewStart: finiteNumber(
value.waveform_preview_start ?? value.waveformPreviewStart,
),
cacheKey: String(value.cache_key || value.cacheKey || ""),
audioFile: String(value.audio_file || value.audioFile || ""),
detector: value.detector || null,
detectorVersion: String(value.detector_version || value.detectorVersion || ""),
bpmSource: String(value.bpm_source || value.bpmSource || ""),
analysisSource: String(value.analysis_source || value.analysisSource || "mix"),
beatAnalysisSource: String(value.beat_analysis_source || value.beatAnalysisSource || "mix"),
analysisCacheHit: Boolean(value.analysis_cache_hit ?? value.analysisCacheHit),
};
}
function sourceAnalysisFromCropPayload(value) {
if (!value) return null;
const sourceStart = Math.max(0, finiteNumber(value.source_start ?? value.sourceStart));
const cropDuration = Math.max(0, finiteNumber(value.audio_duration ?? value.audioDuration));
const sourceDuration = Math.max(
sourceStart + cropDuration,
finiteNumber(value.source_duration ?? value.sourceDuration),
);
if (!(cropDuration > 0) || !(sourceDuration > 0)) return null;
const offset = finiteNumber(value.beat_offset_ms ?? value.beatOffsetMs) / 1000;
const payloadBeats = analysisArray(value, "beat_times", "beatTimes");
const payloadDownbeats = analysisArray(value, "downbeat_times", "downbeatTimes");
const baseBeats = analysisArray(value, "base_beat_times", "baseBeatTimes");
const baseDownbeats = analysisArray(value, "base_downbeat_times", "baseDownbeatTimes");
const baseDetectedBeats = analysisArray(
value,
"base_detected_beat_times",
"baseDetectedBeatTimes",
);
const baseDetectedDownbeats = analysisArray(
value,
"base_detected_downbeat_times",
"baseDetectedDownbeatTimes",
);
const baseBeatConfidences = analysisArray(
value,
"base_detected_beat_confidences",
"baseDetectedBeatConfidences",
);
const baseDownbeatConfidences = analysisArray(
value,
"base_detected_downbeat_confidences",
"baseDetectedDownbeatConfidences",
);
const drums = { ...(value.drum_times || value.drumTimes || {}) };
for (const [snake, camel] of [
["kick_times", "kickTimes"],
["snare_times", "snareTimes"],
["hihat_times", "hihatTimes"],
]) {
drums[snake] = sourceTimes(analysisArray(drums, snake, camel), sourceStart);
}
return sourceAnalysisValue({
type: "fl_audio_source_analysis",
version: SOURCE_ANALYSIS_VERSION,
analysis_version: value.analysis_version ?? value.analysisVersion,
bpm: value.bpm,
base_grid_interval_seconds: value.base_grid_interval_seconds ?? value.baseGridIntervalSeconds,
beat_times: sourceTimes(
baseBeats.length ? baseBeats : payloadBeats.map((entry) => entry - offset),
sourceStart,
),
downbeat_times: sourceTimes(
baseDownbeats.length ? baseDownbeats : payloadDownbeats.map((entry) => entry - offset),
sourceStart,
),
detected_beat_times: sourceTimes(
baseDetectedBeats.length
? baseDetectedBeats
: analysisArray(value, "detected_beat_times", "detectedBeatTimes"),
sourceStart,
),
detected_downbeat_times: sourceTimes(
baseDetectedDownbeats.length
? baseDetectedDownbeats
: analysisArray(value, "detected_downbeat_times", "detectedDownbeatTimes"),
sourceStart,
),
detected_beat_confidences: baseBeatConfidences.length
? baseBeatConfidences
: analysisArray(value, "detected_beat_confidences", "detectedBeatConfidences"),
detected_downbeat_confidences: baseDownbeatConfidences.length
? baseDownbeatConfidences
: analysisArray(value, "detected_downbeat_confidences", "detectedDownbeatConfidences"),
onset_times: sourceTimes(analysisArray(value, "onset_times", "onsetTimes"), sourceStart),
drum_times: drums,
source_duration: sourceDuration,
supports_half_time: false,
waveform_preview: value.waveform_preview || value.waveformPreview,
waveform_preview_start: sourceStart,
cache_key: value.cache_key || value.cacheKey,
audio_file: value.audio_file || value.audioFile,
detector: value.detector,
detector_version: value.detector_version || value.detectorVersion,
bpm_source: value.bpm_source || value.bpmSource,
analysis_source: value.analysis_source || value.analysisSource,
beat_analysis_source: value.beat_analysis_source || value.beatAnalysisSource,
analysis_cache_hit: value.analysis_cache_hit ?? value.analysisCacheHit,
});
}
function medianInterval(values) {
const intervals = values
.slice(1)
.map((value, index) => value - values[index])
.filter((value) => value > EPSILON)
.sort((left, right) => left - right);
if (!intervals.length) return 0;
const middle = Math.floor(intervals.length / 2);
return intervals.length % 2
? intervals[middle]
: (intervals[middle - 1] + intervals[middle]) / 2;
}
function audioViewURL(value) {
const match = String(value || "").match(/^(.*?)(?:\s+\[(input|output|temp)\])?$/);
const relative = (match?.[1] || "").replace(/\\/g, "/");
@@ -933,6 +1100,7 @@ class BeatPromptSequencer {
this.hover = null;
this.sourceWaveformPreview = null;
this.sourceAudioDuration = 0;
this.sourceAnalysis = null;
this.audioElement = null;
this.audioURL = "";
this.playbackFrameRequest = null;
@@ -952,9 +1120,12 @@ class BeatPromptSequencer {
const saved = node.properties?.flBeatPromptSequencer || {};
const savedCompatible = COMPATIBLE_FORMAT_VERSIONS.has(finiteNumber(saved.formatVersion));
this.beatData = savedCompatible ? saved.beatData || null : null;
this.sourceAnalysis = savedCompatible ? sourceAnalysisValue(saved.sourceAnalysis) : null;
this.beatData = savedCompatible && !this.sourceAnalysis ? saved.beatData || null : null;
if (this.beatData) {
this.beatData.waveformPreview = normalizeWaveformPreview(this.beatData.waveformPreview);
this.sourceAnalysis = sourceAnalysisFromCropPayload(this.beatData);
if (this.sourceAnalysis) this.beatData = null;
}
restoreCachedAudioWidgets(this.widgets, saved);
this.dataFresh = false;
@@ -1015,7 +1186,7 @@ class BeatPromptSequencer {
<span class="flbps-source-label" data-role="source-label">No audio selected</span>
<span class="flbps-spacer"></span>
<span class="flbps-status" data-role="status">Choose audio to load the timeline</span>
<label class="flbps-auto" title="Refresh beat, onset, and drum markers after audio or trim changes">
<label class="flbps-auto" title="Refresh beat, onset, and drum markers after the audio or transient source changes">
<input data-role="auto-analyze" type="checkbox"> Auto analyze
</label>
<button class="flbps-button" data-action="analyze" title="Analyze beats, onsets, and drums without queueing the workflow">Analyze</button>
@@ -1222,14 +1393,12 @@ class BeatPromptSequencer {
this.resnapClipsToGrid();
this.zoomToFit();
this.markDirty();
this.scheduleAnalysis();
});
bind(this.widgets.sequenceDuration, () => {
this.refreshBrowserCrop();
this.resnapClipsToGrid();
this.zoomToFit();
this.markDirty();
this.scheduleAnalysis();
});
bind(this.widgets.audioFile, () => {
if (this.widgets.analysisCacheKey) this.widgets.analysisCacheKey.value = "";
@@ -1237,11 +1406,17 @@ class BeatPromptSequencer {
});
bind(this.widgets.trimStartFrame, () => {
this.refreshBrowserCrop();
});
bind(this.widgets.halfTime, () => {
this.refreshBrowserCrop();
this.resnapClipsToGrid();
this.markDirty();
});
bind(this.widgets.beatOffset, (value) => this.setBeatOffset(value, false, true));
bind(this.widgets.analysisSource, () => {
this.invalidateAnalysis();
this.scheduleAnalysis();
});
bind(this.widgets.halfTime, () => this.scheduleAnalysis());
bind(this.widgets.beatOffset, (value) => this.setBeatOffset(value, false, true));
bind(this.widgets.analysisSource, () => this.scheduleAnalysis());
bind(this.widgets.beatGridDensity, (value) => this.setBeatGridDensity(value, false));
bind(this.widgets.defaultFadeIn, () => this.markDirty());
bind(this.widgets.defaultFadeOut, () => this.markDirty());
@@ -1265,23 +1440,76 @@ class BeatPromptSequencer {
`${sign}${offset} ms · ${frameSign}${frames.toFixed(2)} fr`;
}
cropBounds() {
const sourceDuration = this.sourceDurationSeconds();
const start = clamp(this.cropStartSeconds(), 0, sourceDuration);
const duration = Math.min(this.cropDurationSeconds(), Math.max(0, sourceDuration - start));
return { start, end: start + duration, duration, sourceDuration };
}
sourceBeatValues() {
const source = this.sourceAnalysis;
if (!source?.beatTimes?.length) return null;
if (!source.supportsHalfTime || !this.widgets.halfTime?.value) {
const interval = source.baseGridIntervalSeconds ||
medianInterval(source.beatTimes) ||
(source.bpm > 0 ? 60 / source.bpm : 0);
return {
beats: source.beatTimes,
downbeats: source.downbeatTimes,
detectedBeats: source.detectedBeatTimes,
detectedDownbeats: source.detectedDownbeatTimes,
beatConfidences: source.detectedBeatConfidences,
downbeatConfidences: source.detectedDownbeatConfidences,
interval,
bpm: interval > 0 ? 60 / interval : source.bpm,
};
}
const retained = source.beatTimes.map((_, index) => index).filter((index) => index % 2 === 0);
const retainedSet = new Set(retained);
const beats = retained.map((index) => source.beatTimes[index]);
const beatConfidences = retained
.filter((index) => index < source.detectedBeatConfidences.length)
.map((index) => source.detectedBeatConfidences[index]);
const downbeats = [];
const downbeatConfidences = [];
for (let index = 0; index < source.downbeatTimes.length; index++) {
const downbeat = source.downbeatTimes[index];
let nearest = 0;
for (let position = 1; position < source.beatTimes.length; position++) {
if (Math.abs(source.beatTimes[position] - downbeat) <
Math.abs(source.beatTimes[nearest] - downbeat)) {
nearest = position;
}
}
if (!retainedSet.has(nearest)) continue;
downbeats.push(downbeat);
if (index < source.detectedDownbeatConfidences.length) {
downbeatConfidences.push(source.detectedDownbeatConfidences[index]);
}
}
const interval = medianInterval(beats);
return {
beats,
downbeats,
detectedBeats: beats,
detectedDownbeats: downbeats,
beatConfidences,
downbeatConfidences,
interval,
bpm: interval > 0 ? 60 / interval : source.bpm,
};
}
baseGridIntervalSeconds() {
const sourceValues = this.sourceBeatValues();
if (sourceValues) return sourceValues.interval;
const values = this.beatData?.baseBeatTimes || [];
const configured = finiteNumber(this.beatData?.baseGridIntervalSeconds);
if (configured > 0) return configured;
if (values.length > 1) {
const intervals = values
.slice(1)
.map((value, index) => finiteNumber(value) - finiteNumber(values[index]))
.filter((value) => value > EPSILON)
.sort((left, right) => left - right);
if (intervals.length) {
const middle = Math.floor(intervals.length / 2);
return intervals.length % 2
? intervals[middle]
: (intervals[middle - 1] + intervals[middle]) / 2;
}
}
const interval = medianInterval(values);
if (interval > 0) return interval;
const bpm = finiteNumber(this.beatData?.bpm);
return bpm > 0 ? 60 / bpm : 0;
}
@@ -1294,7 +1522,7 @@ class BeatPromptSequencer {
}
baseGridTimes() {
const values = this.beatData?.baseBeatTimes || [];
const values = this.sourceBeatValues()?.beats || this.beatData?.baseBeatTimes || [];
if (this.beatGridDensity() === "every_2_beats") {
return values.filter((_, index) => index % 2 === 0);
}
@@ -1302,16 +1530,17 @@ class BeatPromptSequencer {
const result = [];
for (let index = 0; index < values.length; index++) {
result.push(values[index]);
if (index + 1 < values.length) {
result.push((values[index] + values[index + 1]) / 2);
}
if (index + 1 < values.length) result.push((values[index] + values[index + 1]) / 2);
}
return result;
}
gridBeatTimes(offsetMs = this.beatOffsetMs()) {
const values = this.baseGridTimes();
const duration = Math.max(0, finiteNumber(this.beatData?.audioDuration));
const sourceMode = Boolean(this.sourceAnalysis);
const duration = sourceMode
? this.sourceDurationSeconds()
: Math.max(0, finiteNumber(this.beatData?.audioDuration));
const interval = this.gridIntervalSeconds();
if (!values.length || !(duration > 0) || !(interval > 0)) return [];
const offset = finiteNumber(offsetMs) / 1000;
@@ -1328,13 +1557,19 @@ class BeatPromptSequencer {
if (beatTime >= 0) result.push(beatTime);
}
}
return result;
if (!sourceMode) return result;
const crop = this.cropBounds();
return cropTimes(result, crop.start, crop.end);
}
gridDownbeatTimes(offsetMs = this.beatOffsetMs()) {
const beats = this.beatData?.baseBeatTimes || [];
const downbeats = this.beatData?.baseDownbeatTimes || [];
const duration = Math.max(0, finiteNumber(this.beatData?.audioDuration));
const sourceValues = this.sourceBeatValues();
const beats = sourceValues?.beats || this.beatData?.baseBeatTimes || [];
const downbeats = sourceValues?.downbeats || this.beatData?.baseDownbeatTimes || [];
const sourceMode = Boolean(this.sourceAnalysis);
const duration = sourceMode
? this.sourceDurationSeconds()
: Math.max(0, finiteNumber(this.beatData?.audioDuration));
const offset = finiteNumber(offsetMs) / 1000;
if (!beats.length || !downbeats.length || !(duration > 0)) return [];
const density = this.beatGridDensity();
@@ -1349,7 +1584,82 @@ class BeatPromptSequencer {
const shifted = finiteNumber(beats[nearest]) + offset;
if (shifted >= 0 && shifted < duration) result.push(shifted);
}
return result;
if (!sourceMode) return result;
const crop = this.cropBounds();
return cropTimes(result, crop.start, crop.end);
}
projectSourceAnalysis() {
const source = this.sourceAnalysis;
const sourceValues = this.sourceBeatValues();
if (!source || !sourceValues) return false;
const crop = this.cropBounds();
const [detectedBeatTimes, detectedBeatConfidences] = cropTimesWithValues(
sourceValues.detectedBeats,
sourceValues.beatConfidences,
crop.start,
crop.end,
);
const [detectedDownbeatTimes, detectedDownbeatConfidences] = cropTimesWithValues(
sourceValues.detectedDownbeats,
sourceValues.downbeatConfidences,
crop.start,
crop.end,
);
const drums = { ...(source.drumTimes || {}) };
for (const key of ["kick_times", "snare_times", "hihat_times"]) {
drums[key] = cropTimes(drums[key] || [], crop.start, crop.end);
}
drums.duration = crop.duration;
drums.total_kicks = drums.kick_times.length;
drums.total_snares = drums.snare_times.length;
drums.total_hihats = drums.hihat_times.length;
const interval = this.gridIntervalSeconds();
this.beatData = {
bpm: sourceValues.bpm,
gridBpm: interval > 0 ? 60 / interval : sourceValues.bpm,
baseGridIntervalSeconds: sourceValues.interval,
gridIntervalSeconds: interval,
beatGridDensity: this.beatGridDensity(),
baseBeatTimes: cropTimes(sourceValues.beats, crop.start, crop.end),
baseDetectedBeatTimes: detectedBeatTimes,
baseDownbeatTimes: cropTimes(sourceValues.downbeats, crop.start, crop.end),
baseDetectedDownbeatTimes: detectedDownbeatTimes,
baseDetectedBeatConfidences: detectedBeatConfidences,
baseDetectedDownbeatConfidences: detectedDownbeatConfidences,
beatTimes: this.gridBeatTimes(),
downbeatTimes: this.gridDownbeatTimes(),
detectedBeatTimes,
detectedDownbeatTimes,
detectedBeatConfidences,
detectedDownbeatConfidences,
onsetTimes: cropTimes(source.onsetTimes, crop.start, crop.end),
drumTimes: drums,
audioDuration: crop.duration,
sourceDuration: crop.sourceDuration,
sourceStart: crop.start,
fps: this.fps(),
waveformPreview: cropWaveformPreview(
source.waveformPreview,
crop.start - source.waveformPreviewStart,
crop.duration,
) ||
cropWaveformPreview(this.sourceWaveformPreview, crop.start, crop.duration),
cacheKey: source.cacheKey,
audioFile: source.audioFile,
detector: source.detector,
detectorVersion: source.detectorVersion,
bpmSource: source.bpmSource,
analysisSource: source.analysisSource,
beatAnalysisSource: source.beatAnalysisSource,
analysisCacheHit: source.analysisCacheHit,
beatOffsetMs: this.beatOffsetMs(),
};
this.updateTransportTime();
this.refreshBeatStatus();
this.scheduleDraw();
return true;
}
applyBeatOffset() {
@@ -1361,6 +1671,7 @@ class BeatPromptSequencer {
if (this.controls?.beatGridDensity) {
this.controls.beatGridDensity.value = density;
}
if (this.projectSourceAnalysis()) return;
if (!this.beatData) {
this.scheduleDraw();
return;
@@ -1405,6 +1716,9 @@ class BeatPromptSequencer {
saveViewState() {
this.node.properties = this.node.properties || {};
const savedBeatData = this.beatData ? { ...this.beatData, waveformPreview: null } : null;
const savedSourceAnalysis = this.sourceAnalysis
? { ...this.sourceAnalysis, waveformPreview: null }
: null;
const previous = { ...(this.node.properties.flBeatPromptSequencer || {}) };
delete previous.magnetMode;
delete previous.snapMode;
@@ -1412,7 +1726,8 @@ class BeatPromptSequencer {
this.node.properties.flBeatPromptSequencer = {
...previous,
formatVersion: FORMAT_VERSION,
beatData: savedBeatData,
beatData: this.sourceAnalysis ? null : savedBeatData,
sourceAnalysis: savedSourceAnalysis,
viewStart: this.viewStart,
viewEnd: this.viewEnd,
autoAnalyze: this.autoAnalyze,
@@ -1428,10 +1743,44 @@ class BeatPromptSequencer {
return this.trimStartFrame() / this.fps();
}
sourceDurationSeconds() {
return Math.max(
0,
finiteNumber(
this.sourceAudioDuration || this.sourceAnalysis?.duration || this.beatData?.sourceDuration,
),
);
}
cropDurationSeconds() {
const configured = this.configuredFrameCount();
if (configured > 0) return configured / this.fps();
return Math.max(0, this.sourceAudioDuration - this.cropStartSeconds());
return Math.max(0, this.sourceDurationSeconds() - this.cropStartSeconds());
}
waveformSource() {
if (this.sourceAnalysis?.waveformPreview) {
const analysisWaveform = {
preview: this.sourceAnalysis.waveformPreview,
start: this.sourceAnalysis.waveformPreviewStart,
};
const crop = this.cropBounds();
if (analysisWaveform.start <= crop.start + EPSILON &&
analysisWaveform.start + analysisWaveform.preview.duration >= crop.end - EPSILON) {
return analysisWaveform;
}
if (!this.sourceWaveformPreview) return analysisWaveform;
}
if (this.sourceWaveformPreview) {
return { preview: this.sourceWaveformPreview, start: 0 };
}
if (this.beatData?.waveformPreview) {
return {
preview: this.beatData.waveformPreview,
start: finiteNumber(this.beatData.sourceStart),
};
}
return null;
}
setStatus(text, state = "", progress = null) {
@@ -1476,7 +1825,13 @@ class BeatPromptSequencer {
}
invalidateAnalysis() {
if (!this.beatData) return;
this.sourceAnalysis = null;
if (!this.beatData) {
this.dataFresh = false;
this.setStatus("Audio changed · analysis pending", "cached");
this.scheduleDraw();
return;
}
this.beatData.baseBeatTimes = [];
this.beatData.baseDetectedBeatTimes = [];
this.beatData.baseDownbeatTimes = [];
@@ -1497,6 +1852,10 @@ class BeatPromptSequencer {
}
refreshBrowserCrop() {
if (!this.sourceAnalysis && this.beatData?.beatTimes?.length) {
this.sourceAnalysis = sourceAnalysisFromCropPayload(this.beatData);
}
if (this.projectSourceAnalysis()) return;
if (!this.sourceWaveformPreview || !(this.sourceAudioDuration > 0)) {
this.updateTransportTime();
return;
@@ -1527,6 +1886,7 @@ class BeatPromptSequencer {
this.sourceWaveformPreview = null;
this.sourceAudioDuration = 0;
if (!filename) {
this.sourceAnalysis = null;
this.beatData = null;
if (this.widgets.analysisCacheKey) this.widgets.analysisCacheKey.value = "";
this.audioElement = null;
@@ -1585,8 +1945,11 @@ class BeatPromptSequencer {
this.startPlaybackLoop();
});
this.audioElement.addEventListener("ended", () => this.loopPlayback(true));
if (this.sourceAnalysis?.audioFile !== filename ||
this.sourceAnalysis?.analysisSource !== (this.widgets.analysisSource?.value || "mix")) {
this.invalidateAnalysis();
}
this.refreshBrowserCrop();
this.invalidateAnalysis();
this.zoomToFit(false);
this.scheduleAnalysis(0);
} catch (error) {
@@ -1634,10 +1997,14 @@ class BeatPromptSequencer {
const payload = await response.json();
if (!response.ok) throw new Error(payload.error || `Analysis failed (${response.status}).`);
if (request !== this.analysisRequest) return;
this.applyAnalysis(payload, true);
const completeSource = this.applyAnalysis(payload, true);
if (this.migrationPending) this.loadTimeline();
this.resnapClipsToGrid();
this.clearError();
if (completeSource) {
this.clearError();
} else {
this.showError("Restart ComfyUI and refresh the browser to load full-source beat analysis.");
}
this.saveViewState();
analysisCompleted = true;
} catch (error) {
@@ -1653,6 +2020,41 @@ class BeatPromptSequencer {
}
applyAnalysis(payload, fresh) {
const sourceAnalysis = sourceAnalysisValue(payload.source_analysis);
if (sourceAnalysis) {
this.sourceAnalysis = sourceAnalysis;
const audioFile = String(
sourceAnalysis.audioFile || payload.audio_file || this.widgets.audioFile?.value || "",
);
if (this.widgets.audioFile && !this.widgets.audioFile.value && audioFile) {
this.widgets.audioFile.value = audioFile;
}
if (this.widgets.analysisCacheKey) {
this.widgets.analysisCacheKey.value = sourceAnalysis.cacheKey;
}
this.dataFresh = fresh && !sourceAnalysis.analysisCacheHit;
this.applyBeatOffset();
return true;
}
const partialSource = sourceAnalysisFromCropPayload(payload);
if (partialSource) {
this.sourceAnalysis = partialSource;
const audioFile = String(
partialSource.audioFile || payload.audio_file || this.widgets.audioFile?.value || "",
);
if (this.widgets.audioFile && !this.widgets.audioFile.value && audioFile) {
this.widgets.audioFile.value = audioFile;
}
if (this.widgets.analysisCacheKey) {
this.widgets.analysisCacheKey.value = partialSource.cacheKey;
}
this.dataFresh = fresh && !partialSource.analysisCacheHit;
this.applyBeatOffset();
return false;
}
this.sourceAnalysis = null;
const payloadOffset = finiteNumber(payload.beat_offset_ms, 0) / 1000;
const payloadBeatTimes = (payload.beat_times || []).map((value) => finiteNumber(value));
const payloadDetectedBeatTimes = (payload.detected_beat_times || []).map(
@@ -1723,6 +2125,7 @@ class BeatPromptSequencer {
}
this.dataFresh = fresh && !this.beatData.analysisCacheHit;
this.applyBeatOffset();
return false;
}
updatePlayButton() {
@@ -2325,20 +2728,44 @@ class BeatPromptSequencer {
}
beatFrames() {
if (this.sourceAnalysis) {
return this.gridBeatTimes().map((seconds) => Math.round(seconds * this.fps()));
}
return (this.beatData?.beatTimes || []).map((seconds) => Math.round(seconds * this.fps()));
}
downbeatFrames() {
if (this.sourceAnalysis) {
return this.gridDownbeatTimes().map((seconds) => Math.round(seconds * this.fps()));
}
return (this.beatData?.downbeatTimes || []).map((seconds) => Math.round(seconds * this.fps()));
}
detectedBeatFrames() {
const sourceValues = this.sourceBeatValues();
if (sourceValues) {
const crop = this.cropBounds();
return sourceValues.detectedBeats
.filter((seconds) => seconds >= crop.start && seconds < crop.end)
.map((seconds) => sourceTimeToLocalFrame(seconds, crop.start, this.fps()));
}
return (this.beatData?.detectedBeatTimes || []).map((seconds) => Math.round(seconds * this.fps()));
}
detectedBeatMarkers() {
const times = this.beatData?.detectedBeatTimes || [];
const confidences = this.beatData?.detectedBeatConfidences || [];
const sourceValues = this.sourceBeatValues();
const crop = sourceValues ? this.cropBounds() : null;
const [times, confidences] = sourceValues
? cropTimesWithValues(
sourceValues.detectedBeats,
sourceValues.beatConfidences,
crop.start,
crop.end,
)
: [
this.beatData?.detectedBeatTimes || [],
this.beatData?.detectedBeatConfidences || [],
];
const intervals = times.slice(1).map((value, index) => value - times[index]).filter((value) => value > EPSILON);
const sorted = [...intervals].sort((left, right) => left - right);
const median = sorted.length ? sorted[Math.floor(sorted.length / 2)] : 0;
@@ -2353,8 +2780,19 @@ class BeatPromptSequencer {
}
detectedDownbeatMarkers() {
const times = this.beatData?.detectedDownbeatTimes || [];
const confidences = this.beatData?.detectedDownbeatConfidences || [];
const sourceValues = this.sourceBeatValues();
const crop = sourceValues ? this.cropBounds() : null;
const [times, confidences] = sourceValues
? cropTimesWithValues(
sourceValues.detectedDownbeats,
sourceValues.downbeatConfidences,
crop.start,
crop.end,
)
: [
this.beatData?.detectedDownbeatTimes || [],
this.beatData?.detectedDownbeatConfidences || [],
];
return times.map((seconds, index) => ({
frame: Math.round(seconds * this.fps()),
confidence: clamp(finiteNumber(confidences[index], 1), 0, 1),
@@ -2363,6 +2801,12 @@ class BeatPromptSequencer {
}
onsetFrames() {
if (this.sourceAnalysis) {
const crop = this.cropBounds();
return this.sourceAnalysis.onsetTimes
.filter((seconds) => seconds >= crop.start && seconds < crop.end)
.map((seconds) => sourceTimeToLocalFrame(seconds, crop.start, this.fps()));
}
return (this.beatData?.onsetTimes || []).map((seconds) => Math.round(seconds * this.fps()));
}
@@ -2737,12 +3181,10 @@ class BeatPromptSequencer {
this.resnapClipsToGrid(croppedMarkers);
this.playheadFrame = 0;
this.refreshBrowserCrop();
this.invalidateAnalysis();
this.serialize();
this.syncInspector();
this.zoomToFit(false);
this.markDirty();
this.scheduleAnalysis(0);
}
setAudioIn(frame) {
@@ -3165,7 +3607,6 @@ class BeatPromptSequencer {
);
}
this.refreshBrowserCrop();
this.invalidateAnalysis();
this.markDirty();
}
@@ -3201,7 +3642,6 @@ class BeatPromptSequencer {
end: trimHit.handles.end,
sourceFrames: trimHit.handles.sourceFrames,
},
beatFrames: this.beatFrames(),
pointerStartFrame: this.sourceFrameAtX(x),
active: false,
};
@@ -3473,7 +3913,6 @@ class BeatPromptSequencer {
onPointerUp(event) {
if (!this.drag) return;
const finishedDrag = this.drag;
const trimChanged = this.drag.type === "trim-start" ||
this.drag.type === "trim-end" ||
this.drag.type === "trim-move";
@@ -3485,17 +3924,8 @@ class BeatPromptSequencer {
if (this.canvas.hasPointerCapture(event.pointerId)) this.canvas.releasePointerCapture(event.pointerId);
if (changed) {
if (trimChanged) {
const start = this.trimStartFrame();
const duration = this.sequenceFrameCount();
const markers = (finishedDrag.beatFrames || [])
.map((marker) => finishedDrag.original.start + marker - start)
.filter((marker) => marker >= 0 && marker <= duration);
if (markers.length) {
markers.push(0, duration);
this.resnapClipsToGrid([...new Set(markers)].sort((left, right) => left - right));
}
this.resnapClipsToGrid();
this.zoomToFit(false);
this.scheduleAnalysis(0);
} else if (viewChanged) {
this.saveViewState();
} else {
@@ -3713,7 +4143,8 @@ class BeatPromptSequencer {
drawWaveformLane(ctx, width, top, bottom) {
const right = width - TIMELINE_RIGHT;
const center = (top + bottom) / 2;
const preview = this.beatData?.waveformPreview;
const waveform = this.waveformSource();
const preview = waveform?.preview;
ctx.fillStyle = "#14191e";
ctx.fillRect(TIMELINE_LEFT, top, right - TIMELINE_LEFT, bottom - top);
@@ -3746,9 +4177,11 @@ class BeatPromptSequencer {
}
const binCount = preview.peaks.length / 2;
const waveformEndFrame = preview.duration * this.fps();
const visibleStart = Math.max(this.viewStart, 0);
const visibleEnd = Math.min(this.viewEnd, waveformEndFrame);
const cropStart = this.cropStartSeconds();
const waveformStartFrame = (waveform.start - cropStart) * this.fps();
const waveformEndFrame = (waveform.start + preview.duration - cropStart) * this.fps();
const visibleStart = Math.max(this.viewStart, waveformStartFrame, 0);
const visibleEnd = Math.min(this.viewEnd, waveformEndFrame, this.sequenceFrameCount());
if (!(visibleEnd > visibleStart)) return;
const startX = Math.max(TIMELINE_LEFT, Math.floor(this.frameToX(visibleStart, width)));
@@ -3760,14 +4193,13 @@ class BeatPromptSequencer {
for (let x = startX; x <= endX; x++) {
const firstFrame = this.frameAtX(x);
const lastFrame = this.frameAtX(Math.min(endX, x + 1));
const firstBin = clamp(
Math.floor((firstFrame / this.fps() / preview.duration) * binCount),
0,
binCount - 1,
);
const lastBin = clamp(
Math.ceil((lastFrame / this.fps() / preview.duration) * binCount),
firstBin + 1,
const [firstBin, lastBin] = waveformBinRange(
firstFrame,
lastFrame,
cropStart,
this.fps(),
waveform.start,
preview.duration,
binCount,
);
let minimum = preview.scale;
@@ -3786,7 +4218,7 @@ class BeatPromptSequencer {
this.hover.x < TIMELINE_LEFT || this.hover.x > right) {
return;
}
const frame = clamp(Math.round(this.frameAtX(this.hover.x)), 0, Math.round(waveformEndFrame));
const frame = clamp(Math.round(this.frameAtX(this.hover.x)), 0, this.sequenceFrameCount());
const seconds = frame / this.fps();
const x = this.frameToX(frame, width);
ctx.strokeStyle = "rgba(251,191,36,.42)";
@@ -4866,6 +5298,7 @@ function migrateRemovedBpmMethod(graphData) {
node.properties.flBeatPromptSequencer = {
...(node.properties.flBeatPromptSequencer || {}),
beatData: null,
sourceAnalysis: null,
formatVersion: FORMAT_VERSION,
};
}
@@ -4925,6 +5358,7 @@ app.registerExtension({
delete savedSequencer.snapMode;
if (!COMPATIBLE_FORMAT_VERSIONS.has(previousFormat)) {
savedSequencer.beatData = null;
savedSequencer.sourceAnalysis = null;
savedSequencer.viewStart = 0;
savedSequencer.viewEnd = 0;
}
web/nodes/audio/audio_timeline_coordinates.js
+51
View File
@@ -0,0 +1,51 @@
export function cropTimes(values, start, end) {
return (values || [])
.filter((value) => value >= start && value < end)
.map((value) => value - start);
}
export function cropTimesWithValues(times, values, start, end) {
const croppedTimes = [];
const croppedValues = [];
for (let index = 0; index < (times || []).length; index++) {
const value = times[index];
if (value < start || value >= end) continue;
croppedTimes.push(value - start);
if (index < (values || []).length) croppedValues.push(values[index]);
}
return [croppedTimes, croppedValues];
}
export function sourceTimes(values, sourceStart) {
return (values || []).map((value) => value + sourceStart);
}
export function sourceTimeAtLocalFrame(frame, cropStart, fps) {
return cropStart + frame / fps;
}
export function sourceTimeToLocalFrame(sourceTime, cropStart, fps) {
return Math.round((sourceTime - cropStart) * fps);
}
export function waveformBinRange(
firstFrame,
lastFrame,
cropStart,
fps,
previewStart,
previewDuration,
binCount,
) {
const firstSeconds = sourceTimeAtLocalFrame(firstFrame, cropStart, fps) - previewStart;
const lastSeconds = sourceTimeAtLocalFrame(lastFrame, cropStart, fps) - previewStart;
const firstBin = Math.max(
0,
Math.min(binCount - 1, Math.floor(firstSeconds / previewDuration * binCount)),
);
const lastBin = Math.max(
firstBin + 1,
Math.min(binCount, Math.ceil(lastSeconds / previewDuration * binCount)),
);
return [firstBin, lastBin];
}