From e419e891e73956cd793dd307459932379b808dc3 Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Thu, 11 Jun 2026 17:00:44 +0300 Subject: [PATCH] Fix overlay audio tracks: separate lane, mix on preview, persist. Overlay tracks use kind overlay with absolute timeline placement, original audio stays on its own lane, and the player now rebuilds when audio_tracks change. Co-authored-by: Cursor --- movie_editor/backend/timeline.py | 8 +++--- movie_editor/frontend/player.js | 8 +++--- movie_editor/frontend/store.js | 41 +++++++++++++++++++++++++++++-- movie_editor/frontend/styles.css | 6 +++++ movie_editor/frontend/timeline.js | 29 +++++++++++++--------- movie_editor/frontend/view_bus.js | 12 ++++++--- movie_editor/server.py | 5 +++- 7 files changed, 84 insertions(+), 25 deletions(-) diff --git a/movie_editor/backend/timeline.py b/movie_editor/backend/timeline.py index ffd1fb77..6fe56683 100644 --- a/movie_editor/backend/timeline.py +++ b/movie_editor/backend/timeline.py @@ -584,10 +584,10 @@ class Project: studio_inputs: dict = field(default_factory=dict) sampler_inputs: dict = field(default_factory=dict) # Audio editing (render-time mix). keep_original_audio=False drops the per-clip LTXAV - # audio entirely. audio_tracks = inserted tracks mixed over the montage, each: - # {id, media_ref, start_sec, volume, label} for Media Bin assets, or - # {id, kind:"separated", scene_id, start_sec, source_in_sec, source_dur, volume, label} - # for audio detached from a generated clip. With no tracks and original off → silent. + # audio entirely. audio_tracks = lanes mixed over the montage: + # overlay — {id, kind:"overlay", media_ref, start_sec, source_in_sec?, source_dur?, volume, label} + # separated — {id, kind:"separated", scene_id, start_sec, source_in_sec, source_dur, volume, label} + # Overlay tracks use absolute timeline start_sec and persist across scene edits. keep_original_audio: bool = True audio_tracks: list = field(default_factory=list) # Per-project pipeline config: the configured loader/node slots + linked inputs diff --git a/movie_editor/frontend/player.js b/movie_editor/frontend/player.js index 05631d97..6fe2b8c8 100644 --- a/movie_editor/frontend/player.js +++ b/movie_editor/frontend/player.js @@ -110,7 +110,7 @@ const bin = st.mediaBin || []; const out = []; for (const t of (st.project.audio_tracks || [])) { - if (t.kind === "separated" && t.scene_id) { + if (S.isSeparatedAudioTrack && S.isSeparatedAudioTrack(t)) { const r = sr[t.scene_id]; if (!r?.media || !pid) continue; out.push({ @@ -121,15 +121,15 @@ inSec: t.source_in_sec != null ? t.source_in_sec : (r.inSec || 0), vol: t.volume != null ? t.volume : 1, }); - } else if (t.media_ref) { + } else if (S.isOverlayAudioTrack ? S.isOverlayAudioTrack(t) : (t.media_ref && t.kind !== "separated")) { const asset = bin.find((m) => m.id === t.media_ref); if (!asset) continue; out.push({ id: t.id, url: API.mediaUrl(t.media_ref), startSec: t.start_sec || 0, - durSec: asset.duration_sec || 86400, - inSec: 0, + durSec: t.source_dur || asset.duration_sec || 86400, + inSec: t.source_in_sec || 0, vol: t.volume != null ? t.volume : 1, }); } diff --git a/movie_editor/frontend/store.js b/movie_editor/frontend/store.js index 324bfd05..f9fe43e6 100644 --- a/movie_editor/frontend/store.js +++ b/movie_editor/frontend/store.js @@ -415,6 +415,7 @@ function _afterTimelineStructureChange() { _pinnedGlobalPrompt = null; + _syncSeparatedAudioTracks(); syncGlobalPromptFromTimeline(); } @@ -613,6 +614,7 @@ if (window.EditorHistory) window.EditorHistory.clear(); notify(); syncGlobalPromptFromTimeline(); + _syncSeparatedAudioTracks(); refreshPreview(); loadModels(); _validateSceneRenders(); @@ -1263,6 +1265,7 @@ s.frames = snapFrames(Math.max(1, durationSec) * fps); // Dragging the trim handle opts the scene into timeline-driven length. if (s.frames_mode == null || s.frames_mode === "project") s.frames_mode = "timeline"; + _syncSeparatedAudioTracks(); scheduleSaveSilent(); } @@ -1306,6 +1309,7 @@ }; } notify(); scheduleSave(); + _syncSeparatedAudioTracks(); } function moveScene(id, delta) { @@ -1317,6 +1321,7 @@ if (i < 0 || j < 0 || j >= arr.length) return; [arr[i], arr[j]] = [arr[j], arr[i]]; notify(); scheduleSave(); + _syncSeparatedAudioTracks(); } // Move a scene to an absolute index (post-removal index). Used by timeline drag-reorder. @@ -1331,19 +1336,50 @@ const [it] = arr.splice(from, 1); arr.splice(toIndex, 0, it); notify(); scheduleSave(); + _syncSeparatedAudioTracks(); } // ── audio editing ───────────────────────────────────────────────────────────── function _uid() { return "a" + Math.random().toString(36).slice(2, 9) + Date.now().toString(36); } + function isOverlayAudioTrack(t) { + return !!(t && t.media_ref && t.kind !== "separated"); + } + + function isSeparatedAudioTrack(t) { + return !!(t && t.kind === "separated" && t.scene_id); + } + + function _syncSeparatedAudioTracks() { + if (!state.project) return; + let moved = false; + for (const t of state.project.audio_tracks || []) { + if (!isSeparatedAudioTrack(t)) continue; + const sc = scene(t.scene_id); + if (!sc || sc.excluded) continue; + const off = sceneTimelineOffsetSec(t.scene_id); + if (Math.abs((t.start_sec || 0) - off) > 0.001) { + t.start_sec = off; + moved = true; + } + } + if (moved) scheduleSaveSilent(); + } + function addAudioTrack(mediaId, startSec) { if (!state.project || !mediaId) return; _historyRecord(); const asset = (state.mediaBin || []).find((m) => m.id === mediaId); state.project.audio_tracks = state.project.audio_tracks || []; state.project.audio_tracks.push({ - id: _uid(), media_ref: mediaId, start_sec: +(startSec || 0), - volume: 1.0, label: (asset && asset.name) || "track", + id: _uid(), + kind: "overlay", + media_ref: mediaId, + start_sec: +(startSec || 0), + source_in_sec: 0, + source_dur: asset?.duration_sec || null, + volume: 1.0, + label: (asset && asset.name) || "Overlay", }); notify(); scheduleSave(); } @@ -2457,6 +2493,7 @@ renderPromptForScene, renderPromptMismatch, renderAnchorMismatch, renderMediaLabel, renderIsStale, buildPreviewSegments, previewTotalSec, segmentDurationSec, addAudioTrack, updateAudioTrack, removeAudioTrack, separateSceneAudio, separatedTrackForScene, + isOverlayAudioTrack, isSeparatedAudioTrack, resizeScene, splitScene, snapFrames, setSourceTrim, trimSceneLeft, slipScene, applyEnginePreset, ENGINE_PRESETS, undo, redo, refreshPreview, syncFromPreview, applyGlobalPromptQuiet, scheduleGlobalPromptApply, buildGlobalPromptFromTimeline, syncGlobalPromptFromTimeline, generate, generateMontage, generateSelected, selectedSceneCount, renderFinal, exportSelected, saveSelectedToMediaBin, clipSaveableToMediaBin, interrupt, loadModels, loadImageTargets, setModelInput, setModelLink, clearNotice, diff --git a/movie_editor/frontend/styles.css b/movie_editor/frontend/styles.css index 4446c2ca..bea42aa4 100644 --- a/movie_editor/frontend/styles.css +++ b/movie_editor/frontend/styles.css @@ -612,6 +612,12 @@ input[type=number] { font-family: var(--font-mono); } .tl-audio-head .chk { font-size: 11px; } .tl-audio-add { font-size: 12px; padding: 2px 6px; max-width: 200px; } .tl-audio-lane { position: relative; height: 44px; border-bottom: 1px solid var(--line-2); } +.tl-audio-lane.orig-lane { background: color-mix(in srgb, var(--teal) 4%, var(--ink-0)); } +.tl-audio-lane.overlay-lane { background: color-mix(in srgb, #60a5fa 5%, var(--ink-0)); } +.tl-gutter-lane.orig { color: var(--teal-soft); } +.tl-gutter-lane.overlay { color: #93c5fd; } +.tl-aud-clip.overlay { border-color: color-mix(in srgb, #60a5fa 35%, var(--line)); } +.tl-aud-clip.overlay .tl-aud-wave canvas { opacity: .95; } .tl-audio-lane:last-child { border-bottom: none; } .tl-audio-lane-body { position: absolute; inset: 0; } .tl-aud-clip { position: absolute; top: 4px; bottom: 4px; box-sizing: border-box; diff --git a/movie_editor/frontend/timeline.js b/movie_editor/frontend/timeline.js index dc25f576..cfd51eb7 100644 --- a/movie_editor/frontend/timeline.js +++ b/movie_editor/frontend/timeline.js @@ -912,16 +912,18 @@ } function insertedAudioLane(st, p, track, laneH) { - const lane = el("div", "tl-audio-lane"); lane.style.height = laneH + "px"; - const isSep = track.kind === "separated" && track.scene_id; - const asset = isSep ? null : (st.mediaBin || []).find((m) => m.id === track.media_ref); + const lane = el("div", "tl-audio-lane" + (track.kind === "overlay" || (track.media_ref && track.kind !== "separated") ? " overlay-lane" : "")); + lane.style.height = laneH + "px"; + const isSep = S.isSeparatedAudioTrack ? S.isSeparatedAudioTrack(track) : (track.kind === "separated" && track.scene_id); + const isOverlay = S.isOverlayAudioTrack ? S.isOverlayAudioTrack(track) : (!isSep && track.media_ref); + const asset = isOverlay ? (st.mediaBin || []).find((m) => m.id === track.media_ref) : null; const sepScene = isSep ? p.scenes.find((s) => s.id === track.scene_id) : null; const sepIdx = sepScene ? p.scenes.indexOf(sepScene) + 1 : 0; const body = el("div", "tl-audio-lane-body"); const startSec = track.start_sec || 0; let durSec = track.source_dur || asset?.duration_sec || 0; const w = Math.max((durSec || 2) * pxPerSec, 48); - const block = el("div", "tl-aud-clip ins" + (isSep ? " sep" : "")); + const block = el("div", "tl-aud-clip ins" + (isSep ? " sep" : " overlay")); block.style.left = (startSec * pxPerSec) + "px"; block.style.width = w + "px"; @@ -1014,19 +1016,24 @@ const gTracks = el("div", "tl-gutter-tracks"); gTracks.append(gutterLane("Video", "Video", "video")); const gAud = el("div", "tl-gutter-aud"); - gAud.append(gutterLane("Audio", "Audio — per-scene volume from generated clips", "audio")); + gAud.append(gutterLane("Original", "Original audio from generated clips and video", "audio orig")); (p.audio_tracks || []).forEach((t) => { let name, title; - if (t.kind === "separated" && t.scene_id) { + const isSep = S.isSeparatedAudioTrack ? S.isSeparatedAudioTrack(t) : (t.kind === "separated" && t.scene_id); + const isOverlay = S.isOverlayAudioTrack ? S.isOverlayAudioTrack(t) : (!isSep && t.media_ref); + if (isSep) { name = t.label || "Separated"; - title = `${name} (detached from generated clip)`; - } else { + title = `${name} (detached clip audio)`; + } else if (isOverlay) { const asset = (st.mediaBin || []).find((m) => m.id === t.media_ref); - name = (asset && asset.name) || t.label || "Audio"; + name = t.label || (asset && asset.name) || "Overlay"; + title = `${name} — mixed on top of original audio`; + } else { + name = t.label || "Audio"; title = name; } const short = name.length > 9 ? name.slice(0, 8) + "…" : name; - gAud.append(gutterLane(short, title, "audio")); + gAud.append(gutterLane(short, title, "audio" + (isOverlay ? " overlay" : isSep ? " sep" : ""))); }); gTracks.append(gAud); gutter.append(gTracks); @@ -1035,7 +1042,7 @@ function audioLanes(st, p, lay) { const wrap = el("div", "tl-audio-lanes"); - const origLane = el("div", "tl-audio-lane"); origLane.style.height = AUDIO_LANE_H + "px"; + const origLane = el("div", "tl-audio-lane orig-lane"); origLane.style.height = AUDIO_LANE_H + "px"; const origBody = el("div", "tl-audio-lane-body"); lay.forEach(({ sc, o, d }, i) => origBody.append(sceneAudioClip(st, p, sc, i, o * pxPerSec, d * pxPerSec))); origLane.append(origBody); diff --git a/movie_editor/frontend/view_bus.js b/movie_editor/frontend/view_bus.js index d4778436..d45e8b65 100644 --- a/movie_editor/frontend/view_bus.js +++ b/movie_editor/frontend/view_bus.js @@ -64,14 +64,20 @@ function fpPlayer(st) { if (!st.project) return "none"; + const p = st.project; return hash({ - pid: st.project.id, - scenes: (st.project.scenes || []).map((s) => [s.id, s.excluded, s.frames, s.fps, s.frames_mode]), + pid: p.id, + scenes: (p.scenes || []).map((s) => [ + s.id, s.excluded, s.frames, s.fps, s.frames_mode, s.fps_mode, + s.audio_volume, s.audio_separated, s.source_in, s.source_dur, s.effects, + ]), renders: st.sceneRenders, ghosts: st.sceneGhosts, + audio: p.audio_tracks, + keepAud: p.keep_original_audio, gen: { state: st.gen?.state, media: st.gen?.media }, mediaPreview: st.mediaPreviewId, - fps: st.project.frame_rate, + fps: p.frame_rate, }); } diff --git a/movie_editor/server.py b/movie_editor/server.py index eb2e6144..5eccd907 100644 --- a/movie_editor/server.py +++ b/movie_editor/server.py @@ -1133,7 +1133,10 @@ if web is not None and PromptServer is not None: media_tracks: list = [] separated_tracks: list = [] for t in (getattr(proj, "audio_tracks", None) or []): - if t.get("kind") == "separated" or (t.get("scene_id") and not t.get("media_ref")): + is_sep = t.get("kind") == "separated" or ( + t.get("scene_id") and not t.get("media_ref") and t.get("kind") != "overlay" + ) + if is_sep: sid = t.get("scene_id") c = clip_by_scene.get(sid) if sid else None if not c: