From 54d58a1e6ba8aad4d13cee61b11e1ab223b8a6fa Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Mon, 8 Jun 2026 01:03:44 +0300 Subject: [PATCH] =?UTF-8?q?Movie=20Editor:=20audio=20editing=20=E2=80=94?= =?UTF-8?q?=20per-clip=20volume,=20inserted=20tracks,=20original=20toggle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Per-clip original audio gain (Scene.audio_volume; inspector slider, 0 mutes). Applied at render (volume filter) and in the live preview (video.volume, clamped 0..1). - Project-level audio mix: keep_original_audio toggle (drop the generated LTXAV audio) and audio_tracks[] — inserted background tracks from the Media bin, each with start offset + volume. Mixed at render via adelay + amix(normalize=0), trimmed to the montage length. - All four cases work: lower/boost a clip, add one or many tracks, keep original + tracks, original-only, tracks-only, or fully silent (no audio stream mapped). - Render route resolves track media to files (media.path_for), appends them as inputs after the clips, and maps [aout] only when there's audio. renderFinal flushes the project save first (tracks/toggle live on the project, read from disk at render). - Inspector: per-scene Clip audio + Project Audio section (toggle, track cards, add-from-bin). Co-Authored-By: Claude Opus 4.8 --- movie_editor/backend/timeline.py | 10 +++ movie_editor/frontend/inspector.js | 51 +++++++++++++++ movie_editor/frontend/player.js | 6 +- movie_editor/frontend/store.js | 29 +++++++++ movie_editor/frontend/styles.css | 3 + movie_editor/server.py | 100 +++++++++++++++++++++-------- 6 files changed, 173 insertions(+), 26 deletions(-) diff --git a/movie_editor/backend/timeline.py b/movie_editor/backend/timeline.py index 9390a0ca..70d29020 100644 --- a/movie_editor/backend/timeline.py +++ b/movie_editor/backend/timeline.py @@ -60,6 +60,8 @@ class Scene: # Per-scene post-decode video effects applied to this clip (pixel ops), keys: # {blur: 0..1, fade_in: sec, fade_out: sec, zoom: "none"|"in"|"out"}. effects: dict = field(default_factory=dict) + # Gain applied to this clip's ORIGINAL (LTXAV) audio at render. 1.0 = unchanged, 0 = mute. + audio_volume: float = 1.0 # Forward-compat per-scene knobs (uniform values still win in V1). frames: Optional[int] = None fps: Optional[int] = None @@ -90,6 +92,7 @@ class Scene: transition_frames=d.get("transition_frames"), video_transition=str(d.get("video_transition", "")), effects=dict(d.get("effects") or {}), + audio_volume=float(d.get("audio_volume", 1.0)), frames=d.get("frames"), fps=d.get("fps"), frames_mode=str(d.get("frames_mode") or "timeline"), @@ -145,6 +148,11 @@ class Project: # corresponding slot == "funpack"). Keys match ComfyUI widget/input names exactly. studio_inputs: dict = field(default_factory=dict) sampler_inputs: dict = field(default_factory=dict) + # Audio editing (render-time mix). keep_original_audio=False drops the per-clip LTXAV + # audio entirely. audio_tracks = inserted tracks mixed over the montage, each: + # {id, media_ref, start_sec, volume, label}. With no tracks and original off → silent. + keep_original_audio: bool = True + audio_tracks: list = field(default_factory=list) # Per-project pipeline config: the configured loader/node slots + linked inputs # (same shape as the global models.json). Empty {"slots": []} falls back to the # global default at build/read time; the editor seeds new projects from it. @@ -172,6 +180,8 @@ class Project: sampler_slot=str(d.get("sampler_slot", "funpack")), studio_inputs=dict(d.get("studio_inputs") or {}), sampler_inputs=dict(d.get("sampler_inputs") or {}), + keep_original_audio=bool(d.get("keep_original_audio", True)), + audio_tracks=list(d.get("audio_tracks") or []), models=dict(d.get("models") or {"slots": []}), created_at=float(d.get("created_at", time.time())), updated_at=float(d.get("updated_at", time.time())), diff --git a/movie_editor/frontend/inspector.js b/movie_editor/frontend/inspector.js index 10f0a45f..33011307 100644 --- a/movie_editor/frontend/inspector.js +++ b/movie_editor/frontend/inspector.js @@ -198,6 +198,15 @@ body.append(el("div", "insp-hint", "Crossfades overlap the two clips, so the montage gets a little shorter.")); } + // ── Clip audio (original LTXAV audio gain) + const auTag = el("div", "insp-tag"); auTag.textContent = "Clip audio"; body.append(auTag); + const vol = el("input"); vol.type = "range"; vol.min = "0"; vol.max = "2"; vol.step = "0.05"; + vol.value = scene.audio_volume != null ? scene.audio_volume : 1; vol.dataset.k = "sc-vol"; + const volPct = Math.round((scene.audio_volume != null ? scene.audio_volume : 1) * 100); + vol.oninput = () => S.patchSceneQuiet(scene.id, { audio_volume: parseFloat(vol.value) }); + body.append(field(`Original volume (${volPct}%)`, vol)); + body.append(el("div", "insp-hint", "0% mutes this clip. Add background tracks & toggle original audio in Project settings.")); + const p = st.project; const lenRow = el("div", "fields-row"); lenRow.append(lengthControl(scene, "frames")); @@ -254,6 +263,48 @@ neg.oninput = () => S.patchProjectQuiet({ negative_prompt: neg.value }); body.append(field("Negative prompt", neg)); + // ── Audio (render-time mix) ─────────────────────────────────────────────────── + const audTag = el("div", "insp-tag"); audTag.textContent = "Audio"; body.append(audTag); + const keepRow = el("div", "insp-block"); + const keepLbl = el("label", "chk"); const keepCb = el("input"); keepCb.type = "checkbox"; + keepCb.checked = p.keep_original_audio !== false; + keepCb.onchange = () => S.patchProject({ keep_original_audio: keepCb.checked }); + keepLbl.append(keepCb); keepLbl.append(el("span", null, "Keep original (generated) audio")); + keepRow.append(keepLbl); body.append(keepRow); + if (p.keep_original_audio === false) + body.append(el("div", "insp-hint", "Original audio is OFF — the render uses only inserted tracks below (silent if none).")); + + // inserted tracks + (p.audio_tracks || []).forEach((t, i) => { + const asset = (st.mediaBin || []).find((m) => m.id === t.media_ref); + const card = el("div", "aud-track"); + const head = el("div", "aud-track-head"); + head.append(el("span", "aud-track-name", (asset && asset.name) || t.label || `Track ${i + 1}`)); + const rm = el("button", "ic-btn danger", "✕"); rm.title = "Remove track"; + rm.onclick = () => S.removeAudioTrack(t.id); + head.append(rm); card.append(head); + const r = el("div", "fields-row"); + r.append(numberField("Start (s)", t.start_sec || 0, (v) => S.updateAudioTrack(t.id, { start_sec: v }, true), "aud-st-" + t.id)); + const volWrap = el("label", "field"); + volWrap.append(el("span", "field-label", `Volume (${Math.round((t.volume != null ? t.volume : 1) * 100)}%)`)); + const tv = el("input"); tv.type = "range"; tv.min = "0"; tv.max = "2"; tv.step = "0.05"; + tv.value = t.volume != null ? t.volume : 1; tv.dataset.k = "aud-vol-" + t.id; + tv.oninput = () => S.updateAudioTrack(t.id, { volume: parseFloat(tv.value) }, true); + volWrap.append(tv); r.append(volWrap); + card.append(r); + body.append(card); + }); + + // add a track from the media bin + const audioAssets = (st.mediaBin || []).filter((m) => m.kind === "audio" || /\.(mp3|wav|m4a|aac|ogg|flac)$/i.test(m.name || "")); + const addRow = el("div", "insp-block"); + const addSel = el("select"); addSel.dataset.k = "aud-add"; + addSel.append(new Option(audioAssets.length ? "— add audio track… —" : "— upload audio to the Media bin first —", "")); + audioAssets.forEach((m) => addSel.append(new Option(m.name || m.id, m.id))); + addSel.onchange = () => { if (addSel.value) { S.addAudioTrack(addSel.value, 0); addSel.value = ""; } }; + addRow.append(field("Add background track", addSel)); body.append(addRow); + body.append(el("div", "insp-hint", "Inserted tracks are mixed at render only (not in the live preview). Per-clip original volume is on each scene.")); + const slots = (st.models?.slots || []); // Conditioning slot selector + its settings panel diff --git a/movie_editor/frontend/player.js b/movie_editor/frontend/player.js index dbedd4e1..7fa91769 100644 --- a/movie_editor/frontend/player.js +++ b/movie_editor/frontend/player.js @@ -50,7 +50,7 @@ for (const sc of (p.scenes || [])) { const dur = sFrames(sc) / sFps(sc); const r = sr[sc.id]; - if (r && r.media) out.push({ media: r.media, sceneId: sc.id, startSec: acc, durationSec: dur, inSec: r.inSec || 0, fx: sc.effects || {} }); + if (r && r.media) out.push({ media: r.media, sceneId: sc.id, startSec: acc, durationSec: dur, inSec: r.inSec || 0, fx: sc.effects || {}, vol: sc.audio_volume != null ? sc.audio_volume : 1 }); acc += dur; // timeline advances even over ungenerated scenes (gaps) } return out; @@ -126,6 +126,10 @@ if (fi > 0 && within < fi) op = Math.max(0, within / fi); if (fo > 0 && dur > 0 && within > dur - fo) op = Math.min(op, Math.max(0, (dur - within) / fo)); v.style.opacity = op < 1 ? op.toFixed(3) : ""; + // audio: per-clip original volume, muted when the project drops original audio. + const keepOrig = (S.get().project || {}).keep_original_audio !== false; + const vol = (clip && clip.vol != null) ? clip.vol : 1; + v.volume = keepOrig ? Math.max(0, Math.min(1, vol)) : 0; } // Show `clip` at `offset` seconds into the clip; play if requested. Seeks to the clip's diff --git a/movie_editor/frontend/store.js b/movie_editor/frontend/store.js index 4507b7d8..3905cf4f 100644 --- a/movie_editor/frontend/store.js +++ b/movie_editor/frontend/store.js @@ -259,6 +259,31 @@ notify(); scheduleSave(); } + // ── audio editing ───────────────────────────────────────────────────────────── + function _uid() { return "a" + Math.random().toString(36).slice(2, 9) + Date.now().toString(36); } + + function addAudioTrack(mediaId, startSec) { + if (!state.project || !mediaId) return; + const asset = (state.mediaBin || []).find((m) => m.id === mediaId); + state.project.audio_tracks = state.project.audio_tracks || []; + state.project.audio_tracks.push({ + id: _uid(), media_ref: mediaId, start_sec: +(startSec || 0), + volume: 1.0, label: (asset && asset.name) || "track", + }); + notify(); scheduleSave(); + } + function updateAudioTrack(id, patch, quiet) { + const t = (state.project?.audio_tracks || []).find((x) => x.id === id); + if (!t) return; + Object.assign(t, patch); + notify(); quiet ? scheduleSaveSilent() : scheduleSave(); + } + function removeAudioTrack(id) { + if (!state.project) return; + state.project.audio_tracks = (state.project.audio_tracks || []).filter((x) => x.id !== id); + notify(); scheduleSave(); + } + // ── sync scenes from preview (distribute parsed anchor/transitions back) ────── function syncFromPreview() { if (!state.project || !state.preview) return; @@ -562,6 +587,7 @@ h: (sc.height != null ? sc.height : p.height) || null, transition: sc.video_transition || "", tdur: tFrames > 0 ? tFrames / fps : 0, + volume: sc.audio_volume != null ? sc.audio_volume : 1, }); } return out; @@ -588,7 +614,9 @@ filename: c.media.filename, subfolder: c.media.subfolder || "", type: c.media.type || "output", in: +c.inSec.toFixed(3), dur: +c.durationSec.toFixed(3), fx: c.fx, fps: c.fps, w: c.w, h: c.h, transition: c.transition, tdur: +(c.tdur || 0).toFixed(3), + volume: c.volume, })); + await flushSave(); // audio tracks / keep-original live on the project — render reads it from disk set({ gen: { state: "running", promptId: null, media: [], msg: `Stitching ${clips.length} clip(s)…` } }); try { const r = await API.renderFinal(state.project.id, clips); @@ -707,6 +735,7 @@ get, set, subscribe, init, refreshProjectList, loadProject, newProject, deleteProject, downloadProject, importProject, patchProject, patchProjectQuiet, patchScene, patchSceneQuiet, flushSave, selectScene, addScene, removeScene, moveScene, moveSceneTo, scene, + addAudioTrack, updateAudioTrack, removeAudioTrack, resizeScene, splitScene, snapFrames, refreshPreview, syncFromPreview, applyGlobalPrompt, generate, generateMontage, renderFinal, exportSelected, interrupt, loadModels, loadImageTargets, setModelInput, setModelLink, setConditioningSlot, setSamplerSlot, setSamplerInput, setSamplerInputNow, setStudioInput, setStudioInputNow, diff --git a/movie_editor/frontend/styles.css b/movie_editor/frontend/styles.css index d61953b3..6ef38cc2 100644 --- a/movie_editor/frontend/styles.css +++ b/movie_editor/frontend/styles.css @@ -387,6 +387,9 @@ body.tl-reordering { cursor: grabbing; user-select: none; } .tl-dd-val { display: flex; align-items: center; justify-content: space-between; gap: 8px; font-size: 12px; } .tl-dd-val[hidden] { display: none; } .tl-dd-val input { width: 78px; } +.aud-track { border: 1px solid var(--line); border-radius: 8px; padding: 8px; margin: 6px 0; background: #1a1711; } +.aud-track-head { display: flex; align-items: center; justify-content: space-between; gap: 8px; margin-bottom: 6px; } +.aud-track-name { font-size: 12px; font-weight: 600; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } .clip.selected { border-color: var(--teal); box-shadow: inset 0 0 0 1px var(--teal-soft); z-index: 3; } .clip.selected::before { background: var(--teal); opacity: 1; } .clip.excluded { opacity: .42; } diff --git a/movie_editor/server.py b/movie_editor/server.py index 6f88b2eb..e8cf2966 100644 --- a/movie_editor/server.py +++ b/movie_editor/server.py @@ -160,15 +160,22 @@ _XFADE_MAP = { } -def _build_render_filter(clips: list) -> str: +def _build_render_filter(clips: list, tracks: Optional[list] = None, + keep_original: bool = True, base_input: int = 0) -> tuple[str, bool]: """ffmpeg filter_complex for the final stitch. - Per clip: apply blur / fade in-out / Ken-Burns zoom (scale eval=frame so the scale - animates over the clip; zoom-in crops the overflow, zoom-out pads into the box). Then - fold the clips left-to-right: a seam with a video_transition uses xfade (overlap, so the - total shortens by the transition length) + acrossfade for audio; otherwise a hard-cut - concat. All pure pixel ops — nothing re-encodes to latent. Returns labels [vout]/[aout]. + Video — per clip: normalize to one canvas, then blur / fade in-out / virtual Ken-Burns + zoom (zoompan, fixed output size). Fold left-to-right with xfade (overlap) or concat. + + Audio — per-clip original audio gets its volume; when keep_original the per-clip streams + are folded alongside the video (acrossfade at crossfades, concat otherwise) into [aorig]. + Extra `tracks` (inserted audio, input index base_input+j) are delayed to their start and + volume-scaled, then everything is amixed into [aout]. keep_original=False drops the + original LTXAV audio entirely; with no tracks either, the output is silent (no [aout]). + + Returns (filter_complex, has_audio). """ + tracks = tracks or [] n = len(clips) parts: list[str] = [] @@ -211,9 +218,14 @@ def _build_render_filter(clips: list) -> str: vf.append("format=yuv420p") vf.append("setsar=1") parts.append(f"[{i}:v:0]{','.join(vf)}[v{i}]") - parts.append(f"[{i}:a:0]aformat=sample_fmts=fltp:channel_layouts=stereo[a{i}]") + if keep_original: + vol = float(c.get("volume", 1.0)) + af = "aformat=sample_fmts=fltp:channel_layouts=stereo" + if abs(vol - 1.0) > 1e-3: + af += f",volume={max(0.0, vol):.3f}" + parts.append(f"[{i}:a:0]{af}[a{i}]") - # Fold the processed streams together. + # Fold video (and the original audio when kept) left-to-right. acc_v, acc_a = "[v0]", "[a0]" acc_dur = float(clips[0].get("dur") or 0) or 0.0 for i in range(1, n): @@ -223,20 +235,45 @@ def _build_render_filter(clips: list) -> str: dur_i = float(clips[i].get("dur") or 0) or 0.0 if trans in _XFADE_MAP and td > 0 and acc_dur > td: off = max(0.0, acc_dur - td) - ov, oa = f"[vx{i}]", f"[ax{i}]" + ov = f"[vx{i}]" parts.append(f"{acc_v}[v{i}]xfade=transition={_XFADE_MAP[trans]}:duration={td:.3f}:offset={off:.3f}{ov}") - parts.append(f"{acc_a}[a{i}]acrossfade=d={td:.3f}{oa}") - acc_v, acc_a, acc_dur = ov, oa, acc_dur + dur_i - td + if keep_original: + oa = f"[ax{i}]"; parts.append(f"{acc_a}[a{i}]acrossfade=d={td:.3f}{oa}"); acc_a = oa + acc_v, acc_dur = ov, acc_dur + dur_i - td else: - ov, oa = f"[vc{i}]", f"[ac{i}]" + ov = f"[vc{i}]" parts.append(f"{acc_v}[v{i}]concat=n=2:v=1:a=0{ov}") - parts.append(f"{acc_a}[a{i}]concat=n=2:v=0:a=1{oa}") - acc_v, acc_a, acc_dur = ov, oa, acc_dur + dur_i + if keep_original: + oa = f"[ac{i}]"; parts.append(f"{acc_a}[a{i}]concat=n=2:v=0:a=1{oa}"); acc_a = oa + acc_v, acc_dur = ov, acc_dur + dur_i - # Final relabel to [vout]/[aout]. parts.append(f"{acc_v}null[vout]") - parts.append(f"{acc_a}anull[aout]") - return ";".join(parts) + total = max(0.01, acc_dur) + + # Assemble the audio mix: original (if kept) + each inserted track (delayed/volume'd). + mix: list[str] = [] + if keep_original and n > 0: + mix.append(acc_a) + for j, t in enumerate(tracks): + idx = base_input + j + start = float(t.get("start_sec") or 0) + tvol = float(t.get("volume", 1.0)) + ms = int(max(0.0, start) * 1000) + lbl = f"[at{j}]" + chain = f"[{idx}:a:0]aformat=sample_fmts=fltp:channel_layouts=stereo,volume={max(0.0, tvol):.3f}" + if ms > 0: + chain += f",adelay={ms}|{ms}" + parts.append(f"{chain}{lbl}") + mix.append(lbl) + + if not mix: + return ";".join(parts), False # silent output (video only) + if len(mix) == 1: + parts.append(f"{mix[0]}atrim=0:{total:.3f},asetpts=PTS-STARTPTS[aout]") + else: + parts.append("".join(mix) + f"amix=inputs={len(mix)}:normalize=0:duration=longest[amx]") + parts.append(f"[amx]atrim=0:{total:.3f},asetpts=PTS-STARTPTS[aout]") + return ";".join(parts), True def _segment(p: Project, scene_ids: list) -> Project: @@ -640,9 +677,9 @@ if web is not None and PromptServer is not None: # --- API: final render (hard-cut concat of kept clips, with per-clip in/out) --- @routes.post(UI_PREFIX + "/api/projects/{pid}/render") async def _render_final(req): - _project_or_404(req.match_info["pid"]) + proj = _project_or_404(req.match_info["pid"]) body = await req.json() if req.can_read_body else {} - clips = body.get("clips") or [] # ordered [{filename, subfolder, type, in?, dur?}] + clips = body.get("clips") or [] # ordered [{filename, subfolder, type, in?, dur?, volume?}] if not clips: return web.json_response({"detail": "Nothing to render — no generated clips."}, status=400) import os @@ -668,13 +705,22 @@ if web is not None and PromptServer is not None: if missing: return web.json_response({"detail": f"{len(missing)} clip file(s) not found on disk — regenerate then render."}, status=400) + # Inserted audio tracks (project-level): resolve each media asset to a file. Skip any + # that no longer exist on disk rather than failing the whole render. + keep_original = bool(getattr(proj, "keep_original_audio", True)) + tracks = [] + for t in (getattr(proj, "audio_tracks", None) or []): + mp = media.path_for(t.get("media_ref") or "") + if mp is None: + continue + tracks.append({"path": str(mp), "start_sec": t.get("start_sec") or 0, "volume": t.get("volume", 1.0)}) + # The final render is ephemeral (temp dir) — persist it via the Export Save dialog. out_name = f"funpack_final_{int(_time.time())}.mp4" out_path = os.path.join(tempdir, out_name) # Per-clip in/out via -ss/-t before each input, then a filter graph that applies - # per-clip video effects (blur/fade/zoom — pure pixel ops, never re-encode to - # latent) and folds clips together with xfade/concat seam transitions. Re-encode - # keeps the LTXAV audio. + # per-clip video effects + seam transitions and the audio mix (per-clip volume, + # original-audio toggle, inserted tracks). Re-encode keeps/mixes the audio. cmd = [ff, "-y"] for c, pth in zip(clips, paths): inn, dur = c.get("in"), c.get("dur") @@ -684,10 +730,14 @@ if web is not None and PromptServer is not None: cmd += ["-t", f"{float(dur):.3f}"] cmd += ["-i", pth] n = len(clips) - filt = _build_render_filter(clips) + for t in tracks: # inserted-audio inputs follow the clip inputs (indices n..n+k-1) + cmd += ["-i", t["path"]] + filt, has_audio = _build_render_filter(clips, tracks=tracks, keep_original=keep_original, base_input=n) + cmd += ["-filter_complex", filt, "-map", "[vout]"] + if has_audio: + cmd += ["-map", "[aout]", "-c:a", "aac", "-b:a", "192k"] cmd += [ - "-filter_complex", filt, "-map", "[vout]", "-map", "[aout]", - "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", + "-c:v", "libx264", "-pix_fmt", "yuv420p", "-movflags", "+faststart", out_path, ] proc = subprocess.run(cmd, capture_output=True, text=True)