Movie Editor: audio editing — per-clip volume, inserted tracks, original toggle

- Per-clip original audio gain (Scene.audio_volume; inspector slider, 0 mutes). Applied at
  render (volume filter) and in the live preview (video.volume, clamped 0..1).
- Project-level audio mix: keep_original_audio toggle (drop the generated LTXAV audio) and
  audio_tracks[] — inserted background tracks from the Media bin, each with start offset +
  volume. Mixed at render via adelay + amix(normalize=0), trimmed to the montage length.
- All four cases work: lower/boost a clip, add one or many tracks, keep original + tracks,
  original-only, tracks-only, or fully silent (no audio stream mapped).
- Render route resolves track media to files (media.path_for), appends them as inputs after
  the clips, and maps [aout] only when there's audio. renderFinal flushes the project save
  first (tracks/toggle live on the project, read from disk at render).
- Inspector: per-scene Clip audio + Project Audio section (toggle, track cards, add-from-bin).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

movie_editor/backend/timeline.py

@@ -60,6 +60,8 @@ class Scene:
     # Per-scene post-decode video effects applied to this clip (pixel ops), keys:
     #   {blur: 0..1, fade_in: sec, fade_out: sec, zoom: "none"|"in"|"out"}.
     effects: dict = field(default_factory=dict)
+    # Gain applied to this clip's ORIGINAL (LTXAV) audio at render. 1.0 = unchanged, 0 = mute.
+    audio_volume: float = 1.0
     # Forward-compat per-scene knobs (uniform values still win in V1).
     frames: Optional[int] = None
     fps: Optional[int] = None
@@ -90,6 +92,7 @@ class Scene:
             transition_frames=d.get("transition_frames"),
             video_transition=str(d.get("video_transition", "")),
             effects=dict(d.get("effects") or {}),
+            audio_volume=float(d.get("audio_volume", 1.0)),
             frames=d.get("frames"),
             fps=d.get("fps"),
             frames_mode=str(d.get("frames_mode") or "timeline"),
@@ -145,6 +148,11 @@ class Project:
     # corresponding slot == "funpack"). Keys match ComfyUI widget/input names exactly.
     studio_inputs: dict = field(default_factory=dict)
     sampler_inputs: dict = field(default_factory=dict)
+    # Audio editing (render-time mix). keep_original_audio=False drops the per-clip LTXAV
+    # audio entirely. audio_tracks = inserted tracks mixed over the montage, each:
+    #   {id, media_ref, start_sec, volume, label}. With no tracks and original off → silent.
+    keep_original_audio: bool = True
+    audio_tracks: list = field(default_factory=list)
     # Per-project pipeline config: the configured loader/node slots + linked inputs
     # (same shape as the global models.json). Empty {"slots": []} falls back to the
     # global default at build/read time; the editor seeds new projects from it.
@@ -172,6 +180,8 @@ class Project:
             sampler_slot=str(d.get("sampler_slot", "funpack")),
             studio_inputs=dict(d.get("studio_inputs") or {}),
             sampler_inputs=dict(d.get("sampler_inputs") or {}),
+            keep_original_audio=bool(d.get("keep_original_audio", True)),
+            audio_tracks=list(d.get("audio_tracks") or []),
             models=dict(d.get("models") or {"slots": []}),
             created_at=float(d.get("created_at", time.time())),
             updated_at=float(d.get("updated_at", time.time())),

movie_editor/frontend/inspector.js

@@ -198,6 +198,15 @@
       body.append(el("div", "insp-hint", "Crossfades overlap the two clips, so the montage gets a little shorter."));
     }
 
+    // ── Clip audio (original LTXAV audio gain)
+    const auTag = el("div", "insp-tag"); auTag.textContent = "Clip audio"; body.append(auTag);
+    const vol = el("input"); vol.type = "range"; vol.min = "0"; vol.max = "2"; vol.step = "0.05";
+    vol.value = scene.audio_volume != null ? scene.audio_volume : 1; vol.dataset.k = "sc-vol";
+    const volPct = Math.round((scene.audio_volume != null ? scene.audio_volume : 1) * 100);
+    vol.oninput = () => S.patchSceneQuiet(scene.id, { audio_volume: parseFloat(vol.value) });
+    body.append(field(`Original volume (${volPct}%)`, vol));
+    body.append(el("div", "insp-hint", "0% mutes this clip. Add background tracks & toggle original audio in Project settings."));
+
     const p = st.project;
     const lenRow = el("div", "fields-row");
     lenRow.append(lengthControl(scene, "frames"));
@@ -254,6 +263,48 @@
     neg.oninput = () => S.patchProjectQuiet({ negative_prompt: neg.value });
     body.append(field("Negative prompt", neg));
 
+    // ── Audio (render-time mix) ───────────────────────────────────────────────────
+    const audTag = el("div", "insp-tag"); audTag.textContent = "Audio"; body.append(audTag);
+    const keepRow = el("div", "insp-block");
+    const keepLbl = el("label", "chk"); const keepCb = el("input"); keepCb.type = "checkbox";
+    keepCb.checked = p.keep_original_audio !== false;
+    keepCb.onchange = () => S.patchProject({ keep_original_audio: keepCb.checked });
+    keepLbl.append(keepCb); keepLbl.append(el("span", null, "Keep original (generated) audio"));
+    keepRow.append(keepLbl); body.append(keepRow);
+    if (p.keep_original_audio === false)
+      body.append(el("div", "insp-hint", "Original audio is OFF — the render uses only inserted tracks below (silent if none)."));
+
+    // inserted tracks
+    (p.audio_tracks || []).forEach((t, i) => {
+      const asset = (st.mediaBin || []).find((m) => m.id === t.media_ref);
+      const card = el("div", "aud-track");
+      const head = el("div", "aud-track-head");
+      head.append(el("span", "aud-track-name", (asset && asset.name) || t.label || `Track ${i + 1}`));
+      const rm = el("button", "ic-btn danger", "✕"); rm.title = "Remove track";
+      rm.onclick = () => S.removeAudioTrack(t.id);
+      head.append(rm); card.append(head);
+      const r = el("div", "fields-row");
+      r.append(numberField("Start (s)", t.start_sec || 0, (v) => S.updateAudioTrack(t.id, { start_sec: v }, true), "aud-st-" + t.id));
+      const volWrap = el("label", "field");
+      volWrap.append(el("span", "field-label", `Volume (${Math.round((t.volume != null ? t.volume : 1) * 100)}%)`));
+      const tv = el("input"); tv.type = "range"; tv.min = "0"; tv.max = "2"; tv.step = "0.05";
+      tv.value = t.volume != null ? t.volume : 1; tv.dataset.k = "aud-vol-" + t.id;
+      tv.oninput = () => S.updateAudioTrack(t.id, { volume: parseFloat(tv.value) }, true);
+      volWrap.append(tv); r.append(volWrap);
+      card.append(r);
+      body.append(card);
+    });
+
+    // add a track from the media bin
+    const audioAssets = (st.mediaBin || []).filter((m) => m.kind === "audio" || /\.(mp3|wav|m4a|aac|ogg|flac)$/i.test(m.name || ""));
+    const addRow = el("div", "insp-block");
+    const addSel = el("select"); addSel.dataset.k = "aud-add";
+    addSel.append(new Option(audioAssets.length ? "— add audio track… —" : "— upload audio to the Media bin first —", ""));
+    audioAssets.forEach((m) => addSel.append(new Option(m.name || m.id, m.id)));
+    addSel.onchange = () => { if (addSel.value) { S.addAudioTrack(addSel.value, 0); addSel.value = ""; } };
+    addRow.append(field("Add background track", addSel)); body.append(addRow);
+    body.append(el("div", "insp-hint", "Inserted tracks are mixed at render only (not in the live preview). Per-clip original volume is on each scene."));
+
     const slots = (st.models?.slots || []);
 
     // Conditioning slot selector + its settings panel

movie_editor/frontend/player.js

@@ -50,7 +50,7 @@
     for (const sc of (p.scenes || [])) {
       const dur = sFrames(sc) / sFps(sc);
       const r = sr[sc.id];
-      if (r && r.media) out.push({ media: r.media, sceneId: sc.id, startSec: acc, durationSec: dur, inSec: r.inSec || 0, fx: sc.effects || {} });
+      if (r && r.media) out.push({ media: r.media, sceneId: sc.id, startSec: acc, durationSec: dur, inSec: r.inSec || 0, fx: sc.effects || {}, vol: sc.audio_volume != null ? sc.audio_volume : 1 });
       acc += dur;  // timeline advances even over ungenerated scenes (gaps)
     }
     return out;
@@ -126,6 +126,10 @@
     if (fi > 0 && within < fi) op = Math.max(0, within / fi);
     if (fo > 0 && dur > 0 && within > dur - fo) op = Math.min(op, Math.max(0, (dur - within) / fo));
     v.style.opacity = op < 1 ? op.toFixed(3) : "";
+    // audio: per-clip original volume, muted when the project drops original audio.
+    const keepOrig = (S.get().project || {}).keep_original_audio !== false;
+    const vol = (clip && clip.vol != null) ? clip.vol : 1;
+    v.volume = keepOrig ? Math.max(0, Math.min(1, vol)) : 0;
   }
 
   // Show `clip` at `offset` seconds into the clip; play if requested. Seeks to the clip's

movie_editor/frontend/store.js

@@ -259,6 +259,31 @@
     notify(); scheduleSave();
   }
 
+  // ── audio editing ─────────────────────────────────────────────────────────────
+  function _uid() { return "a" + Math.random().toString(36).slice(2, 9) + Date.now().toString(36); }
+
+  function addAudioTrack(mediaId, startSec) {
+    if (!state.project || !mediaId) return;
+    const asset = (state.mediaBin || []).find((m) => m.id === mediaId);
+    state.project.audio_tracks = state.project.audio_tracks || [];
+    state.project.audio_tracks.push({
+      id: _uid(), media_ref: mediaId, start_sec: +(startSec || 0),
+      volume: 1.0, label: (asset && asset.name) || "track",
+    });
+    notify(); scheduleSave();
+  }
+  function updateAudioTrack(id, patch, quiet) {
+    const t = (state.project?.audio_tracks || []).find((x) => x.id === id);
+    if (!t) return;
+    Object.assign(t, patch);
+    notify(); quiet ? scheduleSaveSilent() : scheduleSave();
+  }
+  function removeAudioTrack(id) {
+    if (!state.project) return;
+    state.project.audio_tracks = (state.project.audio_tracks || []).filter((x) => x.id !== id);
+    notify(); scheduleSave();
+  }
+
   // ── sync scenes from preview (distribute parsed anchor/transitions back) ──────
   function syncFromPreview() {
     if (!state.project || !state.preview) return;
@@ -562,6 +587,7 @@
         h: (sc.height != null ? sc.height : p.height) || null,
         transition: sc.video_transition || "",
         tdur: tFrames > 0 ? tFrames / fps : 0,
+        volume: sc.audio_volume != null ? sc.audio_volume : 1,
       });
     }
     return out;
@@ -588,7 +614,9 @@
       filename: c.media.filename, subfolder: c.media.subfolder || "", type: c.media.type || "output",
       in: +c.inSec.toFixed(3), dur: +c.durationSec.toFixed(3),
       fx: c.fx, fps: c.fps, w: c.w, h: c.h, transition: c.transition, tdur: +(c.tdur || 0).toFixed(3),
+      volume: c.volume,
     }));
+    await flushSave();  // audio tracks / keep-original live on the project — render reads it from disk
     set({ gen: { state: "running", promptId: null, media: [], msg: `Stitching ${clips.length} clip(s)…` } });
     try {
       const r = await API.renderFinal(state.project.id, clips);
@@ -707,6 +735,7 @@
     get, set, subscribe, init,
     refreshProjectList, loadProject, newProject, deleteProject, downloadProject, importProject,
     patchProject, patchProjectQuiet, patchScene, patchSceneQuiet, flushSave, selectScene, addScene, removeScene, moveScene, moveSceneTo, scene,
+    addAudioTrack, updateAudioTrack, removeAudioTrack,
     resizeScene, splitScene, snapFrames,
     refreshPreview, syncFromPreview, applyGlobalPrompt, generate, generateMontage, renderFinal, exportSelected, interrupt, loadModels, loadImageTargets, setModelInput, setModelLink,
     setConditioningSlot, setSamplerSlot, setSamplerInput, setSamplerInputNow, setStudioInput, setStudioInputNow,

movie_editor/frontend/styles.css

@@ -387,6 +387,9 @@ body.tl-reordering { cursor: grabbing; user-select: none; }
 .tl-dd-val { display: flex; align-items: center; justify-content: space-between; gap: 8px; font-size: 12px; }
 .tl-dd-val[hidden] { display: none; }
 .tl-dd-val input { width: 78px; }
+.aud-track { border: 1px solid var(--line); border-radius: 8px; padding: 8px; margin: 6px 0; background: #1a1711; }
+.aud-track-head { display: flex; align-items: center; justify-content: space-between; gap: 8px; margin-bottom: 6px; }
+.aud-track-name { font-size: 12px; font-weight: 600; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
 .clip.selected { border-color: var(--teal); box-shadow: inset 0 0 0 1px var(--teal-soft); z-index: 3; }
 .clip.selected::before { background: var(--teal); opacity: 1; }
 .clip.excluded { opacity: .42; }

movie_editor/server.py

@@ -160,15 +160,22 @@ _XFADE_MAP = {
 }
 
 
-def _build_render_filter(clips: list) -> str:
+def _build_render_filter(clips: list, tracks: Optional[list] = None,
+                         keep_original: bool = True, base_input: int = 0) -> tuple[str, bool]:
     """ffmpeg filter_complex for the final stitch.
 
-    Per clip: apply blur / fade in-out / Ken-Burns zoom (scale eval=frame so the scale
-    animates over the clip; zoom-in crops the overflow, zoom-out pads into the box). Then
-    fold the clips left-to-right: a seam with a video_transition uses xfade (overlap, so the
-    total shortens by the transition length) + acrossfade for audio; otherwise a hard-cut
-    concat. All pure pixel ops — nothing re-encodes to latent. Returns labels [vout]/[aout].
+    Video — per clip: normalize to one canvas, then blur / fade in-out / virtual Ken-Burns
+    zoom (zoompan, fixed output size). Fold left-to-right with xfade (overlap) or concat.
+
+    Audio — per-clip original audio gets its volume; when keep_original the per-clip streams
+    are folded alongside the video (acrossfade at crossfades, concat otherwise) into [aorig].
+    Extra `tracks` (inserted audio, input index base_input+j) are delayed to their start and
+    volume-scaled, then everything is amixed into [aout]. keep_original=False drops the
+    original LTXAV audio entirely; with no tracks either, the output is silent (no [aout]).
+
+    Returns (filter_complex, has_audio).
     """
+    tracks = tracks or []
     n = len(clips)
     parts: list[str] = []
 
@@ -211,9 +218,14 @@ def _build_render_filter(clips: list) -> str:
         vf.append("format=yuv420p")
         vf.append("setsar=1")
         parts.append(f"[{i}:v:0]{','.join(vf)}[v{i}]")
-        parts.append(f"[{i}:a:0]aformat=sample_fmts=fltp:channel_layouts=stereo[a{i}]")
+        if keep_original:
+            vol = float(c.get("volume", 1.0))
+            af = "aformat=sample_fmts=fltp:channel_layouts=stereo"
+            if abs(vol - 1.0) > 1e-3:
+                af += f",volume={max(0.0, vol):.3f}"
+            parts.append(f"[{i}:a:0]{af}[a{i}]")
 
-    # Fold the processed streams together.
+    # Fold video (and the original audio when kept) left-to-right.
     acc_v, acc_a = "[v0]", "[a0]"
     acc_dur = float(clips[0].get("dur") or 0) or 0.0
     for i in range(1, n):
@@ -223,20 +235,45 @@ def _build_render_filter(clips: list) -> str:
         dur_i = float(clips[i].get("dur") or 0) or 0.0
         if trans in _XFADE_MAP and td > 0 and acc_dur > td:
             off = max(0.0, acc_dur - td)
-            ov, oa = f"[vx{i}]", f"[ax{i}]"
+            ov = f"[vx{i}]"
             parts.append(f"{acc_v}[v{i}]xfade=transition={_XFADE_MAP[trans]}:duration={td:.3f}:offset={off:.3f}{ov}")
-            parts.append(f"{acc_a}[a{i}]acrossfade=d={td:.3f}{oa}")
-            acc_v, acc_a, acc_dur = ov, oa, acc_dur + dur_i - td
+            if keep_original:
+                oa = f"[ax{i}]"; parts.append(f"{acc_a}[a{i}]acrossfade=d={td:.3f}{oa}"); acc_a = oa
+            acc_v, acc_dur = ov, acc_dur + dur_i - td
         else:
-            ov, oa = f"[vc{i}]", f"[ac{i}]"
+            ov = f"[vc{i}]"
             parts.append(f"{acc_v}[v{i}]concat=n=2:v=1:a=0{ov}")
-            parts.append(f"{acc_a}[a{i}]concat=n=2:v=0:a=1{oa}")
-            acc_v, acc_a, acc_dur = ov, oa, acc_dur + dur_i
+            if keep_original:
+                oa = f"[ac{i}]"; parts.append(f"{acc_a}[a{i}]concat=n=2:v=0:a=1{oa}"); acc_a = oa
+            acc_v, acc_dur = ov, acc_dur + dur_i
 
-    # Final relabel to [vout]/[aout].
     parts.append(f"{acc_v}null[vout]")
-    parts.append(f"{acc_a}anull[aout]")
-    return ";".join(parts)
+    total = max(0.01, acc_dur)
+
+    # Assemble the audio mix: original (if kept) + each inserted track (delayed/volume'd).
+    mix: list[str] = []
+    if keep_original and n > 0:
+        mix.append(acc_a)
+    for j, t in enumerate(tracks):
+        idx = base_input + j
+        start = float(t.get("start_sec") or 0)
+        tvol = float(t.get("volume", 1.0))
+        ms = int(max(0.0, start) * 1000)
+        lbl = f"[at{j}]"
+        chain = f"[{idx}:a:0]aformat=sample_fmts=fltp:channel_layouts=stereo,volume={max(0.0, tvol):.3f}"
+        if ms > 0:
+            chain += f",adelay={ms}|{ms}"
+        parts.append(f"{chain}{lbl}")
+        mix.append(lbl)
+
+    if not mix:
+        return ";".join(parts), False  # silent output (video only)
+    if len(mix) == 1:
+        parts.append(f"{mix[0]}atrim=0:{total:.3f},asetpts=PTS-STARTPTS[aout]")
+    else:
+        parts.append("".join(mix) + f"amix=inputs={len(mix)}:normalize=0:duration=longest[amx]")
+        parts.append(f"[amx]atrim=0:{total:.3f},asetpts=PTS-STARTPTS[aout]")
+    return ";".join(parts), True
 
 
 def _segment(p: Project, scene_ids: list) -> Project:
@@ -640,9 +677,9 @@ if web is not None and PromptServer is not None:
     # --- API: final render (hard-cut concat of kept clips, with per-clip in/out) ---
     @routes.post(UI_PREFIX + "/api/projects/{pid}/render")
     async def _render_final(req):
-        _project_or_404(req.match_info["pid"])
+        proj = _project_or_404(req.match_info["pid"])
         body = await req.json() if req.can_read_body else {}
-        clips = body.get("clips") or []  # ordered [{filename, subfolder, type, in?, dur?}]
+        clips = body.get("clips") or []  # ordered [{filename, subfolder, type, in?, dur?, volume?}]
         if not clips:
             return web.json_response({"detail": "Nothing to render — no generated clips."}, status=400)
         import os
@@ -668,13 +705,22 @@ if web is not None and PromptServer is not None:
         if missing:
             return web.json_response({"detail": f"{len(missing)} clip file(s) not found on disk — regenerate then render."}, status=400)
 
+        # Inserted audio tracks (project-level): resolve each media asset to a file. Skip any
+        # that no longer exist on disk rather than failing the whole render.
+        keep_original = bool(getattr(proj, "keep_original_audio", True))
+        tracks = []
+        for t in (getattr(proj, "audio_tracks", None) or []):
+            mp = media.path_for(t.get("media_ref") or "")
+            if mp is None:
+                continue
+            tracks.append({"path": str(mp), "start_sec": t.get("start_sec") or 0, "volume": t.get("volume", 1.0)})
+
         # The final render is ephemeral (temp dir) — persist it via the Export Save dialog.
         out_name = f"funpack_final_{int(_time.time())}.mp4"
         out_path = os.path.join(tempdir, out_name)
         # Per-clip in/out via -ss/-t before each input, then a filter graph that applies
-        # per-clip video effects (blur/fade/zoom — pure pixel ops, never re-encode to
-        # latent) and folds clips together with xfade/concat seam transitions. Re-encode
-        # keeps the LTXAV audio.
+        # per-clip video effects + seam transitions and the audio mix (per-clip volume,
+        # original-audio toggle, inserted tracks). Re-encode keeps/mixes the audio.
         cmd = [ff, "-y"]
         for c, pth in zip(clips, paths):
             inn, dur = c.get("in"), c.get("dur")
@@ -684,10 +730,14 @@ if web is not None and PromptServer is not None:
                 cmd += ["-t", f"{float(dur):.3f}"]
             cmd += ["-i", pth]
         n = len(clips)
-        filt = _build_render_filter(clips)
+        for t in tracks:  # inserted-audio inputs follow the clip inputs (indices n..n+k-1)
+            cmd += ["-i", t["path"]]
+        filt, has_audio = _build_render_filter(clips, tracks=tracks, keep_original=keep_original, base_input=n)
+        cmd += ["-filter_complex", filt, "-map", "[vout]"]
+        if has_audio:
+            cmd += ["-map", "[aout]", "-c:a", "aac", "-b:a", "192k"]
         cmd += [
-            "-filter_complex", filt, "-map", "[vout]", "-map", "[aout]",
-            "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k",
+            "-c:v", "libx264", "-pix_fmt", "yuv420p",
             "-movflags", "+faststart", out_path,
         ]
         proc = subprocess.run(cmd, capture_output=True, text=True)