feat(ui): Separate/Remove audio with audio lanes that follow their clips; real saved and ComfyUI chips

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

app/ui/boot.js

@@ -10,6 +10,7 @@ import { api } from "./shell/api.js";
 import { createSelection } from "./shell/selection.js";
 import { createPlayhead } from "./shell/playhead.js";
 import { createMaintenance } from "./shell/maintenance.js";
+import { syncSeparated } from "./shell/audio.js";
 import { createKeys } from "./shell/keys.js";
 import { createDock } from "./shell/dock.js";
 import { createMode } from "./shell/mode.js";
@@ -21,7 +22,7 @@ import paths from "./modules.js";
 const heard = new Set();
 const say = (what) => heard.forEach((fn) => { try { fn(what); } catch (err) { console.error(err); } });
 const on = (fn) => { heard.add(fn); return () => heard.delete(fn); };
-const project = createProject({ onChange: () => say("change"), onOpen: () => say("open"), onError: (err) => console.warn(err) });
+const project = createProject({ keepConsistent: syncSeparated, onChange: () => say("change"), onOpen: () => say("open"), onError: (err) => console.warn(err) });
 const pipeline = createPipelineState(api);
 const generate = createGenerate({ pipeline });
 linkPipeline({ project, pipeline, onOpen: (fn) => on((what) => { if (what === "open") fn(); }) });

app/ui/features/audio/index.js

@@ -0,0 +1,24 @@
+// Separate audio / Remove audio: pull the picked clip's own sound onto a lane of its own, or give it back.
+import { composer as c } from "../../composer/composer.js";
+import { inPlace } from "../../shell/place.js";
+import { hasEmbeddedAudio, removeTrack, separate, trackFor } from "../../shell/audio.js";
+
+export default {
+  id: "audio",
+  mount: "timeline.toolbar",
+  needs: ["project", "selection"],
+  setup({ host, app }) {
+    const p = app.project, sel = app.selection;
+    const put = inPlace(host);
+    const draw = () => {
+      const open = p.project, sc = open && sel.focus ? open.scenes.find((s) => s.id === sel.focus) : null;
+      const lane = open && sc ? trackFor(open, sc.id) : null;
+      put(c.button.sm({ label: "⊟ Separate audio", tone: "ghost", disabled: !(open && sc && hasEmbeddedAudio(sc, open)), title: "Detach this clip's audio onto its own track (the clip keeps picture only)",
+          onClick: () => p.edit((pr) => separate(pr, sel.focus)) }).node,
+        c.button.sm({ label: "Remove audio", tone: "danger", disabled: !lane, title: "Remove this clip's separated audio track",
+          onClick: () => p.edit((pr) => removeTrack(pr, lane.id)) }).node);
+    };
+    draw();
+    return app.on(draw);
+  },
+};

app/ui/features/placeholders/index.js

@@ -1,17 +0,0 @@
-// Stand-ins for the parts of v4's screen whose features are not ported yet: same place, same look,
-// inert. A real feature that mounts at one of these points replaces its row here (delete the row).
-import { composer as c } from "../../composer/composer.js";
-
-const btn = (label, tone = "ghost", extra = {}) => c.button.sm({ label, tone, disabled: true, ...extra });
-const menu = (label) => c.button.menu({ label, tone: "ghost" });          // inert, but not dimmed: a menu is never greyed in v4
-
-const items = {
-  "menubar.right": () => [c.chip.neutral({ label: "saved" }), c.chip.good({ label: "ComfyUI live", dot: true })],
-
-  "timeline.toolbar": () => [btn("⊟ Separate audio"), btn("Remove audio")],
-};
-
-export default Object.entries(items).map(([mount, build]) => ({
-  id: `placeholder:${mount}`, mount,
-  setup: ({ host }) => build().forEach((handle) => host.append(handle.node)),
-}));

app/ui/features/status/index.js

@@ -0,0 +1,21 @@
+// The two chips at the far end of the menubar: whether the project is saved, and whether ComfyUI answers.
+import { composer as c } from "../../composer/composer.js";
+import { inPlace } from "../../shell/place.js";
+
+export default {
+  id: "status",
+  mount: "menubar.right",
+  needs: ["project", "api"],
+  setup({ host, app }) {
+    const put = inPlace(host);
+    let live = true;
+    const draw = () => put((app.project.unsaved ? c.chip.warn({ label: "unsaved" }) : c.chip.neutral({ label: "saved" })).node,
+      (live ? c.chip.good({ label: "ComfyUI live", dot: true }) : c.chip.danger({ label: "ComfyUI offline", dot: true })).node);
+    const ping = async () => { try { live = Boolean((await app.api.health()).ok); } catch { live = false; } draw(); };
+    draw(); ping();
+    const timers = [setInterval(() => { const now = Boolean(app.project.unsaved); if (now !== saved) { saved = now; draw(); } }, 600), setInterval(ping, 10000)];
+    let saved = Boolean(app.project.unsaved);
+    const off = app.on(() => { saved = Boolean(app.project.unsaved); draw(); });
+    return () => { timers.forEach(clearInterval); off(); };
+  },
+};

app/ui/features/timeline/index.js

@@ -5,7 +5,7 @@ import { composer as c } from "../../composer/composer.js";
 import { segments, totalSeconds, clock } from "../../shell/scenes.js";
 import { inPlace } from "../../shell/place.js";
 import * as edits from "../../shell/edits.js";
-import { videoLane, audioLane, framesAt } from "./lanes.js";
+import { videoLane, audioLane, tracksLane, framesAt } from "./lanes.js";
 
 const ZOOM = [20, 40, 80, 160, 320];
 const sceneIds = (p) => segments(p).filter((s) => s.kind === "scene").map((s) => s.id);
@@ -37,7 +37,7 @@ const stage = {
     const ghostActions = (g) => [{ label: "✕", title: "Remove from the timeline",
       onClick: () => edit((pr) => { pr.scene_ghosts = (pr.scene_ghosts || []).filter((x) => x.id !== g.id); return true; }) }];
 
-    const draw = () => { const open = p.project; view.setLanes(open ? [videoLane(open, sel.ids, sel.focus, clipActions, ghostActions), audioLane(open, sel.ids)] : []); };
+    const draw = () => { const open = p.project; view.setLanes(open ? [videoLane(open, sel.ids, sel.focus, clipActions, ghostActions), audioLane(open, sel.ids), tracksLane(open)].filter(Boolean) : []); };
     const setZoom = (z) => { zoom = Math.min(ZOOM.length - 1, Math.max(0, z)); view.setZoom(ZOOM[zoom]); };
     host.append(view.node);
     const off = [app.on((what) => {

app/ui/features/timeline/lanes.js

@@ -31,3 +31,9 @@ export const audioLane = (p, picked) => ({
   id: "audio", label: "Original", kind: "audio",
   clips: segments(p).filter((s) => s.kind === "scene").map((s, i) => ({ id: `a:${s.id}`, start: s.start, dur: s.dur, title: `S${i + 1}`, selected: picked.includes(s.id), excluded: Boolean(s.scene.excluded) })),
 });
+
+/** Audio lanes added to the project (a clip's separated sound, or a file laid over the cut), when there are any. */
+export const tracksLane = (p) => (p.audio_tracks || []).length ? {
+  id: "tracks", label: "Audio", kind: "audio",
+  clips: p.audio_tracks.map((t) => ({ id: `t:${t.id}`, start: t.start_sec || 0, dur: t.pinned_dur ?? t.source_dur ?? 1, title: t.label || "Audio" })),
+} : null;

app/ui/modules.js

@@ -14,8 +14,9 @@ export default [
   "./features/menus/index.js",
   "./features/render/index.js",
   "./features/render/selected.js",
+  "./features/status/index.js",
+  "./features/audio/index.js",
   "./features/montage/index.js",
-  "./features/placeholders/index.js",
   "./features/generate/warn.js",
   "./features/composer/index.js",
   "./features/settings/index.js",

app/ui/shell/audio.js

@@ -0,0 +1,54 @@
+// Audio lanes. A clip's own sound can be pulled onto a lane of its own ("separated"): the lane plays the sound pinned from
+// that clip's picture, and the clip itself goes quiet. A separated lane follows its clip around the timeline.
+import { segments, isVideoClip } from "./scenes.js";
+
+const newId = () => "t" + Math.random().toString(36).slice(2, 10) + Date.now().toString(36);
+
+export const trackFor = (p, sceneId) => (p.audio_tracks || []).find((t) => t.kind === "separated" && t.scene_id === sceneId);
+
+/** Does this clip have sound to pull out: a bin video clip, or a generated clip with a render. */
+export const hasEmbeddedAudio = (sc, p) => {
+  if (!sc || sc.excluded || sc.audio_separated) return false;
+  if (isVideoClip(sc)) return Boolean(sc.source && sc.source.media_ref);
+  return Boolean(((p.scene_renders || {})[sc.id] || {}).media);
+};
+
+/** Pull a clip's sound onto its own lane. -> the lane, or false (nothing to pull, or already pulled). */
+export function separate(p, sceneId) {
+  const at = (p.scenes || []).findIndex((s) => s.id === sceneId), sc = p.scenes[at];
+  if (!hasEmbeddedAudio(sc, p) || trackFor(p, sceneId)) return false;
+  const seg = segments(p).find((s) => s.kind === "scene" && s.id === sceneId);
+  const dur = sc.source_dur != null ? sc.source_dur : seg.dur;
+  const render = (p.scene_renders || {})[sceneId];
+  const video = isVideoClip(sc);
+  const inSec = video ? sc.source_in || 0 : (render.inSec || 0) + (sc.source_in || 0);
+  const lane = { id: newId(), kind: "separated", scene_id: sceneId, start_sec: seg.start, source_in_sec: inSec, source_dur: dur,
+    pinned_media: video ? null : JSON.parse(JSON.stringify(render.media)), pinned_bin_ref: video ? sc.source.media_ref : null,
+    pinned_in_sec: inSec, pinned_dur: dur, volume: sc.audio_volume != null ? sc.audio_volume : 1, label: `${video ? "V" : "S"}${at + 1} audio` };
+  p.audio_tracks = [...(p.audio_tracks || []), lane];
+  sc.audio_separated = true;
+  sc.audio_volume = 0;
+  return lane;
+}
+
+/** Take a lane away; a separated one gives its clip its sound (and volume) back. */
+export function removeTrack(p, id) {
+  const t = (p.audio_tracks || []).find((x) => x.id === id);
+  if (!t) return false;
+  const sc = t.kind === "separated" && (p.scenes || []).find((s) => s.id === t.scene_id);
+  if (sc) { sc.audio_separated = false; if (t.volume != null) sc.audio_volume = t.volume; }
+  p.audio_tracks = p.audio_tracks.filter((x) => x.id !== id);
+  return true;
+}
+
+/** A separated lane sits where its clip sits. Run after every edit; changes nothing when nothing moved. */
+export function syncSeparated(p) {
+  const tracks = p.audio_tracks || [];
+  if (!tracks.some((t) => t.kind === "separated")) return;
+  const starts = new Map(segments(p).filter((s) => s.kind === "scene").map((s) => [s.id, s.start]));
+  for (const t of tracks) {
+    if (t.kind !== "separated" || !starts.has(t.scene_id)) continue;
+    const sc = p.scenes.find((s) => s.id === t.scene_id);
+    if (!sc.excluded && Math.abs((t.start_sec || 0) - starts.get(t.scene_id)) > 0.001) t.start_sec = starts.get(t.scene_id);
+  }
+}

app/ui/shell/project.js

@@ -53,7 +53,7 @@ const put = (project) => json("PUT", `${BASE}/${encodeURIComponent(project.id)}`
  * `onChange` fires for anything that alters what is on screen, including the
  * selection. Nothing here draws; the timeline subscribes.
  */
-export function createProject({ onChange, onError, onOpen } = {}) {
+export function createProject({ onChange, onError, onOpen, keepConsistent } = {}) {
   let project = null;
   let selected = null;
   // Every project this browser knows about, for the File menu. Read once at
@@ -83,7 +83,8 @@ export function createProject({ onChange, onError, onOpen } = {}) {
   // whatever the user is actually looking at now.
   let generation = 0;
 
-  const changed = () => { if (onChange) onChange(); };
+  // `keepConsistent(project)` restates what must follow from an edit (audio lanes follow their clips) before anyone is told.
+  const changed = () => { if (project && keepConsistent) keepConsistent(project); if (onChange) onChange(); };
 
   /** Before an edit. Anything that changes what is SAVED calls this; selecting
    *  a scene does not, because a view is not an edit. */
@@ -169,7 +170,6 @@ export function createProject({ onChange, onError, onOpen } = {}) {
 
   return {
     get project() { return project; },
-    get unsaved() { return dirty; },
     get scenes() { return project ? project.scenes || [] : []; },
     get selectedId() { return selected; },
     get selected() { return sceneAt(selected); },

app/ui/shell/tests/audio.test.js

@@ -0,0 +1,50 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { hasEmbeddedAudio, removeTrack, separate, syncSeparated, trackFor } from "../audio.js";
+
+const media = { filename: "a.mp4", subfolder: "", type: "output" };
+const proj = () => ({ num_frames_per_scene: 50, frame_rate: 25, scenes: [{ id: "a", text: "" }, { id: "b", text: "", audio_volume: 0.5 }, { id: "v", source: { type: "video", media_ref: "m1" }, source_dur: 3 }],
+  scene_renders: { a: { media, inSec: 0 }, b: { media, inSec: 1 } }, audio_tracks: [] });
+
+test("only a clip with a render or a bin video has sound to pull out", () => {
+  const p = proj();
+  assert.equal(hasEmbeddedAudio(p.scenes[0], p), true);
+  assert.equal(hasEmbeddedAudio(p.scenes[2], p), true);
+  assert.equal(hasEmbeddedAudio({ id: "x" }, p), false);
+  assert.equal(separate(p, "x"), false);
+});
+
+test("separate: the lane takes the clip's sound, place and volume; the clip goes quiet; a second time does nothing", () => {
+  const p = proj();
+  const lane = separate(p, "b");
+  assert.deepEqual([lane.kind, lane.scene_id, lane.start_sec, lane.source_in_sec, lane.source_dur, lane.volume, lane.label], ["separated", "b", 2, 1, 2, 0.5, "S2 audio"]);
+  assert.deepEqual(lane.pinned_media, media);
+  assert.deepEqual([p.scenes[1].audio_separated, p.scenes[1].audio_volume], [true, 0]);
+  assert.equal(separate(p, "b"), false);
+  assert.equal(trackFor(p, "b"), lane);
+});
+
+test("a bin video clip's lane points at the bin file", () => {
+  const p = proj();
+  const lane = separate(p, "v");
+  assert.deepEqual([lane.pinned_bin_ref, lane.pinned_media, lane.source_dur, lane.label], ["m1", null, 3, "V3 audio"]);
+});
+
+test("removing the lane gives the clip its sound and volume back", () => {
+  const p = proj();
+  const lane = separate(p, "b");
+  assert.equal(removeTrack(p, lane.id), true);
+  assert.deepEqual([p.scenes[1].audio_separated, p.scenes[1].audio_volume, p.audio_tracks.length], [false, 0.5, 0]);
+  assert.equal(removeTrack(p, "nope"), false);
+});
+
+test("a separated lane follows its clip when the cut changes, and an untouched project is left alone", () => {
+  const p = proj();
+  const lane = separate(p, "b");
+  p.scenes.unshift({ id: "n", text: "" });                 // a new first clip pushes everything 2 s later
+  syncSeparated(p);
+  assert.equal(lane.start_sec, 4);
+  const before = JSON.stringify(p);
+  syncSeparated(p);
+  assert.equal(JSON.stringify(p), before);
+});