diff --git a/app/ui/boot.js b/app/ui/boot.js index 8fff8c27..41f50b98 100644 --- a/app/ui/boot.js +++ b/app/ui/boot.js @@ -10,6 +10,7 @@ import { api } from "./shell/api.js"; import { createSelection } from "./shell/selection.js"; import { createPlayhead } from "./shell/playhead.js"; import { createMaintenance } from "./shell/maintenance.js"; +import { syncSeparated } from "./shell/audio.js"; import { createKeys } from "./shell/keys.js"; import { createDock } from "./shell/dock.js"; import { createMode } from "./shell/mode.js"; @@ -21,7 +22,7 @@ import paths from "./modules.js"; const heard = new Set(); const say = (what) => heard.forEach((fn) => { try { fn(what); } catch (err) { console.error(err); } }); const on = (fn) => { heard.add(fn); return () => heard.delete(fn); }; -const project = createProject({ onChange: () => say("change"), onOpen: () => say("open"), onError: (err) => console.warn(err) }); +const project = createProject({ keepConsistent: syncSeparated, onChange: () => say("change"), onOpen: () => say("open"), onError: (err) => console.warn(err) }); const pipeline = createPipelineState(api); const generate = createGenerate({ pipeline }); linkPipeline({ project, pipeline, onOpen: (fn) => on((what) => { if (what === "open") fn(); }) }); diff --git a/app/ui/features/audio/index.js b/app/ui/features/audio/index.js new file mode 100644 index 00000000..fd667bbb --- /dev/null +++ b/app/ui/features/audio/index.js @@ -0,0 +1,24 @@ +// Separate audio / Remove audio: pull the picked clip's own sound onto a lane of its own, or give it back. +import { composer as c } from "../../composer/composer.js"; +import { inPlace } from "../../shell/place.js"; +import { hasEmbeddedAudio, removeTrack, separate, trackFor } from "../../shell/audio.js"; + +export default { + id: "audio", + mount: "timeline.toolbar", + needs: ["project", "selection"], + setup({ host, app }) { + const p = app.project, sel = app.selection; + const put = inPlace(host); + const draw = () => { + const open = p.project, sc = open && sel.focus ? open.scenes.find((s) => s.id === sel.focus) : null; + const lane = open && sc ? trackFor(open, sc.id) : null; + put(c.button.sm({ label: "⊟ Separate audio", tone: "ghost", disabled: !(open && sc && hasEmbeddedAudio(sc, open)), title: "Detach this clip's audio onto its own track (the clip keeps picture only)", + onClick: () => p.edit((pr) => separate(pr, sel.focus)) }).node, + c.button.sm({ label: "Remove audio", tone: "danger", disabled: !lane, title: "Remove this clip's separated audio track", + onClick: () => p.edit((pr) => removeTrack(pr, lane.id)) }).node); + }; + draw(); + return app.on(draw); + }, +}; diff --git a/app/ui/features/placeholders/index.js b/app/ui/features/placeholders/index.js deleted file mode 100644 index d0193da7..00000000 --- a/app/ui/features/placeholders/index.js +++ /dev/null @@ -1,17 +0,0 @@ -// Stand-ins for the parts of v4's screen whose features are not ported yet: same place, same look, -// inert. A real feature that mounts at one of these points replaces its row here (delete the row). -import { composer as c } from "../../composer/composer.js"; - -const btn = (label, tone = "ghost", extra = {}) => c.button.sm({ label, tone, disabled: true, ...extra }); -const menu = (label) => c.button.menu({ label, tone: "ghost" }); // inert, but not dimmed: a menu is never greyed in v4 - -const items = { - "menubar.right": () => [c.chip.neutral({ label: "saved" }), c.chip.good({ label: "ComfyUI live", dot: true })], - - "timeline.toolbar": () => [btn("⊟ Separate audio"), btn("Remove audio")], -}; - -export default Object.entries(items).map(([mount, build]) => ({ - id: `placeholder:${mount}`, mount, - setup: ({ host }) => build().forEach((handle) => host.append(handle.node)), -})); diff --git a/app/ui/features/status/index.js b/app/ui/features/status/index.js new file mode 100644 index 00000000..7d6dffda --- /dev/null +++ b/app/ui/features/status/index.js @@ -0,0 +1,21 @@ +// The two chips at the far end of the menubar: whether the project is saved, and whether ComfyUI answers. +import { composer as c } from "../../composer/composer.js"; +import { inPlace } from "../../shell/place.js"; + +export default { + id: "status", + mount: "menubar.right", + needs: ["project", "api"], + setup({ host, app }) { + const put = inPlace(host); + let live = true; + const draw = () => put((app.project.unsaved ? c.chip.warn({ label: "unsaved" }) : c.chip.neutral({ label: "saved" })).node, + (live ? c.chip.good({ label: "ComfyUI live", dot: true }) : c.chip.danger({ label: "ComfyUI offline", dot: true })).node); + const ping = async () => { try { live = Boolean((await app.api.health()).ok); } catch { live = false; } draw(); }; + draw(); ping(); + const timers = [setInterval(() => { const now = Boolean(app.project.unsaved); if (now !== saved) { saved = now; draw(); } }, 600), setInterval(ping, 10000)]; + let saved = Boolean(app.project.unsaved); + const off = app.on(() => { saved = Boolean(app.project.unsaved); draw(); }); + return () => { timers.forEach(clearInterval); off(); }; + }, +}; diff --git a/app/ui/features/timeline/index.js b/app/ui/features/timeline/index.js index ff550e62..5f68594b 100644 --- a/app/ui/features/timeline/index.js +++ b/app/ui/features/timeline/index.js @@ -5,7 +5,7 @@ import { composer as c } from "../../composer/composer.js"; import { segments, totalSeconds, clock } from "../../shell/scenes.js"; import { inPlace } from "../../shell/place.js"; import * as edits from "../../shell/edits.js"; -import { videoLane, audioLane, framesAt } from "./lanes.js"; +import { videoLane, audioLane, tracksLane, framesAt } from "./lanes.js"; const ZOOM = [20, 40, 80, 160, 320]; const sceneIds = (p) => segments(p).filter((s) => s.kind === "scene").map((s) => s.id); @@ -37,7 +37,7 @@ const stage = { const ghostActions = (g) => [{ label: "✕", title: "Remove from the timeline", onClick: () => edit((pr) => { pr.scene_ghosts = (pr.scene_ghosts || []).filter((x) => x.id !== g.id); return true; }) }]; - const draw = () => { const open = p.project; view.setLanes(open ? [videoLane(open, sel.ids, sel.focus, clipActions, ghostActions), audioLane(open, sel.ids)] : []); }; + const draw = () => { const open = p.project; view.setLanes(open ? [videoLane(open, sel.ids, sel.focus, clipActions, ghostActions), audioLane(open, sel.ids), tracksLane(open)].filter(Boolean) : []); }; const setZoom = (z) => { zoom = Math.min(ZOOM.length - 1, Math.max(0, z)); view.setZoom(ZOOM[zoom]); }; host.append(view.node); const off = [app.on((what) => { diff --git a/app/ui/features/timeline/lanes.js b/app/ui/features/timeline/lanes.js index 61a98c2f..07e3cfaf 100644 --- a/app/ui/features/timeline/lanes.js +++ b/app/ui/features/timeline/lanes.js @@ -31,3 +31,9 @@ export const audioLane = (p, picked) => ({ id: "audio", label: "Original", kind: "audio", clips: segments(p).filter((s) => s.kind === "scene").map((s, i) => ({ id: `a:${s.id}`, start: s.start, dur: s.dur, title: `S${i + 1}`, selected: picked.includes(s.id), excluded: Boolean(s.scene.excluded) })), }); + +/** Audio lanes added to the project (a clip's separated sound, or a file laid over the cut), when there are any. */ +export const tracksLane = (p) => (p.audio_tracks || []).length ? { + id: "tracks", label: "Audio", kind: "audio", + clips: p.audio_tracks.map((t) => ({ id: `t:${t.id}`, start: t.start_sec || 0, dur: t.pinned_dur ?? t.source_dur ?? 1, title: t.label || "Audio" })), +} : null; diff --git a/app/ui/modules.js b/app/ui/modules.js index cc108862..ae7b3219 100644 --- a/app/ui/modules.js +++ b/app/ui/modules.js @@ -14,8 +14,9 @@ export default [ "./features/menus/index.js", "./features/render/index.js", "./features/render/selected.js", + "./features/status/index.js", + "./features/audio/index.js", "./features/montage/index.js", - "./features/placeholders/index.js", "./features/generate/warn.js", "./features/composer/index.js", "./features/settings/index.js", diff --git a/app/ui/shell/audio.js b/app/ui/shell/audio.js new file mode 100644 index 00000000..6662dfb8 --- /dev/null +++ b/app/ui/shell/audio.js @@ -0,0 +1,54 @@ +// Audio lanes. A clip's own sound can be pulled onto a lane of its own ("separated"): the lane plays the sound pinned from +// that clip's picture, and the clip itself goes quiet. A separated lane follows its clip around the timeline. +import { segments, isVideoClip } from "./scenes.js"; + +const newId = () => "t" + Math.random().toString(36).slice(2, 10) + Date.now().toString(36); + +export const trackFor = (p, sceneId) => (p.audio_tracks || []).find((t) => t.kind === "separated" && t.scene_id === sceneId); + +/** Does this clip have sound to pull out: a bin video clip, or a generated clip with a render. */ +export const hasEmbeddedAudio = (sc, p) => { + if (!sc || sc.excluded || sc.audio_separated) return false; + if (isVideoClip(sc)) return Boolean(sc.source && sc.source.media_ref); + return Boolean(((p.scene_renders || {})[sc.id] || {}).media); +}; + +/** Pull a clip's sound onto its own lane. -> the lane, or false (nothing to pull, or already pulled). */ +export function separate(p, sceneId) { + const at = (p.scenes || []).findIndex((s) => s.id === sceneId), sc = p.scenes[at]; + if (!hasEmbeddedAudio(sc, p) || trackFor(p, sceneId)) return false; + const seg = segments(p).find((s) => s.kind === "scene" && s.id === sceneId); + const dur = sc.source_dur != null ? sc.source_dur : seg.dur; + const render = (p.scene_renders || {})[sceneId]; + const video = isVideoClip(sc); + const inSec = video ? sc.source_in || 0 : (render.inSec || 0) + (sc.source_in || 0); + const lane = { id: newId(), kind: "separated", scene_id: sceneId, start_sec: seg.start, source_in_sec: inSec, source_dur: dur, + pinned_media: video ? null : JSON.parse(JSON.stringify(render.media)), pinned_bin_ref: video ? sc.source.media_ref : null, + pinned_in_sec: inSec, pinned_dur: dur, volume: sc.audio_volume != null ? sc.audio_volume : 1, label: `${video ? "V" : "S"}${at + 1} audio` }; + p.audio_tracks = [...(p.audio_tracks || []), lane]; + sc.audio_separated = true; + sc.audio_volume = 0; + return lane; +} + +/** Take a lane away; a separated one gives its clip its sound (and volume) back. */ +export function removeTrack(p, id) { + const t = (p.audio_tracks || []).find((x) => x.id === id); + if (!t) return false; + const sc = t.kind === "separated" && (p.scenes || []).find((s) => s.id === t.scene_id); + if (sc) { sc.audio_separated = false; if (t.volume != null) sc.audio_volume = t.volume; } + p.audio_tracks = p.audio_tracks.filter((x) => x.id !== id); + return true; +} + +/** A separated lane sits where its clip sits. Run after every edit; changes nothing when nothing moved. */ +export function syncSeparated(p) { + const tracks = p.audio_tracks || []; + if (!tracks.some((t) => t.kind === "separated")) return; + const starts = new Map(segments(p).filter((s) => s.kind === "scene").map((s) => [s.id, s.start])); + for (const t of tracks) { + if (t.kind !== "separated" || !starts.has(t.scene_id)) continue; + const sc = p.scenes.find((s) => s.id === t.scene_id); + if (!sc.excluded && Math.abs((t.start_sec || 0) - starts.get(t.scene_id)) > 0.001) t.start_sec = starts.get(t.scene_id); + } +} diff --git a/app/ui/shell/project.js b/app/ui/shell/project.js index c6c67f82..001a8e97 100644 --- a/app/ui/shell/project.js +++ b/app/ui/shell/project.js @@ -53,7 +53,7 @@ const put = (project) => json("PUT", `${BASE}/${encodeURIComponent(project.id)}` * `onChange` fires for anything that alters what is on screen, including the * selection. Nothing here draws; the timeline subscribes. */ -export function createProject({ onChange, onError, onOpen } = {}) { +export function createProject({ onChange, onError, onOpen, keepConsistent } = {}) { let project = null; let selected = null; // Every project this browser knows about, for the File menu. Read once at @@ -83,7 +83,8 @@ export function createProject({ onChange, onError, onOpen } = {}) { // whatever the user is actually looking at now. let generation = 0; - const changed = () => { if (onChange) onChange(); }; + // `keepConsistent(project)` restates what must follow from an edit (audio lanes follow their clips) before anyone is told. + const changed = () => { if (project && keepConsistent) keepConsistent(project); if (onChange) onChange(); }; /** Before an edit. Anything that changes what is SAVED calls this; selecting * a scene does not, because a view is not an edit. */ @@ -169,7 +170,6 @@ export function createProject({ onChange, onError, onOpen } = {}) { return { get project() { return project; }, - get unsaved() { return dirty; }, get scenes() { return project ? project.scenes || [] : []; }, get selectedId() { return selected; }, get selected() { return sceneAt(selected); }, diff --git a/app/ui/shell/tests/audio.test.js b/app/ui/shell/tests/audio.test.js new file mode 100644 index 00000000..2d649250 --- /dev/null +++ b/app/ui/shell/tests/audio.test.js @@ -0,0 +1,50 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { hasEmbeddedAudio, removeTrack, separate, syncSeparated, trackFor } from "../audio.js"; + +const media = { filename: "a.mp4", subfolder: "", type: "output" }; +const proj = () => ({ num_frames_per_scene: 50, frame_rate: 25, scenes: [{ id: "a", text: "" }, { id: "b", text: "", audio_volume: 0.5 }, { id: "v", source: { type: "video", media_ref: "m1" }, source_dur: 3 }], + scene_renders: { a: { media, inSec: 0 }, b: { media, inSec: 1 } }, audio_tracks: [] }); + +test("only a clip with a render or a bin video has sound to pull out", () => { + const p = proj(); + assert.equal(hasEmbeddedAudio(p.scenes[0], p), true); + assert.equal(hasEmbeddedAudio(p.scenes[2], p), true); + assert.equal(hasEmbeddedAudio({ id: "x" }, p), false); + assert.equal(separate(p, "x"), false); +}); + +test("separate: the lane takes the clip's sound, place and volume; the clip goes quiet; a second time does nothing", () => { + const p = proj(); + const lane = separate(p, "b"); + assert.deepEqual([lane.kind, lane.scene_id, lane.start_sec, lane.source_in_sec, lane.source_dur, lane.volume, lane.label], ["separated", "b", 2, 1, 2, 0.5, "S2 audio"]); + assert.deepEqual(lane.pinned_media, media); + assert.deepEqual([p.scenes[1].audio_separated, p.scenes[1].audio_volume], [true, 0]); + assert.equal(separate(p, "b"), false); + assert.equal(trackFor(p, "b"), lane); +}); + +test("a bin video clip's lane points at the bin file", () => { + const p = proj(); + const lane = separate(p, "v"); + assert.deepEqual([lane.pinned_bin_ref, lane.pinned_media, lane.source_dur, lane.label], ["m1", null, 3, "V3 audio"]); +}); + +test("removing the lane gives the clip its sound and volume back", () => { + const p = proj(); + const lane = separate(p, "b"); + assert.equal(removeTrack(p, lane.id), true); + assert.deepEqual([p.scenes[1].audio_separated, p.scenes[1].audio_volume, p.audio_tracks.length], [false, 0.5, 0]); + assert.equal(removeTrack(p, "nope"), false); +}); + +test("a separated lane follows its clip when the cut changes, and an untouched project is left alone", () => { + const p = proj(); + const lane = separate(p, "b"); + p.scenes.unshift({ id: "n", text: "" }); // a new first clip pushes everything 2 s later + syncSeparated(p); + assert.equal(lane.start_sec, 4); + const before = JSON.stringify(p); + syncSeparated(p); + assert.equal(JSON.stringify(p), before); +});