From 01fc0e4a71292cf809f047bcd873d00e7f5da0df Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Sun, 4 Oct 2026 03:00:12 +0300 Subject: [PATCH] feat(ui): clip effects + transitions, original-sound volume, video/audio import, file-lane slide/trim, audio lanes heard on the monitor, wheel customisation; fix: timeline syntax break; render: separated lane starts at its clip's place in the render Co-Authored-By: Claude Sonnet 5.5 --- app/ui/boot.js | 2 +- app/ui/features/clipfx/index.js | 59 ++++++++++++++++++++++ app/ui/features/imports/index.js | 47 +++++++++++++++++ app/ui/features/inspector/index.js | 5 +- app/ui/features/preview/index.js | 5 ++ app/ui/features/preview/lanes.js | 47 +++++++++++++++++ app/ui/features/preview/lanes.test.js | 11 ++++ app/ui/features/timeline/index.js | 2 +- app/ui/features/timeline/lanes.js | 6 +-- app/ui/features/wheel/index.js | 26 ++++++++-- app/ui/modules.js | 3 ++ app/ui/shell/api.js | 1 + app/ui/shell/audio.js | 21 ++++++-- app/ui/shell/clipfx.js | 46 +++++++++++++++++ app/ui/shell/imports.js | 23 +++++++++ app/ui/shell/playhead.js | 1 + app/ui/shell/tests/audio.test.js | 12 +++++ app/ui/shell/tests/clipfx.test.js | 22 ++++++++ app/ui/shell/tests/imports.test.js | 20 ++++++++ app/ui/shell/tests/syntax.test.js | 16 ++++++ modules/output/render/stitch.py | 16 +++++- modules/output/render/tests/test_stitch.py | 13 +++++ 22 files changed, 389 insertions(+), 15 deletions(-) create mode 100644 app/ui/features/clipfx/index.js create mode 100644 app/ui/features/imports/index.js create mode 100644 app/ui/features/preview/lanes.js create mode 100644 app/ui/features/preview/lanes.test.js create mode 100644 app/ui/shell/clipfx.js create mode 100644 app/ui/shell/imports.js create mode 100644 app/ui/shell/tests/clipfx.test.js create mode 100644 app/ui/shell/tests/imports.test.js create mode 100644 app/ui/shell/tests/syntax.test.js diff --git a/app/ui/boot.js b/app/ui/boot.js index 1bf5c4ab..d0e349c5 100644 --- a/app/ui/boot.js +++ b/app/ui/boot.js @@ -29,7 +29,7 @@ linkPipeline({ project, pipeline, onOpen: (fn) => on((what) => { if (what === "o const frame = build(document.getElementById("app")); const selection = createSelection({ project }); selection.on(() => say("select")); -const app = { editorSettings: [], timelineLanes: [], has: new Set(), actions: [], inputHooks: [], lastRun: { typed: undefined, n: 0 }, project, pipeline, generate, api, on, say, title: frame.setZoneTitle, selection, playhead: createPlayhead(), theme: window.ComposerTheme, mode: createMode(), panel: frame.togglePanel, closePanels: frame.closePanels, dock: createDock(), maintenance: createMaintenance({ api, flush: async () => { await project.flush(); if (project.unsaved) throw new Error("Your latest edits could not be saved, so nothing was changed. Try again in a moment."); } }), keys: createKeys(), resetLayout: () => { frame.resetLayout(); app.dock.reset(); } }; +const app = { editorSettings: [], timelineLanes: [], sceneSections: [], clipTags: [], has: new Set(), actions: [], inputHooks: [], lastRun: { typed: undefined, n: 0 }, project, pipeline, generate, api, on, say, title: frame.setZoneTitle, selection, playhead: createPlayhead(), theme: window.ComposerTheme, mode: createMode(), panel: frame.togglePanel, closePanels: frame.closePanels, dock: createDock(), maintenance: createMaintenance({ api, flush: async () => { await project.flush(); if (project.unsaved) throw new Error("Your latest edits could not be saved, so nothing was changed. Try again in a moment."); } }), keys: createKeys(), resetLayout: () => { frame.resetLayout(); app.dock.reset(); } }; const settings = createSettings({ menubar: hostFor("menubar.menus") }); app.openSettings = (id) => settings.open(id); const result = await loadFeatures(paths, { app }); diff --git a/app/ui/features/clipfx/index.js b/app/ui/features/clipfx/index.js new file mode 100644 index 00000000..fb7ab3fc --- /dev/null +++ b/app/ui/features/clipfx/index.js @@ -0,0 +1,59 @@ +// Clip effects and seam transitions: "+ Effect" on the timeline toolbar, the tags on a clip saying what is on it, and the +// original-sound volume under the scene's fields. The render reads scene.effects / video_transition / audio_volume. +import { composer as c } from "../../composer/composer.js"; +import { applyEffect, applyTransition, clearTransition, tags } from "../../shell/clipfx.js"; + +export default { + id: "clipfx", + mount: "timeline.toolbar", + needs: ["project", "api", "selection"], + setup({ host, app }) { + const p = app.project; + let library = null; + const load = async () => (library ||= await app.api.renderLibrary()); + + async function ask(kind) { // kind: "effects" | "video_transitions" + const sc = p.selected; + if (!sc) return c.toast.warn({ text: "Pick a clip first." }); + let items; + try { items = (await load())[kind] || []; } catch (err) { return c.toast.warn({ text: `The effects list could not be loaded: ${err.message}` }); } + if (!items.length) return c.toast.warn({ text: "Nothing to offer: the list is empty." }); + const isFx = kind === "effects"; + let item = items[0], value = item.param ? item.param.default : null; + const slot = c.region.stack({ gap: "sm" }); + const draw = () => slot.set([ + c.field.default({ label: isFx ? "Effect" : "Transition", hint: item.description, control: c.select.md({ label: isFx ? "Effect" : "Transition", value: item.id, options: items.map((i) => ({ value: i.id, label: i.name || i.id })), + onChange: (v) => { item = items.find((i) => i.id === v); value = item.param ? item.param.default : null; draw(); } }) }), + item.param ? c.field.default({ label: item.param.label, control: c.number.md({ label: item.param.label, value, min: item.param.min, max: item.param.max, step: item.param.step, onChange: (v) => { value = v; } }) }) : null, + ].filter(Boolean)); + draw(); + const win = c.modal.generic({ title: isFx ? "Add effect" : "Add transition", subtitle: isFx ? "On the picked clip." : "A blend on the picked clip's outgoing edge.", size: "sm", body: slot }); + win.setFooter({ actions: [c.button.sm({ label: "Cancel", tone: "ghost", onClick: () => win.close("cancel") }), + c.button.sm({ label: "Apply", tone: "primary", onClick: () => { + const ok = p.edit((pr) => { const s = pr.scenes.find((x) => x.id === sc.id); return s && (isFx ? applyEffect(s, item.id, value) : applyTransition(s, item, value)); }); + if (!ok) return c.toast.warn({ text: "Could not apply that." }); + win.close("done"); + } })] }); + } + + const menu = c.button.menu({ label: "+ Effect", tone: "ghost", onClick: () => c.menu.dropdown({ anchor: menu, items: [ + { id: "fx", label: "Effect…", disabled: !p.selected }, { id: "tr", label: "Transition…", disabled: !p.selected }, + { separator: true }, { id: "off", label: "Remove effects and transition", danger: true, disabled: !p.selected || !tags(p.selected).length }], + onPick: (id) => { + if (id === "fx") ask("effects"); else if (id === "tr") ask("video_transitions"); + else p.edit((pr) => { const s = pr.scenes.find((x) => x.id === p.selectedId); if (!s) return false; applyEffect(s, "reset"); return clearTransition(s); }); + } }) }); + host.append(menu.node); + + app.clipTags.push(tags); + const section = { key: (sc) => `${sc.audio_volume}|${sc.audio_separated}`, rows: (sc) => [ + c.label.section({ text: "Sound" }), + c.field.default({ label: "Original sound volume", hint: sc.audio_separated ? "This clip's sound is on its own audio track: set the volume there." : "How loud this clip's own sound is in the render.", + control: c.slider.readout({ label: "Original sound volume", value: sc.audio_volume != null ? sc.audio_volume : 1, min: 0, max: 1, step: 0.05, precision: 2, disabled: Boolean(sc.audio_separated), onCommit: (v) => p.setScene(sc.id, "audio_volume", v) }) })] }; + app.sceneSections.push(section); + return () => { + for (const [list, x] of [[app.clipTags, tags], [app.sceneSections, section]]) { const i = list.indexOf(x); if (i >= 0) list.splice(i, 1); } + menu.node.remove(); + }; + }, +}; diff --git a/app/ui/features/imports/index.js b/app/ui/features/imports/index.js new file mode 100644 index 00000000..0fa24659 --- /dev/null +++ b/app/ui/features/imports/index.js @@ -0,0 +1,47 @@ +// "+ Import" on the timeline toolbar: a video from the media bin as a clip of its own, or a sound as an audio track at the playhead. +import { composer as c } from "../../composer/composer.js"; +import { addVideoClip, addAudioTrack } from "../../shell/imports.js"; +import { learn } from "../../shell/bin.js"; + +/** How long a bin file is, from the browser's own reading of it (null when it cannot say). */ +const lengthOf = (asset) => new Promise((resolve) => { + const m = document.createElement(asset.kind === "audio" ? "audio" : "video"); + const done = (v) => { m.removeAttribute("src"); m.load(); resolve(v); }; + m.preload = "metadata"; + m.onloadedmetadata = () => done(Number.isFinite(m.duration) ? m.duration : null); + m.onerror = () => done(null); + setTimeout(() => done(null), 8000); + m.src = `/funpack/api/media/${encodeURIComponent(asset.id)}/file`; +}); + +export default { + id: "imports", + mount: "timeline.toolbar", + needs: ["project", "api", "playhead"], + setup({ host, app }) { + const p = app.project; + + async function choose(kind) { + let bin; + try { const all = (await app.api.media()).media || []; learn(all); bin = all.filter((m) => m.kind === kind); } catch (err) { return c.toast.warn({ text: `Could not read the media bin: ${err.message}` }); } + const win = c.modal.generic({ title: kind === "video" ? "Add video clip" : "Add audio track", size: "sm", body: c.region.stack({ gap: "sm", children: [ + c.hint.default({ text: kind === "video" ? "Video clips play as they are and are skipped by Generate." : "The sound starts at the playhead." }), + bin.length ? c.checklist.default({ label: "Media", items: bin.map((m) => ({ value: m.id, label: m.name })), values: [], onChange: (v) => { const id = v[v.length - 1]; if (id) { win.close("done"); add(bin.find((m) => m.id === id)); } } }) + : c.hint.default({ text: `The media bin has no ${kind} yet: drop one onto the Assets panel first.` })] }) }); + } + + async function add(asset) { + const seconds = await lengthOf(asset), at = app.playhead.at; + if (!p.project) return; + const made = p.edit((pr) => (asset.kind === "video" ? addVideoClip(pr, asset, seconds) : addAudioTrack(pr, asset, at, seconds))); + if (!made) return c.toast.warn({ text: "Could not add that." }); + if (asset.kind === "video") p.select(p.scenes[p.scenes.length - 1].id); + if (seconds == null) c.toast.warn({ text: "The file's length could not be read; the clip uses the project's length." }); + } + + const menu = c.button.menu({ label: "+ Import", tone: "ghost", onClick: () => c.menu.dropdown({ anchor: menu, items: [{ id: "video", label: "Video clip…", disabled: !p.project }, { id: "audio", label: "Audio track…", disabled: !p.project }], + onPick: (id) => choose(id) }) }); + host.append(menu.node); + return () => menu.node.remove(); + }, +}; diff --git a/app/ui/features/inspector/index.js b/app/ui/features/inspector/index.js index 495fdbf2..0695ede5 100644 --- a/app/ui/features/inspector/index.js +++ b/app/ui/features/inspector/index.js @@ -23,13 +23,14 @@ export default { const unit = p.scenes.filter((s) => genUnitId(s) === genUnitId(sc)); app.title && app.title("inspector", `Scene · ${p.scenes.indexOf(sc) + 1}${unit.length > 1 ? ` · cut ${unit.indexOf(sc) + 1}/${unit.length}` : ""}`); }; + const sections = app.sceneSections || []; // extra rows other features put under the scene's own: { rows(scene, project), key(scene, project) } function draw(force) { // not on every keystroke: that would rebuild the box being typed in const sc = p.selected, open = p.project; - const next = `${tab}|${open && open.id}|${p.selectedId}|${tab === "project" && open ? [open.postfix_enabled !== false, open.generation_mode] : ""}|${tab === "scene" && sc && open ? [effFrames(sc, open), effFps(sc, open), p.scenes.indexOf(sc), sc.frames_mode, sc.fps_mode, p.scenes.length, sc.source_image, (sc.references || []).join(), bin.version] : ""}`; + const next = `${tab}|${open && open.id}|${p.selectedId}|${tab === "project" && open ? [open.postfix_enabled !== false, open.generation_mode] : ""}|${tab === "scene" && sc && open ? [effFrames(sc, open), effFps(sc, open), p.scenes.indexOf(sc), sc.frames_mode, sc.fps_mode, p.scenes.length, sc.source_image, (sc.references || []).join(), bin.version, ...sections.map((s) => s.key(sc, open))] : ""}`; titled(); if (!force && next === key) return; key = next; - body.set((tab === "scene" ? sceneRows(p, app) : projectRows(p, app.pipeline && app.pipeline.slots(), app.api))); + body.set((tab === "scene" ? [...sceneRows(p, app), ...(p.selected ? sections.flatMap((s) => s.rows(p.selected, p)) : [])] : projectRows(p, app.pipeline && app.pipeline.slots(), app.api))); } draw(true); const offSlots = app.pipeline && app.pipeline.subscribe ? app.pipeline.subscribe(() => draw(true)) : null; // the pipeline arrives after the project diff --git a/app/ui/features/preview/index.js b/app/ui/features/preview/index.js index 2d0abe59..f5f0d11e 100644 --- a/app/ui/features/preview/index.js +++ b/app/ui/features/preview/index.js @@ -62,7 +62,12 @@ export default { function sync() { seg = segAt(head.at); show(seg); + head.playing = playing; const v = viewer.element; + if (v && seg) { // the clip's own sound: its volume, nothing when moved to a lane of its own or dropped by the project + const sc = seg.scene, vol = sc.audio_separated || (open() || {}).keep_original_audio === false ? 0 : Math.max(0, Math.min(1, sc.audio_volume != null ? +sc.audio_volume : 1)); + if (Number.isFinite(vol) && v.volume !== vol) v.volume = vol; + } if (v && seg && !quiet) { const want = from + head.at - seg.start; if (Math.abs(v.currentTime - want) > 0.25) v.currentTime = want; diff --git a/app/ui/features/preview/lanes.js b/app/ui/features/preview/lanes.js new file mode 100644 index 00000000..821a1512 --- /dev/null +++ b/app/ui/features/preview/lanes.js @@ -0,0 +1,47 @@ +// The audio lanes on the monitor: each lane's sound plays while the playhead is inside it and the monitor is playing. + +const view = (m) => `/view?filename=${encodeURIComponent(m.filename || "")}&subfolder=${encodeURIComponent(m.subfolder || "")}&type=${encodeURIComponent(m.type || "output")}`; +const bin = (id) => `/funpack/api/media/${encodeURIComponent(id)}/file`; + +/** Where a lane's sound comes from, and which part of it plays: {url, from, dur} or null. */ +export function laneSource(t, open) { + if (t.kind === "separated") { + const url = t.pinned_bin_ref ? bin(t.pinned_bin_ref) : t.pinned_media && t.pinned_media.filename ? view(t.pinned_media) : null; + return url && { url, from: t.pinned_in_sec ?? t.source_in_sec ?? 0, dur: t.pinned_dur ?? t.source_dur ?? 0 }; + } + if (t.media_ref) return { url: bin(t.media_ref), from: t.source_in_sec || 0, dur: t.source_dur != null ? t.source_dur : 86400 }; + return null; +} + +export default { + id: "preview-lanes", + mount: "preview", + needs: ["project", "playhead"], + setup({ app }) { + const p = app.project, head = app.playhead, pool = new Map(); + const stopAll = () => pool.forEach((a) => a.pause()); + function tick() { + const open = p.project, live = new Set(); + if (open && head.playing) { + for (const t of open.audio_tracks || []) { + const src = laneSource(t, open); + if (!src) continue; + live.add(t.id); + let a = pool.get(t.id); + if (!a) { a = new Audio(); a.preload = "auto"; pool.set(t.id, a); } + if (a.dataset.src !== src.url) { a.dataset.src = src.url; a.src = src.url; } + const start = t.start_sec || 0, within = head.at - start, inside = within >= 0 && within < src.dur; + const vol = Math.max(0, Math.min(1, t.volume != null ? +t.volume : 1)); + if (a.volume !== vol && Number.isFinite(vol)) a.volume = vol; + if (!inside) { if (!a.paused) a.pause(); continue; } + if (Math.abs(a.currentTime - (src.from + within)) > 0.3) a.currentTime = src.from + within; + if (a.paused) a.play().catch(() => {}); + } + } + for (const [id, a] of pool) if (!live.has(id)) { a.pause(); if (!open || !(open.audio_tracks || []).some((t) => t.id === id)) { a.removeAttribute("src"); a.load(); delete a.dataset.src; pool.delete(id); } } // a lane that is gone gives its connection back + } + const off = [head.on(tick), app.on(() => tick())]; + const timer = setInterval(tick, 200); // the playhead only moves while playing; this catches pause/stop + return () => { off.forEach((f) => f()); clearInterval(timer); stopAll(); pool.forEach((a) => { a.removeAttribute("src"); a.load(); }); pool.clear(); }; + }, +}; diff --git a/app/ui/features/preview/lanes.test.js b/app/ui/features/preview/lanes.test.js new file mode 100644 index 00000000..5b095b4a --- /dev/null +++ b/app/ui/features/preview/lanes.test.js @@ -0,0 +1,11 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { laneSource } from "./lanes.js"; + +test("a file lane plays from its in-point; a separated lane from its pinned bin file or ComfyUI render; none without a source", () => { + assert.deepEqual(laneSource({ media_ref: "m", source_in_sec: 1, source_dur: 2 }), { url: "/funpack/api/media/m/file", from: 1, dur: 2 }); + assert.deepEqual(laneSource({ kind: "separated", pinned_bin_ref: "v", pinned_in_sec: 0.5, pinned_dur: 3 }), { url: "/funpack/api/media/v/file", from: 0.5, dur: 3 }); + assert.match(laneSource({ kind: "separated", pinned_media: { filename: "a b.mp4", type: "output" }, pinned_dur: 1 }).url, /^\/view\?filename=a%20b\.mp4&subfolder=&type=output$/); + assert.equal(laneSource({ kind: "separated" }), null); + assert.equal(laneSource({}), null); +}); diff --git a/app/ui/features/timeline/index.js b/app/ui/features/timeline/index.js index d9049828..bdee1fd9 100644 --- a/app/ui/features/timeline/index.js +++ b/app/ui/features/timeline/index.js @@ -44,7 +44,7 @@ const stage = { const ghostActions = (g) => [{ label: "✕", title: "Remove from the timeline", onClick: () => edit((pr) => { pr.scene_ghosts = (pr.scene_ghosts || []).filter((x) => x.id !== g.id); return true; }) }]; - const draw = () => { const open = p.project; view.setLanes(open ? [videoLane(open, sel.ids, sel.focus, clipActions, ghostActions), audioLane(open, sel.ids), tracksLane(open), ...extra.flatMap((e) => { try { return e.lanes(open); } catch { return []; } })] // a feature's lanes failing must not stop the baseline's.filter(Boolean) : []); }; + const draw = () => { const open = p.project; view.setLanes(open ? [videoLane(open, sel.ids, sel.focus, clipActions, ghostActions, (sc) => (app.clipTags || []).flatMap((f) => { try { return f(sc); } catch { return []; } })), audioLane(open, sel.ids), tracksLane(open), ...extra.flatMap((e) => { try { return e.lanes(open); } catch { return []; } })].filter(Boolean) : []); }; // a feature's lanes failing must not stop the baseline's const setZoom = (z) => { zoom = Math.min(ZOOM.length - 1, Math.max(0, z)); view.setZoom(ZOOM[zoom]); }; host.append(view.node); const off = [app.on((what) => { diff --git a/app/ui/features/timeline/lanes.js b/app/ui/features/timeline/lanes.js index edaea026..02220c93 100644 --- a/app/ui/features/timeline/lanes.js +++ b/app/ui/features/timeline/lanes.js @@ -3,7 +3,7 @@ import { segments, isVideoClip, effFps } from "../../shell/scenes.js"; const first = (t) => ((t || "").split("\n")[0] || "").slice(0, 80); -export function videoLane(p, picked, focus, actions = () => [], ghostActions = () => []) { +export function videoLane(p, picked, focus, actions = () => [], ghostActions = () => [], tags = () => []) { const renders = p.scene_renders || {}; return { id: "video", label: "Video", kind: "video", reorder: true, @@ -14,7 +14,7 @@ export function videoLane(p, picked, focus, actions = () => [], ghostActions = ( const rendered = Boolean(renders[sc.id]); return { id: sc.id, start: seg.start, dur: seg.dur, trim: true, - head: [isVideoClip(sc) ? "▶ video" : rendered ? "✓" : "◌", `${seg.dur.toFixed(1)}s`], + head: [isVideoClip(sc) ? "▶ video" : rendered ? "✓" : "◌", `${seg.dur.toFixed(1)}s`, ...tags(sc)], title: first(sc.text) || "(empty scene)", selected: picked.includes(sc.id), focus: sc.id === focus, excluded: Boolean(sc.excluded), actions: actions(sc), @@ -35,5 +35,5 @@ export const audioLane = (p, picked) => ({ /** Audio lanes added to the project (a clip's separated sound, or a file laid over the cut), when there are any. */ export const tracksLane = (p) => (p.audio_tracks || []).length ? { id: "tracks", label: "Audio", kind: "audio", move: true, - clips: p.audio_tracks.map((t) => ({ id: `t:${t.id}`, start: t.start_sec || 0, dur: t.pinned_dur ?? t.source_dur ?? 1, title: t.label || "Audio", trim: t.kind === "separated" })), + clips: p.audio_tracks.map((t) => ({ id: `t:${t.id}`, start: t.start_sec || 0, dur: t.pinned_dur ?? t.source_dur ?? 1, title: t.label || "Audio", trim: t.kind === "separated" ? t.pinned_dur != null : t.source_dur != null })), } : null; diff --git a/app/ui/features/wheel/index.js b/app/ui/features/wheel/index.js index 1e832656..9adfac21 100644 --- a/app/ui/features/wheel/index.js +++ b/app/ui/features/wheel/index.js @@ -2,6 +2,14 @@ import { composer as c } from "../../composer/composer.js"; import { offer } from "../../shell/actions.js"; +// Per browser, not per project: the wheel is a habit of the hand. enabled null = everything offered. +const KEY = "funpack_wheel"; +export function readPrefs() { + try { const p = JSON.parse(localStorage.getItem(KEY)) || {}; return { shape: p.shape === "half" ? "half" : "full", side: p.side === "left" ? "left" : "right", enabled: Array.isArray(p.enabled) ? p.enabled : null }; } + catch { return { shape: "full", side: "right", enabled: null }; } +} +const writePrefs = (p) => { try { localStorage.setItem(KEY, JSON.stringify(p)); } catch { /* private window: lasts until reload */ } }; + export default { id: "wheel", mount: "menubar.menus", @@ -17,10 +25,22 @@ export default { const onDown = (e) => { if (e.button !== 1 || wheel || document.querySelector('[role="dialog"]')) return; // nothing behind a dialog e.preventDefault(); - const items = (app.actions || []).filter((a) => !a.when || a.when()).slice(0, 12).map((a) => ({ id: a.id, label: a.label, icon: a.icon })); - wheel = c.wheel.picker({ items, x: e.clientX, y: e.clientY, onClose: () => { wheel = null; }, - onPick: (item) => { const a = (app.actions || []).find((x) => x.id === item.id); if (a) Promise.resolve().then(() => a.run()).catch((err) => c.toast.warn({ text: err.message })); } }); + const prefs = readPrefs(), on = (a) => !prefs.enabled || prefs.enabled.includes(a.id); + const items = (app.actions || []).filter((a) => on(a) && (!a.when || a.when())).slice(0, 12).map((a) => ({ id: a.id, label: a.label, icon: a.icon })); + if (!items.length) return; + const shared = { items, onClose: () => { wheel = null; }, + onPick: (item) => { const a = (app.actions || []).find((x) => x.id === item.id); if (a) Promise.resolve().then(() => a.run()).catch((err) => c.toast.warn({ text: err.message })); } }; + wheel = prefs.shape === "half" ? c.wheel.half({ ...shared, edge: prefs.side, at: e.clientY }) : c.wheel.picker({ ...shared, x: e.clientX, y: e.clientY }); }; + app.editorSettings.push(() => { + const prefs = readPrefs(), set = (patch) => writePrefs({ ...readPrefs(), ...patch }); + const offered = (app.actions || []).map((a) => ({ value: a.id, label: `${a.icon || ""} ${a.label}`.trim() })); + return c.region.stack({ gap: "sm", children: [c.label.section({ text: "Picker wheel" }), + c.hint.default({ text: "Hold the middle mouse button anywhere, flick toward an action and let go." }), + c.segmented.md({ label: "Shape", value: prefs.shape, onChange: (v) => set({ shape: v }), options: [{ value: "full", label: "Full wheel" }, { value: "half", label: "Half wheel" }] }), + c.segmented.md({ label: "A half wheel opens", value: prefs.side, onChange: (v) => set({ side: v }), options: [{ value: "left", label: "Left" }, { value: "right", label: "Right" }] }), + c.checklist.default({ label: "On the wheel", items: offered, values: prefs.enabled || offered.map((o) => o.value), onChange: (v) => set({ enabled: v }) })] }); + }); document.addEventListener("mousedown", onDown, true); return () => { document.removeEventListener("mousedown", onDown, true); if (wheel) wheel.close && wheel.close(); offs.forEach((f) => f()); }; }, diff --git a/app/ui/modules.js b/app/ui/modules.js index 083fb1ea..0287ffd7 100644 --- a/app/ui/modules.js +++ b/app/ui/modules.js @@ -24,8 +24,11 @@ export default [ "./features/wheel/index.js", "./features/status/index.js", "./features/audio/index.js", + "./features/imports/index.js", + "./features/clipfx/index.js", "./features/overlays/index.js", "./features/overlays/monitor.js", + "./features/preview/lanes.js", "./features/montage/index.js", "./features/generate/warn.js", "./features/composer/index.js", diff --git a/app/ui/shell/api.js b/app/ui/shell/api.js index ecb9f392..245cb0f9 100644 --- a/app/ui/shell/api.js +++ b/app/ui/shell/api.js @@ -84,6 +84,7 @@ export const api = { const videos = Object.values(entry.outputs || {}).flatMap((o) => o.videos || []); return videos.length ? { videos } : { error: "no video came out, check the ComfyUI terminal" }; }, + renderLibrary: () => call("GET", "/api/m/render/library"), suggestionStats: () => call("GET", "/api/shortcuts/suggestion_stats"), expandPrompt: (body) => call("POST", "/api/prompt/expand", { ...body, seed: 1 }), packs: () => call("GET", "/api/packs"), diff --git a/app/ui/shell/audio.js b/app/ui/shell/audio.js index 1d154f5e..f9a31bc1 100644 --- a/app/ui/shell/audio.js +++ b/app/ui/shell/audio.js @@ -63,8 +63,13 @@ export function syncSeparated(p) { /** Slide a separated lane against its clip (it still follows the clip when the clip moves). */ export function moveLane(p, id, deltaSec) { - const t = (p.audio_tracks || []).find((x) => x.id === id && x.kind === "separated"); + const t = (p.audio_tracks || []).find((x) => x.id === id); if (!t || !deltaSec) return false; + if (t.kind !== "separated") { // a file laid over the cut: it simply starts later or earlier + const was = t.start_sec || 0; + t.start_sec = Math.max(0, was + deltaSec); + return t.start_sec !== was; + } const seg = segments(p).find((s) => s.kind === "scene" && s.id === t.scene_id); const was = t.offset_sec || 0; t.offset_sec = Math.max(seg ? -seg.start : -Infinity, was + deltaSec); // never before 0: a slide past the edge must not be owed on the way back @@ -74,8 +79,18 @@ export function moveLane(p, id, deltaSec) { /** Trim a separated lane's sound. "in" cuts the head off (positive) or gives it back (negative, never before the sound begins); * "out" sets the tail (never past the sound's own length). */ export function trimLane(p, id, edge, deltaSec) { - const t = (p.audio_tracks || []).find((x) => x.id === id && x.kind === "separated"); - if (!t || !deltaSec || t.pinned_dur == null) return false; + const t = (p.audio_tracks || []).find((x) => x.id === id); + if (!t || !deltaSec) return false; + if (t.kind !== "separated") { // a file laid over the cut: its own in-point and length + const dur = t.source_dur, inAt = t.source_in_sec || 0; + if (dur == null) return false; + if (edge === "out") { t.source_dur = Math.max(0.1, dur + deltaSec); return true; } + const d = Math.min(Math.max(deltaSec, -inAt, -(t.start_sec || 0)), dur - 0.1); + if (!d) return false; + t.source_in_sec = inAt + d; t.source_dur = dur - d; t.start_sec = (t.start_sec || 0) + d; + return true; + } + if (t.pinned_dur == null) return false; const full = t.full_dur != null ? t.full_dur : t.pinned_dur; if (edge === "out") { t.user_dur = Math.max(0.1, Math.min(full, t.pinned_dur + deltaSec)); return true; } const d = Math.min(Math.max(deltaSec, -(t.pinned_in_sec || 0), -(t.start_sec || 0)), t.pinned_dur - 0.1); // nor earlier than the timeline's start diff --git a/app/ui/shell/clipfx.js b/app/ui/shell/clipfx.js new file mode 100644 index 00000000..fde3beae --- /dev/null +++ b/app/ui/shell/clipfx.js @@ -0,0 +1,46 @@ +// Per-clip effects (stored on scene.effects) and the transition on a clip's outgoing edge. The render reads these as they are. +const ZOOM = { zoom_ratio: 0.15, zoom_frames: 25, zoom_start_frame: 0 }; + +/** Apply a library effect to a scene. Flips, fill and reverse are switches: applying again turns them off. -> false if the id is unknown. */ +export function applyEffect(sc, id, value) { + const fx = { ...(sc.effects || {}) }; + const zoom = (way) => Object.assign(fx, { zoom: way, zoom_ratio: fx.zoom_ratio ?? ZOOM.zoom_ratio, zoom_frames: fx.zoom_frames ?? ZOOM.zoom_frames, zoom_start_frame: fx.zoom_start_frame ?? ZOOM.zoom_start_frame }); + const ops = { + zoom_in: () => zoom("in"), zoom_out: () => zoom("out"), + blur: () => { fx.blur = value; }, fade_in: () => { fx.fade_in = value; }, fade_out: () => { fx.fade_out = value; }, + flip_h: () => { fx.flip_h = !fx.flip_h; }, flip_v: () => { fx.flip_v = !fx.flip_v; }, + fill_frame: () => { fx.fit = fx.fit === "fill" ? "contain" : "fill"; }, + crop: () => { fx.crop_inset = Math.max(0, Math.min(0.4, (+value || 0) / 100)); }, + reverse: () => { fx.reverse = !fx.reverse; }, + reset: () => { for (const k of Object.keys(fx)) delete fx[k]; }, + }; + if (!ops[id]) return false; + ops[id](); + sc.effects = fx; + return true; +} + +/** Put a library transition on the clip's outgoing edge. */ +export function applyTransition(sc, item, value) { + if (!item) return false; + sc.video_transition = item.type || item.id; + sc.transition_frames = value != null ? Math.round(value) : Math.round((item.param && item.param.default) || 16); + return true; +} + +export const clearTransition = (sc) => { delete sc.video_transition; delete sc.transition_frames; return true; }; + +const LABELS = [ + ["reverse", () => "◀ reverse"], ["flip_h", () => "⇄ flip"], ["flip_v", () => "⇅ flip"], + ["fit", (v) => (v === "fill" ? "fill" : null)], ["crop_inset", (v) => (v > 0 ? `crop ${Math.round(v * 100)}%` : null)], + ["zoom", (v) => (v === "in" ? "zoom in" : v === "out" ? "zoom out" : null)], ["blur", (v) => (v > 0 ? `blur ${(+v).toFixed(2)}` : null)], + ["fade_in", (v) => (v > 0 ? `fade in ${v}s` : null)], ["fade_out", (v) => (v > 0 ? `fade out ${v}s` : null)], +]; + +/** What is on this clip, as short tags (so a switch can be seen to be on). */ +export function tags(sc) { + const fx = sc.effects || {}, out = []; + for (const [key, fmt] of LABELS) { const raw = fx[key]; if (raw !== undefined && raw !== null && raw !== false && raw !== "") { const t = fmt(raw); if (t) out.push(t); } } + if (sc.video_transition) out.push(`→ ${sc.video_transition}`); + return out; +} diff --git a/app/ui/shell/imports.js b/app/ui/shell/imports.js new file mode 100644 index 00000000..f171af40 --- /dev/null +++ b/app/ui/shell/imports.js @@ -0,0 +1,23 @@ +// Bringing files from the media bin onto the timeline: a video as a clip of its own, a sound as an audio lane. +import { snapFrames } from "./scenes.js"; + +const newId = () => "c" + Math.random().toString(36).slice(2, 10) + Date.now().toString(36); + +/** A video from the bin as a clip at the end of the cut. `seconds` is its length when known (else the project's own length until it is). */ +export function addVideoClip(p, asset, seconds) { + if (!asset || asset.kind !== "video") return false; + const id = newId(), fps = p.frame_rate || 25, dur = seconds > 0 ? seconds : null; + const sc = { id, gen_unit_id: id, text: "", transition_to_next: "", source: { type: "video", media_ref: asset.id }, source_dur: dur, + frames_mode: dur != null ? "timeline" : "project", frames: dur != null ? snapFrames(dur * fps) : null, excluded: false }; + p.scenes.push(sc); + return sc; +} + +/** A sound from the bin as an audio lane starting at `startSec`. */ +export function addAudioTrack(p, asset, startSec, seconds) { + if (!asset || asset.kind !== "audio") return false; + const lane = { id: "t" + Math.random().toString(36).slice(2, 10) + Date.now().toString(36), kind: "overlay", media_ref: asset.id, start_sec: Math.max(0, startSec || 0), + source_in_sec: 0, source_dur: seconds > 0 ? seconds : null, volume: 1, label: asset.name || "Audio" }; + p.audio_tracks = [...(p.audio_tracks || []), lane]; + return lane; +} diff --git a/app/ui/shell/playhead.js b/app/ui/shell/playhead.js index 0849c725..a378bbe6 100644 --- a/app/ui/shell/playhead.js +++ b/app/ui/shell/playhead.js @@ -3,6 +3,7 @@ export function createPlayhead() { let at = 0; const heard = new Set(); return { + playing: false, // set by the monitor; read by whatever sounds along with it get at() { return at; }, set(sec) { const next = Math.max(0, sec); if (next === at) return; at = next; heard.forEach((fn) => fn(at)); }, on(fn) { heard.add(fn); return () => heard.delete(fn); }, diff --git a/app/ui/shell/tests/audio.test.js b/app/ui/shell/tests/audio.test.js index 76fa9a70..30865432 100644 --- a/app/ui/shell/tests/audio.test.js +++ b/app/ui/shell/tests/audio.test.js @@ -126,3 +126,15 @@ test("sliding past the timeline start is not owed on the way back; trimming in c assert.equal(trimLane(q, first.id, "in", -0.5), false); assert.equal(first.pinned_in_sec, before); }); + +test("a file lane slides and trims on its own numbers", () => { + const p = { audio_tracks: [{ id: "f", kind: "overlay", media_ref: "m", start_sec: 1, source_in_sec: 0, source_dur: 4 }] }; + moveLane(p, "f", -9); + assert.equal(p.audio_tracks[0].start_sec, 0); + assert.equal(trimLane(p, "f", "in", 1), true); + assert.deepEqual([p.audio_tracks[0].source_in_sec, p.audio_tracks[0].source_dur, p.audio_tracks[0].start_sec], [1, 3, 1]); + assert.equal(trimLane(p, "f", "in", -5), true); + assert.deepEqual([p.audio_tracks[0].source_in_sec, p.audio_tracks[0].start_sec], [0, 0]); + trimLane(p, "f", "out", -9); + assert.equal(p.audio_tracks[0].source_dur, 0.1); +}); diff --git a/app/ui/shell/tests/clipfx.test.js b/app/ui/shell/tests/clipfx.test.js new file mode 100644 index 00000000..518a6017 --- /dev/null +++ b/app/ui/shell/tests/clipfx.test.js @@ -0,0 +1,22 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { applyEffect, applyTransition, tags } from "../clipfx.js"; + +test("switches toggle, values set, reset clears; unknown ids change nothing", () => { + const sc = {}; + applyEffect(sc, "flip_h"); applyEffect(sc, "fill_frame"); applyEffect(sc, "blur", 0.3); applyEffect(sc, "crop", 50); + assert.deepEqual(tags(sc), ["⇄ flip", "fill", "crop 40%", "blur 0.30"]); + applyEffect(sc, "flip_h"); applyEffect(sc, "fill_frame"); + assert.deepEqual(tags(sc), ["crop 40%", "blur 0.30"]); + assert.equal(applyEffect(sc, "nope"), false); + applyEffect(sc, "reset"); + assert.deepEqual(sc.effects, {}); +}); + +test("zoom keeps the ramp settings it already has; a transition names the seam and its frames", () => { + const sc = { effects: { zoom_ratio: 0.3 } }; + applyEffect(sc, "zoom_in"); + assert.deepEqual([sc.effects.zoom, sc.effects.zoom_ratio, sc.effects.zoom_frames], ["in", 0.3, 25]); + applyTransition(sc, { id: "x", type: "wipeleft", param: { default: 8 } }); + assert.deepEqual([sc.video_transition, sc.transition_frames, tags(sc).pop()], ["wipeleft", 8, "→ wipeleft"]); +}); diff --git a/app/ui/shell/tests/imports.test.js b/app/ui/shell/tests/imports.test.js new file mode 100644 index 00000000..faf8214e --- /dev/null +++ b/app/ui/shell/tests/imports.test.js @@ -0,0 +1,20 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { addVideoClip, addAudioTrack } from "../imports.js"; + +test("a bin video becomes a clip as long as the file; one of unknown length follows the project until it is probed", () => { + const p = { scenes: [], frame_rate: 25 }; + const known = addVideoClip(p, { id: "m1", kind: "video" }, 4); + assert.deepEqual([known.source.media_ref, known.frames_mode, known.frames], ["m1", "timeline", 97]); + const unknown = addVideoClip(p, { id: "m2", kind: "video" }, 0); + assert.deepEqual([unknown.frames_mode, unknown.source_dur], ["project", null]); + assert.equal(addVideoClip(p, { id: "i", kind: "image" }), false); + assert.equal(p.scenes.length, 2); +}); + +test("a bin sound becomes a lane from the playhead; only audio is accepted", () => { + const p = {}; + const t = addAudioTrack(p, { id: "a", kind: "audio", name: "tone" }, -3, 2); + assert.deepEqual([t.start_sec, t.source_dur, t.label, p.audio_tracks.length], [0, 2, "tone", 1]); + assert.equal(addAudioTrack(p, { id: "v", kind: "video" }, 0), false); +}); diff --git a/app/ui/shell/tests/syntax.test.js b/app/ui/shell/tests/syntax.test.js new file mode 100644 index 00000000..4c5bb3fb --- /dev/null +++ b/app/ui/shell/tests/syntax.test.js @@ -0,0 +1,16 @@ +// Every module the page loads must at least parse: a feature with a syntax error is hidden, and a baseline one blanks the screen. +import test from "node:test"; +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { readdirSync, statSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const walk = (d) => readdirSync(d).flatMap((n) => { const p = join(d, n); return statSync(p).isDirectory() ? walk(p) : p.endsWith(".js") ? [p] : []; }); + +test("every .js under app/ui parses", () => { + const bad = []; + for (const f of walk(root)) { try { execFileSync(process.execPath, ["--check", f], { stdio: "pipe" }); } catch (e) { bad.push(`${f}: ${String(e.stderr).split("\n").slice(0, 3).join(" ")}`); } } + assert.deepEqual(bad, []); +}); diff --git a/modules/output/render/stitch.py b/modules/output/render/stitch.py index d6c1cde7..09e3372b 100644 --- a/modules/output/render/stitch.py +++ b/modules/output/render/stitch.py @@ -147,7 +147,17 @@ def _live_tracks(project, clips_by_scene: dict) -> list[dict]: if t.get("kind") != "separated" or not t.get("scene_id") or t["scene_id"] in clips_by_scene] -def _audio_tracks(project, clips_by_scene: dict, resolve) -> list[dict]: +def _clip_starts(clips: list[dict]) -> dict: + """Where each clip begins in THIS render (clips that are out of the cut take no room).""" + at, out = 0.0, {} + for c in clips: + if c.get("scene_id"): + out[c["scene_id"]] = at + at += _f(c.get("dur")) + return out + + +def _audio_tracks(project, clips_by_scene: dict, resolve, starts: dict | None = None) -> list[dict]: """The project's extra audio lanes as {path, start_sec, volume[, source_in, source_dur]}. A "separated" lane is a clip's own sound pulled onto a lane: it plays the audio pinned at @@ -158,6 +168,8 @@ def _audio_tracks(project, clips_by_scene: dict, resolve) -> list[dict]: separated = t.get("kind") == "separated" or (t.get("scene_id") and not t.get("media_ref") and t.get("kind") != "overlay") start, vol = _f(t.get("start_sec")), _f(t.get("volume"), 1.0) + if separated and starts and t.get("scene_id") in starts: # a separated lane rides its clip: where the clip is in this render, plus how far it was slid + start = max(0.0, starts[t["scene_id"]] + _f(t.get("offset_sec"))) if separated: clip = clips_by_scene.get(t.get("scene_id")) or {} path = None @@ -279,7 +291,7 @@ def render(project, clips: list[dict]) -> dict: raise RenderError(str(exc)) from exc by_scene = {c["scene_id"]: c for c in clips if c.get("scene_id")} - tracks = _audio_tracks(project, by_scene, files.clip_path) if clips or project.audio_tracks else [] + tracks = _audio_tracks(project, by_scene, files.clip_path, _clip_starts(clips)) if clips or project.audio_tracks else [] keep = bool(project.keep_original_audio) and not blank cmd = [ffmpeg, "-y"] diff --git a/modules/output/render/tests/test_stitch.py b/modules/output/render/tests/test_stitch.py index c07777bb..c7be15ed 100644 --- a/modules/output/render/tests/test_stitch.py +++ b/modules/output/render/tests/test_stitch.py @@ -337,3 +337,16 @@ def test_a_separated_lane_goes_quiet_with_a_clip_that_is_not_in_the_cut(): {"id": "a", "kind": "separated", "scene_id": "gone"}, {"id": "b", "kind": "separated", "scene_id": "here"}, {"id": "c", "media_ref": "aaaaaaaaaaaa", "start_sec": 0}]) assert [t["id"] for t in stitch._live_tracks(proj, {"here": {}})] == ["b", "c"] + + +def test_a_separated_lane_starts_where_its_clip_is_in_this_render_plus_its_slide(tmp_path, monkeypatch): + f = tmp_path / "x.mp4" + f.write_bytes(b"") + monkeypatch.setattr(stitch.files, "has_audio", lambda p: True) + proj = projects.Project(width=160, height=120, audio_tracks=[ + {"id": "a", "kind": "separated", "scene_id": "s2", "start_sec": 99, "offset_sec": 0.5}, # start_sec is the editor's guess, the render's own timeline wins + {"id": "b", "kind": "separated", "scene_id": "s2", "offset_sec": -9}]) + clips = [{"scene_id": "s1", "dur": 2.0}, {"scene_id": "s2", "dur": 3.0}] + by = {c["scene_id"]: c for c in clips} + out = stitch._audio_tracks(proj, by, lambda c: str(f), stitch._clip_starts(clips)) + assert [round(t["start_sec"], 2) for t in out] == [2.5, 0.0]