From a1b2eec52346c5a6cefb72d76f05ae5f175979b2 Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Thu, 11 Jun 2026 16:10:23 +0300 Subject: [PATCH] Add scene vs video clips with convert and archive round-trip. Generative scenes can lock to imported video (skipped on Generate); video clips can convert back to v2v with full state restored from scene_archive. --- movie_editor/backend/timeline.py | 25 +++- movie_editor/frontend/inspector.js | 78 ++++++++++- movie_editor/frontend/mediabrowser.js | 6 +- movie_editor/frontend/player.js | 42 ++++-- movie_editor/frontend/store.js | 190 ++++++++++++++++++++++++-- movie_editor/frontend/styles.css | 6 + movie_editor/frontend/timeline.js | 106 ++++++++++++-- movie_editor/frontend/view_bus.js | 2 +- movie_editor/server.py | 9 +- movie_editor/tests/test_timeline.py | 18 +++ 10 files changed, 433 insertions(+), 49 deletions(-) diff --git a/movie_editor/backend/timeline.py b/movie_editor/backend/timeline.py index 03a3a5d6..ffd1fb77 100644 --- a/movie_editor/backend/timeline.py +++ b/movie_editor/backend/timeline.py @@ -382,6 +382,15 @@ def source_type(scene: Scene) -> str: return (scene.source.type if scene.source else None) or "carry" +def is_video_clip(scene: Scene) -> bool: + """Locked timeline video — plays as-is, excluded from generation and global prompt.""" + return source_type(scene) == "video" + + +def is_generative_scene(scene: Scene) -> bool: + return not is_video_clip(scene) + + def solo_run_wants_prior_guides(scene: Scene) -> bool: """Solo i2v runs: only mixed borrows prior-scene guides; image/empty/generated_frame are anchor-only.""" return is_mixed_source(scene) @@ -420,8 +429,11 @@ def build_scene_anchors_payload(project: Project) -> Optional[dict]: @dataclass class SceneSource: """How a scene's latent is born. V1 ignores this (uniform chain); Phase 2 maps - it onto EmptyLTXVLatent (empty/t2v) or LTXV Image to Video (image/i2v).""" - type: str = "carry" # "carry" | "empty" | "image" | "generated_frame" | "mixed" + it onto EmptyLTXVLatent (empty/t2v) or LTXV Image to Video (image/i2v). + + ``video`` = locked NLE clip (imported or converted from a scene); never generated. + ``v2v`` = generative scene using a video file as the source (video-to-video).""" + type: str = "carry" # carry | empty | image | generated_frame | mixed | video | v2v media_ref: Optional[str] = None # asset id — image/mixed anchor (Img2Video) frame_ref: Optional[dict[str, Any]] = None # {scene_id, frame_idx}, for "generated_frame" target: Optional[str] = None # wire dest for the image: "port:" | "node::" @@ -493,6 +505,9 @@ class Scene: # Source trim inside generated media (slip edit): in-point seconds and optional duration. source_in: float = 0.0 source_dur: Optional[float] = None + # Saved generative state when this clip is converted to a locked ``video`` clip. + # Restored on Convert to scene (scene → video → scene round-trip). + scene_archive: Optional[dict] = None @staticmethod def from_dict(d: dict) -> "Scene": @@ -520,6 +535,7 @@ class Scene: character_ids=[str(x) for x in (d.get("character_ids") or []) if x], source_in=float(d.get("source_in") or 0), source_dur=d.get("source_dur"), + scene_archive=d.get("scene_archive"), ) def to_dict(self) -> dict: @@ -707,7 +723,10 @@ def build_combined_prompt(project: Project, include_excluded: bool = False, transition_to_next, else the generic 'scene N' label) to force the split. This affects only what's sent to Studio, never the displayed global prompt. """ - scenes = [s for s in project.scenes if include_excluded or not s.excluded] + scenes = [ + s for s in project.scenes + if (include_excluded or not s.excluded) and is_generative_scene(s) + ] units = group_generative_units(scenes) char_map = _load_character_map() parts: list[str] = [] diff --git a/movie_editor/frontend/inspector.js b/movie_editor/frontend/inspector.js index 8e72c59c..8bd5aca1 100644 --- a/movie_editor/frontend/inspector.js +++ b/movie_editor/frontend/inspector.js @@ -9,6 +9,7 @@ ["empty", "Empty · text-to-video"], ["image", "Image · i2v anchor"], ["generated_frame", "From generated frame"], + ["v2v", "Video · v2v source"], ["carry", "Carry i2v guide · continue previous"], ["mixed", "Mixed · Img2Video anchor + prior guides"], ]; @@ -143,6 +144,71 @@ return inner; } + function renderVideoSource(st, scene, parent, label) { + parent = parent || body; + if (!window.MediaPicker) return; + const pick = window.MediaPicker.create({ + mediaBin: st.mediaBin, + value: scene.source?.media_ref, + filter: (m) => m.kind === "video", + noneLabel: "— pick video —", + onChange: (mediaRef) => { + S.patchScene(scene.id, { + source: { ...(scene.source || {}), type: scene.source?.type || "v2v", media_ref: mediaRef }, + }); + }, + }); + parent.append(field(label || "V2V source video", pick)); + } + + function renderVideoClip(st, scene) { + title.textContent = "Video clip"; + body.append(el("div", "insp-hint", "Imported or converted video — plays as-is. Not included in Generate or the global prompt montage.")); + const ref = scene.source?.media_ref; + const asset = ref ? (st.mediaBin || []).find((m) => m.id === ref) : null; + body.append(field("Source", el("span", null, asset?.name || ref || "From last render"))); + renderOutgoingSeam(st, scene); + + const actions = el("div", "insp-block"); + const toScene = el("button", "btn primary", "Convert to scene"); + toScene.title = scene.scene_archive + ? "Restore prompt, source, guides, and settings from before this was locked as video" + : "Make this a generative v2v scene (prompt + Generate)"; + toScene.onclick = () => S.convertToScene(scene.id); + actions.append(toScene); + const del = el("button", "btn danger", "Delete"); + del.style.marginLeft = "8px"; + del.onclick = () => S.removeScene(scene.id); + actions.append(del); + body.append(actions); + + foldSection("More editing", false, (more) => { + const fx = scene.effects || {}; + const patchFx = (k, v, quiet) => { + const next = { ...(scene.effects || {}), [k]: v }; + quiet ? S.patchSceneQuiet(scene.id, { effects: next }) : S.patchScene(scene.id, { effects: next }); + }; + const blur = el("input"); blur.type = "range"; blur.min = "0"; blur.max = "1"; blur.step = "0.05"; + blur.value = fx.blur || 0; + blur.oninput = () => patchFx("blur", parseFloat(blur.value), true); + more.append(field(`Blur (${Math.round((fx.blur || 0) * 100)}%)`, blur)); + const lenRow = el("div", "fields-row"); + lenRow.append(lengthControl(scene, "frames")); + lenRow.append(lengthControl(scene, "fps")); + more.append(lenRow); + const r = (st.sceneRenders || {})[scene.id]; + if (r?.media || ref) { + const srcTag = el("div", "insp-tag"); srcTag.textContent = "Source trim (slip)"; more.append(srcTag); + const inRow = el("div", "fields-row"); + const iIn = el("input"); iIn.type = "number"; iIn.min = "0"; iIn.step = "0.05"; + iIn.value = scene.source_in || 0; + iIn.oninput = () => S.setSourceTrim(scene.id, { source_in: parseFloat(iIn.value || 0) }); + inRow.append(field("Source in (s)", iIn)); + more.append(inRow); + } + }); + } + function renderScene(st, scene) { const root = S.genUnitRoot(S.genUnitId(scene)) || scene; const unitScenes = (st.project.scenes || []).filter((s) => S.genUnitId(s) === S.genUnitId(scene)) @@ -177,6 +243,7 @@ body.append(field("Source", src)); if ((root.source?.type) === "image" || (root.source?.type) === "mixed") renderImageSource(st, root, body); if ((root.source?.type) === "generated_frame") renderGeneratedFrameSource(st, root); + if ((root.source?.type) === "v2v") renderVideoSource(st, root, body, "V2V source video"); const effFrames = effOf(scene, "frames"), effFps = effOf(scene, "fps") || 1; body.append(el("div", "insp-hint", `Duration ≈ ${(effFrames / effFps).toFixed(2)}s · trim on timeline · generation splits: global prompt or Outgoing seam below`)); @@ -187,6 +254,11 @@ const genBtn = el("button", "btn primary", "Generate this scene"); genBtn.onclick = () => S.generate(scene.id); actions.append(genBtn); + const toVid = el("button", "btn ghost", "Convert to video"); + toVid.title = "Lock as a plain video clip — skipped by Generate; settings saved for Convert back to scene"; + toVid.style.marginLeft = "8px"; + toVid.onclick = () => S.convertToVideo(scene.id); + actions.append(toVid); const del = el("button", "btn danger", "Delete"); del.style.marginLeft = "8px"; del.onclick = () => S.removeScene(scene.id); actions.append(del); body.append(actions); @@ -638,8 +710,10 @@ renderGlobalPrompt(st); renderSwitch(st, scene); renderEngineStrip(st); - if (scene) renderScene(st, scene); - else renderProject(st); + if (scene) { + if (S.isVideoClip(scene)) renderVideoClip(st, scene); + else renderScene(st, scene); + } else renderProject(st); renderExposed(st); renderSplit(st); body.scrollTop = scrollTop; // restore so editing doesn't jump to the top diff --git a/movie_editor/frontend/mediabrowser.js b/movie_editor/frontend/mediabrowser.js index a7d9bf94..ec511a4c 100644 --- a/movie_editor/frontend/mediabrowser.js +++ b/movie_editor/frontend/mediabrowser.js @@ -222,7 +222,9 @@ ? `${m.name}\nClick to preview · drag onto a clip to set anchor` : m.kind === "audio" ? `${m.name}\nClick to preview · add via timeline + Add → Audio` - : `${m.name}\nDrag onto a clip to set anchor`; + : m.kind === "video" + ? `${m.name}\nDrag onto timeline · + Add → Video` + : `${m.name}\nDrag onto a clip to set anchor`; const thumb = el("div", "media-thumb"); if (m.kind === "image") { const img = el("img"); img.src = API.mediaUrl(m.id); img.loading = "lazy"; thumb.append(img); } else if (m.kind === "audio") thumb.append(el("span", "media-icon", "♪")); @@ -234,6 +236,8 @@ card.append(del); if (m.kind === "image" || m.kind === "audio") { card.onclick = () => S.previewMedia(m.id); + } else if (m.kind === "video") { + card.ondblclick = () => S.addVideoClip(m.id); } grid.append(card); }); diff --git a/movie_editor/frontend/player.js b/movie_editor/frontend/player.js index 2a2cb845..b58832d4 100644 --- a/movie_editor/frontend/player.js +++ b/movie_editor/frontend/player.js @@ -39,6 +39,7 @@ let _currentClip = null; const _urlFor = (media) => API.resultUrl(S.get().project?.id, media); + const _clipUrl = (clip) => clip.binUrl || (clip.media ? _urlFor(clip.media) : ""); const _clampT = (v, t) => Math.max(0, Math.min(t, (v.duration || (t + 1)) - 0.02)); function _segmentSourceBadge(sceneId) { @@ -77,13 +78,21 @@ } const sc = seg.scene; const r = sr[sc.id]; + const fx = sc.effects || {}; + const vol = sc.audio_separated ? 0 : (sc.audio_volume != null ? sc.audio_volume : 1); + const inSec = (sc.source_in || 0) + (r?.inSec || 0); if (r && r.media) { out.push({ - media: r.media, sceneId: sc.id, startSec, durationSec: dur, inSec: r.inSec || 0, - fx: sc.effects || {}, - vol: sc.audio_separated ? 0 : (sc.audio_volume != null ? sc.audio_volume : 1), + media: r.media, sceneId: sc.id, startSec, durationSec: dur, inSec, + fx, vol, }); - } else if (!sc.excluded) { + } else if (S.isVideoClip && S.isVideoClip(sc) && sc.source?.media_ref) { + out.push({ + binUrl: API.mediaUrl(sc.source.media_ref), + sceneId: sc.id, startSec, durationSec: dur, inSec: sc.source_in || 0, + fx, vol, + }); + } else if (!sc.excluded && S.isGenerativeScene && S.isGenerativeScene(sc)) { out.push({ pending: true, sceneId: sc.id, startSec, durationSec: dur, slate: "Not generated yet", @@ -204,7 +213,7 @@ } function _syncPool() { - const urls = new Set(_clips.filter((c) => c.media).map((c) => _urlFor(c.media))); + const urls = new Set(_clips.filter((c) => c.media || c.binUrl).map((c) => _clipUrl(c))); urls.forEach((u) => _ensureVideo(u)); for (const [u, v] of [...pool]) { if (urls.has(u)) continue; @@ -275,8 +284,8 @@ if (play) _startTick(); return; } - if (!clip.media) return; - const v = _ensureVideo(_urlFor(clip.media)); + if (!clip.media && !clip.binUrl) return; + const v = _ensureVideo(_clipUrl(clip)); _currentClip = clip; _setActive(v); _showSlate(""); const badge = clip.ghost ? (clip.slate || "Removed — preview only") : _segmentSourceBadge(clip.sceneId); @@ -304,7 +313,8 @@ const prev = _currentClip; const sameSource = next.media && prev.media && _urlFor(next.media) === _urlFor(prev.media); - const contiguous = sameSource + const sameBin = next.binUrl && prev.binUrl && next.binUrl === prev.binUrl; + const contiguous = (sameSource || sameBin) && Math.abs((next.inSec || 0) - ((prev.inSec || 0) + prev.durationSec)) < 0.05; _phSec = next.startSec; _currentClip = next; @@ -548,7 +558,7 @@ _syncPool(); // preload current clips, drop stale ones (timeline is dynamic) // Re-resolve the clip under the playhead each render (positions change with edits). - const hash = _clips.map((c) => (c.pending ? "pending:" + c.sceneId : c.ghost ? "ghost:" + c.ghostId : c.media?.filename) + "@" + c.startSec.toFixed(2) + "+" + (c.inSec || 0).toFixed(2)).join("|"); + const hash = _clips.map((c) => (c.pending ? "pending:" + c.sceneId : c.ghost ? "ghost:" + c.ghostId : (_clipUrl(c) || "x")) + "@" + c.startSec.toFixed(2) + "+" + (c.inSec || 0).toFixed(2)).join("|"); if (hash !== _lastSegHash) { _lastSegHash = hash; const clip = _clipAt(_phSec); @@ -660,15 +670,19 @@ } else { const sc = seg.scene; const stype = sc.source?.type || "empty"; - const rendered = !!((st.sceneRenders || {})[sc.id] || {}).media; - const stale = rendered && S.renderIsStale?.(sc.id); + const isVideo = S.isVideoClip && S.isVideoClip(sc); + const rendered = isVideo + ? !!(sc.source?.media_ref || ((st.sceneRenders || {})[sc.id] || {}).media) + : !!((st.sceneRenders || {})[sc.id] || {}).media; + const stale = !isVideo && rendered && S.renderIsStale?.(sc.id); cls += rendered ? " rendered" : " pending"; - if (stale) cls += " stale"; - if (stype === "mixed") { cls += " mixed"; title = "Mixed · anchor + prior guides"; } + if (isVideo) { cls += " video"; title = "Video clip (locked)"; } + else if (stale) cls += " stale"; + else if (stype === "mixed") { cls += " mixed"; title = "Mixed · anchor + prior guides"; } else if (stype === "carry") { cls += " carry"; title = "Carry · prior guides only"; } const idx = (p.scenes || []).indexOf(sc) + 1; if (!title) title = `Scene ${idx}${rendered ? (stale ? " (previous generation)" : " (rendered)") : " (not rendered)"}`; - else title = `Scene ${idx} · ${title}${rendered ? (stale ? " · previous generation" : "") : " · not rendered"}`; + else if (!isVideo) title = `Scene ${idx} · ${title}${rendered ? (stale ? " · previous generation" : "") : " · not rendered"}`; } const chip = el("div", cls); chip.style.width = (d / _totalSecCur * 100).toFixed(3) + "%"; diff --git a/movie_editor/frontend/store.js b/movie_editor/frontend/store.js index 637192fe..e1040da3 100644 --- a/movie_editor/frontend/store.js +++ b/movie_editor/frontend/store.js @@ -146,7 +146,7 @@ const parts = []; const anchor = (project.anchor || "").trim(); if (anchor) parts.push(anchor); - const active = (project.scenes || []).filter((s) => !s.excluded); + const active = (project.scenes || []).filter((s) => !s.excluded && isGenerativeScene(s)); const units = _groupGenerativeUnits(active); units.forEach(([, group], i) => { const root = _rootFromGroup(group); @@ -299,10 +299,28 @@ return out; } + function _mergeGenerativeWithVideoClips(oldScenes, generativeNext) { + const oldOrder = (oldScenes || []).filter((s) => !s.excluded); + const videoIds = new Set(oldOrder.filter(isVideoClip).map((s) => s.id)); + if (!videoIds.size) return generativeNext; + const byId = new Map((oldScenes || []).map((s) => [s.id, s])); + const out = []; + let g = 0; + for (const sc of oldOrder) { + if (videoIds.has(sc.id)) { + out.push(JSON.parse(JSON.stringify(byId.get(sc.id) || sc))); + } else if (isGenerativeScene(sc)) { + if (g < generativeNext.length) out.push(generativeNext[g++]); + } + } + while (g < generativeNext.length) out.push(generativeNext[g++]); + return out; + } + // Global prompt is authoritative: parsed scenes become the timeline; unmatched clips // are ghosted (if rendered) or dropped. Matching is by normalized text only. function _alignScenesFromParsed(oldScenes, parsedScenes, ghosts) { - const old = (oldScenes || []).filter((s) => !s.excluded); + const old = (oldScenes || []).filter((s) => !s.excluded && isGenerativeScene(s)); const usedOld = new Set(); const usedGhost = new Set(); const next = []; @@ -340,7 +358,8 @@ } ghostsOut = ghostsOut.filter((g) => !usedGhost.has(g.id)); - return { next, ghosts: ghostsOut, usedGhost }; + const merged = _mergeGenerativeWithVideoClips(oldScenes, next); + return { next: merged, ghosts: ghostsOut, usedGhost }; } function _normalizeGlobalPromptParse(trimmed, v) { @@ -676,6 +695,8 @@ function scene(id) { return state.project?.scenes.find((s) => s.id === id) || null; } function genUnitId(sc) { return (sc && sc.gen_unit_id) || (sc && sc.id) || ""; } + function isVideoClip(sc) { return (sc && sc.source && sc.source.type) === "video"; } + function isGenerativeScene(sc) { return !!sc && !isVideoClip(sc); } function isGenSubclip(sc) { return !!((sc && sc.cut_offset_frames) > 0); } function genUnitRoot(unitId) { const group = (state.project?.scenes || []).filter((s) => genUnitId(s) === unitId); @@ -872,6 +893,127 @@ notify(); scheduleSave(); // server assigns id; reselect after commit } + function _snapshotSceneForArchive(sc) { + const r = state.sceneRenders[sc.id]; + return { + text: sc.text || "", + source: JSON.parse(JSON.stringify(sc.source || {})), + guides: JSON.parse(JSON.stringify(sc.guides || [])), + character_ids: [...(sc.character_ids || [])], + effects: JSON.parse(JSON.stringify(sc.effects || {})), + frames: sc.frames, + frames_mode: sc.frames_mode, + fps: sc.fps, + fps_mode: sc.fps_mode, + transition_to_next: sc.transition_to_next || "", + video_transition: sc.video_transition || "", + transition_frames: sc.transition_frames, + rating: sc.rating || "", + audio_volume: sc.audio_volume, + audio_separated: sc.audio_separated, + gen_unit_id: sc.gen_unit_id, + cut_offset_frames: sc.cut_offset_frames, + source_in: sc.source_in, + source_dur: sc.source_dur, + width: sc.width, + height: sc.height, + render: r ? JSON.parse(JSON.stringify(r)) : null, + }; + } + + function _applySceneArchive(sc, archive) { + if (!archive) return; + sc.text = archive.text || ""; + sc.source = JSON.parse(JSON.stringify(archive.source || { type: "carry" })); + sc.guides = JSON.parse(JSON.stringify(archive.guides || [])); + sc.character_ids = [...(archive.character_ids || [])]; + sc.effects = JSON.parse(JSON.stringify(archive.effects || {})); + sc.frames = archive.frames; + sc.frames_mode = archive.frames_mode; + sc.fps = archive.fps; + sc.fps_mode = archive.fps_mode; + sc.transition_to_next = archive.transition_to_next || ""; + sc.video_transition = archive.video_transition || ""; + sc.transition_frames = archive.transition_frames; + sc.rating = archive.rating || ""; + sc.audio_volume = archive.audio_volume; + sc.audio_separated = !!archive.audio_separated; + sc.gen_unit_id = archive.gen_unit_id; + sc.cut_offset_frames = archive.cut_offset_frames || 0; + sc.source_in = archive.source_in || 0; + sc.source_dur = archive.source_dur; + sc.width = archive.width; + sc.height = archive.height; + sc.scene_archive = null; + if (archive.render) state.sceneRenders[sc.id] = JSON.parse(JSON.stringify(archive.render)); + } + + function convertToVideo(sceneId) { + if (!state.project) return; + const sc = scene(sceneId); + if (!sc || isVideoClip(sc)) return; + _historyRecord(); + const uid = genUnitId(sc); + const root = genUnitRoot(uid) || sc; + const archive = _snapshotSceneForArchive(root); + const mediaRef = (root.source && root.source.media_ref) || null; + state.project.scenes = state.project.scenes.filter( + (s) => genUnitId(s) !== uid || s.id === root.id + ); + Object.assign(root, { + scene_archive: archive, + source: { type: "video", media_ref: mediaRef }, + gen_unit_id: root.id, + cut_offset_frames: 0, + }); + if (state.selectedSceneId && !scene(state.selectedSceneId)) { + state.selectedSceneId = root.id; + state.selectedSceneIds = [root.id]; + _selectionAnchorId = root.id; + } + _afterTimelineStructureChange(); + notify(); scheduleSave(); + } + + function convertToScene(sceneId) { + if (!state.project) return; + const sc = scene(sceneId); + if (!sc || !isVideoClip(sc)) return; + _historyRecord(); + if (sc.scene_archive) { + _applySceneArchive(sc, sc.scene_archive); + } else { + const mediaRef = sc.source?.media_ref || null; + sc.scene_archive = null; + sc.source = { type: "v2v", media_ref: mediaRef }; + if (!sc.text) sc.text = ""; + sc.gen_unit_id = sc.gen_unit_id || sc.id; + } + _afterTimelineStructureChange(); + notify(); scheduleSave(); + } + + function addVideoClip(mediaId) { + if (!state.project || !mediaId) return; + const asset = (state.mediaBin || []).find((m) => m.id === mediaId); + if (!asset || asset.kind !== "video") return; + _historyRecord(); + const id = _newSceneId(); + state.project.scenes.push({ + id, + gen_unit_id: id, + text: "", + transition_to_next: "", + source: { type: "video", media_ref: mediaId }, + excluded: false, + }); + state.selectedSceneId = id; + state.selectedSceneIds = [id]; + _selectionAnchorId = id; + window.Timeline?.requestAutoFit?.(); + notify(); scheduleSave(); + } + function _removeSceneNoHistory(id) { if (!state.project) return; const arr = state.project.scenes; @@ -1362,19 +1504,19 @@ // back to carry / i2v guides, not a broken anchor). function _anchorAvailable(s) { const t = s.source && s.source.type; - if (t !== "image" && t !== "generated_frame" && t !== "mixed") return false; + if (t !== "image" && t !== "generated_frame" && t !== "mixed" && t !== "v2v") return false; const ref = s.source.media_ref; return !!(ref && (state.mediaBin || []).some((m) => m.id === ref)); } function _runs() { - const active = state.project.scenes.filter((s) => !s.excluded); + const active = state.project.scenes.filter((s) => !s.excluded && isGenerativeScene(s)); const runs = []; for (const s of active) { const t = (s.source && s.source.type) || "empty"; // Editorial subclips always continue the current run; carry / broken anchors too. const isCarry = isGenSubclip(s) || t === "carry" - || ((t === "image" || t === "generated_frame" || t === "mixed") && !_anchorAvailable(s)); + || ((t === "image" || t === "generated_frame" || t === "mixed" || t === "v2v") && !_anchorAvailable(s)); if (isCarry && runs.length) runs[runs.length - 1].push(s.id); else runs.push([s.id]); } @@ -1716,7 +1858,7 @@ if (!state.project) return; const ids = (state.selectedSceneIds || []).filter((sid) => { const s = scene(sid); - return s && !s.excluded; + return s && !s.excluded && isGenerativeScene(s); }); if (!ids.length) { set({ gen: { state: "error", promptId: null, media: [], msg: "No scenes selected." } }); @@ -1782,12 +1924,10 @@ for (const sc of (p.scenes || [])) { if (sc.excluded) continue; const r = state.sceneRenders[sc.id]; - if (!(r && r.media)) continue; const fps = (sc.fps_mode !== "project" && sc.fps != null ? sc.fps : p.frame_rate) || 25; const tFrames = sc.transition_frames || 0; - out.push({ - media: r.media, - inSec: (r.inSec || 0) + (sc.source_in || 0), + const base = { + inSec: (r?.inSec || 0) + (sc.source_in || 0), durationSec: sc.source_dur != null ? sc.source_dur : sceneDurationSec(sc), fx: sc.effects || {}, fps, w: (sc.width != null ? sc.width : p.width) || null, @@ -1796,7 +1936,12 @@ tdur: tFrames > 0 ? tFrames / fps : 0, sceneId: sc.id, volume: sc.audio_separated ? 0 : (sc.audio_volume != null ? sc.audio_volume : 1), - }); + }; + if (r && r.media) { + out.push({ media: r.media, ...base }); + } else if (isVideoClip(sc) && sc.source?.media_ref) { + out.push({ bin_media_ref: sc.source.media_ref, ...base }); + } } return out; } @@ -1808,9 +1953,15 @@ if (!id) { alert("Select a clip to export."); return; } const sc = scene(id); const r = state.sceneRenders[id]; - if (!r || !r.media) { alert("That clip hasn't been generated yet — generate it first."); return; } const idx = state.project.scenes.findIndex((s) => s.id === id); const proj = (state.project.name || "montage").replace(/[^\w.-]+/g, "_"); + if (isVideoClip(sc) && sc.source?.media_ref && !r?.media) { + const asset = (state.mediaBin || []).find((m) => m.id === sc.source.media_ref); + const name = asset?.name || `${proj}_video_${_stamp()}.mp4`; + await _saveBlobAs(API.mediaUrl(sc.source.media_ref), name.replace(/[^\w.-]+/g, "_")); + return; + } + if (!r || !r.media) { alert("That clip hasn't been generated yet — generate it first."); return; } const inSec = (r.inSec || 0) + (sc?.source_in || 0); const dur = sc?.source_dur != null ? sc.source_dur : sceneDurationSec(sc); try { @@ -1836,7 +1987,8 @@ `Stitch ${rc.length} clip(s) into one final video?\n\nThis replaces per-scene renders with a single stitched file on the timeline.` )) return; const clips = rc.map((c) => ({ - filename: c.media.filename, subfolder: c.media.subfolder || "", type: c.media.type || "output", + filename: c.media?.filename, subfolder: c.media?.subfolder || "", type: c.media?.type || "output", + bin_media_ref: c.bin_media_ref || "", in: +c.inSec.toFixed(3), dur: +c.durationSec.toFixed(3), fx: c.fx, fps: c.fps, w: c.w, h: c.h, transition: c.transition, tdur: +(c.tdur || 0).toFixed(3), volume: c.volume, scene_id: c.sceneId, @@ -2052,6 +2204,15 @@ function assignMediaToScene(sceneId, mediaId) { const s = scene(sceneId); if (!s) return; + const asset = (state.mediaBin || []).find((m) => m.id === mediaId); + if (asset?.kind === "video") { + if (isVideoClip(s)) { + patchScene(sceneId, { source: { ...(s.source || {}), type: "video", media_ref: mediaId } }); + } else if (confirm("Replace this scene with the imported video clip? Use Convert to video in the inspector to keep a backup of scene settings.")) { + patchScene(sceneId, { source: { type: "video", media_ref: mediaId } }); + } + return; + } // Drag-drop is an explicit new anchor — use image i2v (not mixed/carry guides). const patch = { source: { ...(s.source || {}), type: "image", media_ref: mediaId } }; if ((s.source?.media_ref || null) !== mediaId) patch.guides = []; @@ -2083,6 +2244,7 @@ scheduleSaveFromHistory, notifyHistoryState, refreshProjectList, loadProject, newProject, deleteProject, downloadProject, importProject, patchProject, patchProjectQuiet, patchScene, patchSceneQuiet, flushSave, selectScene, addScene, removeScene, removeSelectedScenes, dismissGhost, moveScene, moveSceneTo, scene, + addVideoClip, convertToVideo, convertToScene, isVideoClip, isGenerativeScene, sceneCharacterIds, toggleSceneCharacter, genUnitId, isGenSubclip, genUnitRoot, genUnitSceneIds, renderPromptForScene, renderPromptMismatch, renderAnchorMismatch, renderMediaLabel, renderIsStale, diff --git a/movie_editor/frontend/styles.css b/movie_editor/frontend/styles.css index 7f68c010..5bc555a2 100644 --- a/movie_editor/frontend/styles.css +++ b/movie_editor/frontend/styles.css @@ -377,6 +377,7 @@ input[type=number] { font-family: var(--font-mono); } .pm-chip.ghost { background: repeating-linear-gradient(-45deg, #2a2418, #2a2418 4px, #3d3420 4px, #3d3420 8px); opacity: .65; } .pm-chip.mixed { background: linear-gradient(180deg, color-mix(in srgb, var(--amber) 55%, #3ab3a8), #2a6a62); opacity: .72; } .pm-chip.carry { background: repeating-linear-gradient(-45deg, #1e2a28, #1e2a28 3px, #243532 3px, #243532 6px); opacity: .5; } +.pm-chip.video { background: linear-gradient(180deg, #3a5a8a, #2a4468); opacity: .7; } .pm-chip:hover { opacity: 1 !important; } .pm-slate { position: absolute; inset: 0; z-index: 2; display: flex; flex-direction: column; align-items: center; justify-content: center; gap: 6px; background: #080808ee; color: var(--muted); pointer-events: none; text-align: center; padding: 16px; } .pm-slate-title { font-size: 14px; color: var(--amber-2); font-family: var(--font-mono); } @@ -639,6 +640,11 @@ body.tl-reordering { cursor: grabbing; user-select: none; } .tl-aud-clip.scene-aud.selected:not(.focus) { border-color: var(--teal-soft); box-shadow: inset 0 0 0 1px rgba(45, 212, 191, 0.35); } .clip.excluded { opacity: .42; } .clip.excluded::before { background: var(--faint); } +.clip.clip-video { background: linear-gradient(#141c28, #0f141c); border-color: #2a4468; } +.clip.clip-video::before { background: #5b8def; opacity: .75; } +.clip.clip-video .clip-no { color: #7eb0ff; font-weight: 600; } +.clip.src-v2v::before { background: #7c6cf0; } +.tl-track2.drop-target-track { outline: 1px dashed var(--teal); outline-offset: -2px; background: rgba(45, 212, 191, 0.04); } .clip.pending:not(.rendered) { border-style: dashed; border-color: color-mix(in srgb, var(--muted) 55%, var(--line)); } .clip.ghost { border-style: dashed; border-color: color-mix(in srgb, var(--amber) 45%, var(--line)); opacity: .72; cursor: default; } .clip.ghost::before { background: var(--amber); opacity: .35; } diff --git a/movie_editor/frontend/timeline.js b/movie_editor/frontend/timeline.js index 2837a512..77c5a49b 100644 --- a/movie_editor/frontend/timeline.js +++ b/movie_editor/frontend/timeline.js @@ -7,7 +7,7 @@ const body = document.getElementById("timeline-body"); const meta = document.getElementById("timeline-meta"); - const SRC_ICON = { empty: "▦", image: "◐", generated_frame: "⛶", carry: "⇥", mixed: "◑" }; + const SRC_ICON = { empty: "▦", image: "◐", generated_frame: "⛶", carry: "⇥", mixed: "◑", video: "▶", v2v: "⟳" }; function genUnitRootScene(scene, p) { const uid = scene.gen_unit_id || scene.id; @@ -23,7 +23,7 @@ function anchorMediaRef(scene, p) { const src = sceneSourceForClip(scene, p); const t = src.type; - if (t === "image" || t === "mixed" || t === "generated_frame") return src.media_ref || null; + if (t === "image" || t === "mixed" || t === "generated_frame" || t === "video" || t === "v2v") return src.media_ref || null; return null; } @@ -234,7 +234,7 @@ const sep = bar.querySelector("[data-separate-audio]"); if (sep) { const sc = hasSel ? S.scene(st.selectedSceneId) : null; - sep.disabled = !(sc && hasRender(st, st.selectedSceneId) && !sc.audio_separated); + sep.disabled = !(sc && S.isGenerativeScene(sc) && hasRender(st, st.selectedSceneId) && !sc.audio_separated); } const oldRating = bar.querySelector(".tl-rating-block"); const freshRating = toolbarRatingBlock(st, st.project); @@ -379,11 +379,12 @@ const src = sceneSourceForClip(scene, p); const srcType = src.type || "empty"; const clip = el("div", "clip" + clipSelClass(st, scene.id) + + (S.isVideoClip(scene) ? " clip-video" : "") + (scene.excluded ? " excluded" : "") - + (hasRender(st, scene.id) ? " rendered" : (!scene.excluded ? " pending" : "")) + + (hasRender(st, scene.id) ? " rendered" : (!scene.excluded && S.isGenerativeScene(scene) ? " pending" : "")) + (hasRender(st, scene.id) && S.renderIsStale?.(scene.id) ? " stale-render" : "") + (unitCuts > 1 ? " gen-cut" : "") + (subclip ? " subclip" : "") - + (srcType === "mixed" ? " src-mixed" : srcType === "carry" ? " src-carry" : srcType === "image" ? " src-image" : "")); + + (srcType === "mixed" ? " src-mixed" : srcType === "carry" ? " src-carry" : srcType === "image" ? " src-image" : srcType === "video" ? " src-video" : srcType === "v2v" ? " src-v2v" : "")); clip.dataset.sceneId = scene.id; clip.style.left = leftPx + "px"; clip.style.width = Math.max(widthPx, 8) + "px"; @@ -464,7 +465,7 @@ } const head = el("div", "clip-head"); - head.append(el("span", "clip-no", p2(index + 1))); + head.append(el("span", "clip-no", S.isVideoClip(scene) ? "V" : p2(index + 1))); appendSrcBadge(head, srcType); head.append(el("span", "clip-dur", timecode(sDur(scene, p), sFps(scene, p)))); clip.append(head); @@ -472,8 +473,11 @@ const root = unitCuts > 1 ? (p.scenes || []).find((s) => (s.gen_unit_id || s.id) === (scene.gen_unit_id || scene.id) && !(s.cut_offset_frames > 0)) : null; - const label = scene.text || (root && root.text) || (subclip ? "cut" : "empty scene"); - clip.append(el("div", "clip-text" + (label && label !== "empty scene" && label !== "cut" ? "" : " empty"), label)); + const mref = anchorMediaRef(scene, p); + const label = S.isVideoClip(scene) + ? ((mref && (st.mediaBin || []).find((m) => m.id === mref)?.name) || "Video clip") + : (scene.text || (root && root.text) || (subclip ? "cut" : "empty scene")); + clip.append(el("div", "clip-text" + (label && label !== "empty scene" && label !== "cut" && label !== "Video clip" ? "" : " empty"), label)); const charIds = S.sceneCharacterIds(scene.id); if (charIds.length) { @@ -690,14 +694,15 @@ panel.append(row); }; - addRow("Clip", "Append a new scene clip to the timeline", () => S.addScene()); + addRow("Clip", "Append a new generative scene clip to the timeline", () => S.addScene()); + addRow("Video", "Add an imported video clip from the Media Browser", () => openVideoClipModal(st)); addRow("Effects", "Post-render clip effect (zoom, blur, fade…)", () => openNleSettingsModal("effect", st)); addRow("Transitions", "Video blend on the outgoing edge of the clip", () => openNleSettingsModal("transition", st)); addRow("Text", "Coming soon", null, true); addRow("Image", "Coming soon", null, true); addRow("Audio", "Add an audio track from the Media Browser", () => openAudioTrackModal(st)); const selSc = st.selectedSceneId ? S.scene(st.selectedSceneId) : null; - const canSepAud = !!(selSc && hasRender(st, st.selectedSceneId) && !selSc.audio_separated); + const canSepAud = !!(selSc && hasRender(st, st.selectedSceneId) && !selSc.audio_separated && S.isGenerativeScene(selSc)); addRow("Separate audio", "Move selected clip's audio to its own track", () => S.separateSceneAudio(st.selectedSceneId), !canSepAud); btn.onclick = (e) => { @@ -713,6 +718,66 @@ return wrap; } + function openVideoClipModal(st) { + closeAddModal(); + const videoAssets = (st.mediaBin || []).filter((m) => m.kind === "video"); + + _addModal = el("div", "modal-overlay"); + const box = el("div", "modal"); + const head = el("div", "modal-head"); + head.append(el("div", "modal-title", "Add video clip")); + const headRight = el("div", "modal-head-right"); + const closeBtn = el("button", "btn ghost tiny", "✕"); + closeBtn.onclick = closeAddModal; + headRight.append(closeBtn); + head.append(headRight); + box.append(head); + + const content = el("div", "modal-content"); + const pick = (mediaId) => { + S.addVideoClip(mediaId); + closeAddModal(); + }; + + if (!videoAssets.length) { + content.append(el("div", "pj-meta", "Upload video in the Media Browser (mp4, mov, webm…), then pick it here.")); + const uploadBtn = el("button", "btn primary tiny", "Upload video file"); + const fileIn = el("input"); + fileIn.type = "file"; + fileIn.accept = "video/*,.mp4,.mov,.webm,.mkv"; + fileIn.style.display = "none"; + uploadBtn.onclick = () => fileIn.click(); + fileIn.onchange = async () => { + const files = [...(fileIn.files || [])]; + fileIn.value = ""; + if (!files.length) return; + await S.uploadMedia(files); + const fresh = S.get(); + const added = (fresh.mediaBin || []).filter((m) => m.kind === "video"); + if (added[0]) pick(added[0].id); + else closeAddModal(); + }; + content.append(uploadBtn); + content.append(fileIn); + } else { + content.append(el("div", "pj-meta", "Video clips play as-is and are skipped by Generate.")); + videoAssets.forEach((m) => { + const row = el("button", "tl-add-row aud-pick-row", m.name || m.id); + row.type = "button"; + row.onclick = () => pick(m.id); + content.append(row); + }); + } + + const cancel = el("button", "btn ghost tiny", "Cancel"); + cancel.onclick = closeAddModal; + content.append(cancel); + box.append(content); + _addModal.append(box); + _addModal.addEventListener("click", (e) => { if (e.target === _addModal) closeAddModal(); }); + document.body.append(_addModal); + } + // ── audio lanes (NLE-style, below video) ─────────────────────────────────────── function openAudioTrackModal(st) { closeAddModal(); @@ -953,7 +1018,7 @@ const Picker = window.MovieRatingPicker; if (!studioCond || !hasSel || !hasRender(st, st.selectedSceneId) || !Picker) return wrap; const sc = S.scene(st.selectedSceneId); - if (!sc) return wrap; + if (!sc || !S.isGenerativeScene(sc)) return wrap; const sceneNo = p.scenes.indexOf(sc) + 1; const raw = sceneRatingRaw(st, sc); const rlabel = el("span", "tl-keys", `Scene ${sceneNo}`); @@ -1000,7 +1065,7 @@ sepAud.dataset.separateAudio = "1"; sepAud.title = "Detach this clip's audio onto its own track (video keeps picture only)"; const selSc = hasSel ? S.scene(st.selectedSceneId) : null; - sepAud.disabled = !(selSc && hasRender(st, st.selectedSceneId) && !selSc?.audio_separated); + sepAud.disabled = !(selSc && hasRender(st, st.selectedSceneId) && !selSc?.audio_separated && S.isGenerativeScene(selSc)); sepAud.onclick = () => S.separateSceneAudio(st.selectedSceneId); bar.append(split); bar.append(del); bar.append(exp); bar.append(sepAud); bar.append(toolbarRatingBlock(st, p)); @@ -1101,6 +1166,23 @@ const tracks = el("div", "tl-tracks"); const track = el("div", "tl-track2"); + track.addEventListener("dragover", (e) => { + if (e.dataTransfer?.types?.includes("application/funpack-media")) { + e.preventDefault(); + track.classList.add("drop-target-track"); + } + }); + track.addEventListener("dragleave", () => track.classList.remove("drop-target-track")); + track.addEventListener("drop", (e) => { + track.classList.remove("drop-target-track"); + const id = e.dataTransfer?.getData("application/funpack-media"); + if (!id) return; + const asset = (st.mediaBin || []).find((m) => m.id === id); + if (asset?.kind === "video") { + e.preventDefault(); + S.addVideoClip(id); + } + }); track.addEventListener("click", (e) => { const clip = e.target.closest(".clip:not(.ghost)[data-scene-id]"); if (!clip) return; diff --git a/movie_editor/frontend/view_bus.js b/movie_editor/frontend/view_bus.js index c9c74ae5..d4778436 100644 --- a/movie_editor/frontend/view_bus.js +++ b/movie_editor/frontend/view_bus.js @@ -13,7 +13,7 @@ s.id, s.text, s.excluded, s.frames, s.fps, s.frames_mode, s.fps_mode, s.video_transition, s.transition_frames, s.transition_to_next, s.effects, s.gen_unit_id, s.cut_offset_frames, s.source?.type, s.source?.media_ref, - s.source_in, s.source_dur, s.rating, s.audio_volume, s.audio_separated, + s.source_in, s.source_dur, s.rating, s.audio_volume, s.audio_separated, s.scene_archive, ]), renders: st.sceneRenders, ghosts: st.sceneGhosts, diff --git a/movie_editor/server.py b/movie_editor/server.py index 3fa8df96..abf4dc92 100644 --- a/movie_editor/server.py +++ b/movie_editor/server.py @@ -840,7 +840,7 @@ if web is not None and PromptServer is not None: return web.json_response({"detail": f"Node registry unavailable: {e}"}, status=502) bridge.reset_progress() bridge.current_progress() # ensure the sampler step-progress hook is installed - active_scenes = [s for s in target.scenes if not s.excluded] + active_scenes = [s for s in target.scenes if not s.excluded and not is_video_clip(s)] active_scene_count = len(active_scenes) # V1 uniform chain: if trimmed scenes all agree on a frame count, use it. # This makes the timeline trim handle actually affect generation length, @@ -856,7 +856,7 @@ if web is not None and PromptServer is not None: if trimmed and all(f == trimmed[0] for f in trimmed) else target.num_frames_per_scene ) - from .backend.timeline import continuity_media_refs + from .backend.timeline import continuity_media_refs, is_video_clip media_pack = _prepare_media(target, continuity_media_refs(p, target)) sampler_inputs = _run_sampler_inputs(target, active_scene_count, full=p) _attach_scene_anchors(sampler_inputs, media_pack, target) @@ -1003,6 +1003,11 @@ if web is not None and PromptServer is not None: return web.json_response({"detail": f"Output directory unavailable: {e}"}, status=500) def _resolve(c): + if c.get("bin_media_ref"): + mp = media.path_for(c.get("bin_media_ref") or "") + if mp is None: + return "" + return str(mp) base = outdir if c.get("type", "output") == "output" else tempdir return os.path.join(base, c.get("subfolder", ""), c.get("filename", "")) diff --git a/movie_editor/tests/test_timeline.py b/movie_editor/tests/test_timeline.py index 03a80d75..dc9747cb 100644 --- a/movie_editor/tests/test_timeline.py +++ b/movie_editor/tests/test_timeline.py @@ -103,6 +103,24 @@ def test_missing_markers_fall_back_to_scene_labels(): assert "scene 2" in prompt +def test_video_clips_excluded_from_combined_prompt(): + p = _project(scenes=[ + {"text": "gen scene", "source": {"type": "carry"}}, + {"text": "", "source": {"type": "video", "media_ref": "vid1"}}, + ]) + prompt = build_combined_prompt(p) + assert "gen scene" in prompt + assert build_combined_prompt(p, include_excluded=True).count("gen scene") >= 1 + + +def test_is_video_clip(): + from movie_editor.backend.timeline import Scene, SceneSource, is_generative_scene, is_video_clip + + sc = Scene(source=SceneSource(type="video", media_ref="v1")) + assert is_video_clip(sc) + assert not is_generative_scene(sc) + + def test_excluded_scenes_dropped_unless_requested(): p = _project( anchor="a cat",