From 4da41f3333e8dd07054be66c754aeabb894abcc9 Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Mon, 8 Jun 2026 00:28:41 +0300 Subject: [PATCH] Movie Editor: per-scene video effects + seam transitions (render + live preview) Post-decode pixel ops (never re-encode to latent): - Per-scene effects: gaussian blur, fade in/out, Ken-Burns zoom in (crop) / out (pad). Stored in Scene.effects {blur, fade_in, fade_out, zoom}. - Seam video transitions: crossfade/dissolve, fade-through-black, wipe L/R via Scene.video_transition + transition_frames (length). Crossfade overlaps the clips (xfade + audio acrossfade) so the montage shortens by the transition length. - Render route builds a filter_complex (_build_render_filter): per-clip effect chain via scale eval=frame + crop/pad/gblur/fade, then a left-fold of xfade/concat seams. - Live preview approximates per-clip fx (CSS blur/scale/opacity, playhead-synced so it tracks scrubbing); crossfades preview as hard cuts but render correctly. - Inspector exposes all of it per scene. Co-Authored-By: Claude Opus 4.8 --- movie_editor/backend/timeline.py | 11 ++++ movie_editor/frontend/inspector.js | 48 +++++++++++++++++ movie_editor/frontend/player.js | 33 ++++++++++-- movie_editor/frontend/store.js | 16 +++++- movie_editor/server.py | 83 ++++++++++++++++++++++++++++-- 5 files changed, 182 insertions(+), 9 deletions(-) diff --git a/movie_editor/backend/timeline.py b/movie_editor/backend/timeline.py index c840b7a2..9390a0ca 100644 --- a/movie_editor/backend/timeline.py +++ b/movie_editor/backend/timeline.py @@ -51,6 +51,15 @@ class Scene: # Crossfade / transition length at that seam, in frames (post-decode pixel op only; # never re-encodes to latent). None/0 = hard cut. Editor-set via the timeline. transition_frames: Optional[int] = None + # Post-decode VIDEO transition at the seam AFTER this scene (pure pixel op; never + # re-encodes to latent): "" = hard cut | "crossfade" | "fadeblack" | "wipeleft" | + # "wiperight" | "dissolve". Length comes from transition_frames. Distinct from + # transition_to_next (a prompt trigger that shapes generation) — this only affects the + # rendered/previewed pixels at the seam. + video_transition: str = "" + # Per-scene post-decode video effects applied to this clip (pixel ops), keys: + # {blur: 0..1, fade_in: sec, fade_out: sec, zoom: "none"|"in"|"out"}. + effects: dict = field(default_factory=dict) # Forward-compat per-scene knobs (uniform values still win in V1). frames: Optional[int] = None fps: Optional[int] = None @@ -79,6 +88,8 @@ class Scene: text=str(d.get("text", "")), transition_to_next=str(d.get("transition_to_next", "")), transition_frames=d.get("transition_frames"), + video_transition=str(d.get("video_transition", "")), + effects=dict(d.get("effects") or {}), frames=d.get("frames"), fps=d.get("fps"), frames_mode=str(d.get("frames_mode") or "timeline"), diff --git a/movie_editor/frontend/inspector.js b/movie_editor/frontend/inspector.js index b3e91f84..5059e907 100644 --- a/movie_editor/frontend/inspector.js +++ b/movie_editor/frontend/inspector.js @@ -150,6 +150,54 @@ body.append(field("Transition to next scene", transitionSelect(scene.transition_to_next || "", (v) => S.patchScene(scene.id, { transition_to_next: v })))); + // ── Video effects (post-decode pixel ops; applied at render + approximated in preview) + const fxTag = el("div", "insp-tag"); fxTag.textContent = "Video effects"; body.append(fxTag); + const fx = scene.effects || {}; + const patchFx = (k, v, quiet) => { + const next = { ...(scene.effects || {}), [k]: v }; + quiet ? S.patchSceneQuiet(scene.id, { effects: next }) : S.patchScene(scene.id, { effects: next }); + }; + const _num = (val, k, opts) => { + const i = el("input"); i.type = "number"; i.value = val; i.dataset.k = k; + if (opts) Object.assign(i, opts); + return i; + }; + + const blur = el("input"); blur.type = "range"; blur.min = "0"; blur.max = "1"; blur.step = "0.05"; + blur.value = fx.blur || 0; blur.dataset.k = "sc-fx-blur"; + blur.oninput = () => patchFx("blur", parseFloat(blur.value), true); + body.append(field(`Gaussian blur (${Math.round((fx.blur || 0) * 100)}%)`, blur)); + + const fadeRow = el("div", "fields-row"); + const fi = _num(fx.fade_in || 0, "sc-fx-fi", { min: 0, max: 10, step: 0.1 }); + fi.oninput = () => patchFx("fade_in", parseFloat(fi.value || "0"), true); + const fo = _num(fx.fade_out || 0, "sc-fx-fo", { min: 0, max: 10, step: 0.1 }); + fo.oninput = () => patchFx("fade_out", parseFloat(fo.value || "0"), true); + fadeRow.append(field("Fade in (s)", fi)); fadeRow.append(field("Fade out (s)", fo)); + body.append(fadeRow); + + const zoom = el("select"); zoom.dataset.k = "sc-fx-zoom"; + [["none", "None"], ["in", "Zoom in (crop)"], ["out", "Zoom out (pad)"]].forEach(([v, label]) => { + const o = el("option", null, label); o.value = v; if ((fx.zoom || "none") === v) o.selected = true; zoom.append(o); + }); + zoom.onchange = () => patchFx("zoom", zoom.value); + body.append(field("Zoom (Ken Burns)", zoom)); + + // ── Seam transition (rendered crossfade / fade — overlaps & shortens total) + const vt = el("select"); vt.dataset.k = "sc-vt"; + [["", "Hard cut"], ["crossfade", "Crossfade (dissolve)"], ["fadeblack", "Fade through black"], + ["wipeleft", "Wipe left"], ["wiperight", "Wipe right"]].forEach(([v, label]) => { + const o = el("option", null, label); o.value = v; if ((scene.video_transition || "") === v) o.selected = true; vt.append(o); + }); + vt.onchange = () => S.patchScene(scene.id, { video_transition: vt.value }); + body.append(field("Video transition to next", vt)); + if (scene.video_transition) { + const tf = _num(scene.transition_frames || 16, "sc-tf", { min: 1, max: 120, step: 1 }); + tf.oninput = () => S.patchSceneQuiet(scene.id, { transition_frames: parseInt(tf.value || "0", 10) }); + body.append(field("Transition length (frames)", tf)); + body.append(el("div", "insp-hint", "Crossfades overlap the two clips, so the montage gets a little shorter.")); + } + const p = st.project; const lenRow = el("div", "fields-row"); lenRow.append(lengthControl(scene, "frames")); diff --git a/movie_editor/frontend/player.js b/movie_editor/frontend/player.js index 18ef60a2..a1021c54 100644 --- a/movie_editor/frontend/player.js +++ b/movie_editor/frontend/player.js @@ -50,7 +50,7 @@ for (const sc of (p.scenes || [])) { const dur = sFrames(sc) / sFps(sc); const r = sr[sc.id]; - if (r && r.media) out.push({ media: r.media, sceneId: sc.id, startSec: acc, durationSec: dur, inSec: r.inSec || 0 }); + if (r && r.media) out.push({ media: r.media, sceneId: sc.id, startSec: acc, durationSec: dur, inSec: r.inSec || 0, fx: sc.effects || {} }); acc += dur; // timeline advances even over ungenerated scenes (gaps) } return out; @@ -97,10 +97,36 @@ function _setActive(v) { if (_active === v) return; - if (_active) { _active.pause(); _active.classList.remove("active"); } + if (_active) { + _active.pause(); _active.classList.remove("active"); + _active.style.filter = ""; _active.style.transform = ""; _active.style.opacity = ""; // drop stale fx + } _active = v; if (v) v.classList.add("active"); } + // Live preview approximation of a clip's video effects (the render does the real thing). + // within = seconds into the clip. Blur -> CSS blur; zoom in -> scale up (overflow hidden + // by the canvas = crop); zoom out -> scale down (canvas bg shows through = pad); fade -> + // opacity. Computed from the playhead so it tracks scrubbing too. + function _applyFx(clip, within) { + const v = _active; if (!v) return; + const fx = (clip && clip.fx) || {}; + const dur = (clip && clip.durationSec) || 0; + const t = dur > 0 ? Math.max(0, Math.min(1, within / dur)) : 0; + const blur = +fx.blur || 0; + v.style.filter = blur > 0 ? `blur(${(blur * 8).toFixed(1)}px)` : ""; + let scale = 1; + if (fx.zoom === "in") scale = 1 + 0.2 * t; + else if (fx.zoom === "out") scale = 1 - 0.2 * t; + v.style.transformOrigin = "center"; + v.style.transform = scale !== 1 ? `scale(${scale.toFixed(4)})` : ""; + let op = 1; + const fi = +fx.fade_in || 0, fo = +fx.fade_out || 0; + if (fi > 0 && within < fi) op = Math.max(0, within / fi); + if (fo > 0 && dur > 0 && within > dur - fo) op = Math.min(op, Math.max(0, (dur - within) / fo)); + v.style.opacity = op < 1 ? op.toFixed(3) : ""; + } + // Show `clip` at `offset` seconds into the clip; play if requested. Seeks to the clip's // in-point + offset within the (preloaded) source video. function _goto(clip, offset, play) { @@ -115,6 +141,7 @@ _seekPending = target; _playPending = !!play; if (play) _startTick(); } + _applyFx(clip, Math.max(0, offset)); } // Advance to the next clip at a clip's out-point (or stop). Same-source contiguous @@ -141,7 +168,7 @@ if (!_playing || !_active || !_currentClip) return; const within = _active.currentTime - (_currentClip.inSec || 0); if (within >= _currentClip.durationSec - 0.001) { _advance(); } - else { _phSec = _currentClip.startSec + Math.max(0, within); _notifyPh(); } + else { _phSec = _currentClip.startSec + Math.max(0, within); _applyFx(_currentClip, within); _notifyPh(); } if (_playing) _raf = requestAnimationFrame(_tick); } diff --git a/movie_editor/frontend/store.js b/movie_editor/frontend/store.js index c80bbd90..65d58d81 100644 --- a/movie_editor/frontend/store.js +++ b/movie_editor/frontend/store.js @@ -534,11 +534,22 @@ // each non-excluded scene that has a render. This is what plays/exports — so splits // and deletes are honoured (deleted scenes simply aren't here). function _renderClips() { + const p = state.project; const out = []; - for (const sc of (state.project.scenes || [])) { + for (const sc of (p.scenes || [])) { if (sc.excluded) continue; const r = state.sceneRenders[sc.id]; - if (r && r.media) out.push({ media: r.media, inSec: r.inSec || 0, durationSec: sceneDurationSec(sc) }); + if (!(r && r.media)) continue; + const fps = (sc.fps_mode !== "project" && sc.fps != null ? sc.fps : p.frame_rate) || 25; + const tFrames = sc.transition_frames || 0; + out.push({ + media: r.media, inSec: r.inSec || 0, durationSec: sceneDurationSec(sc), + fx: sc.effects || {}, fps, + w: (sc.width != null ? sc.width : p.width) || null, + h: (sc.height != null ? sc.height : p.height) || null, + transition: sc.video_transition || "", + tdur: tFrames > 0 ? tFrames / fps : 0, + }); } return out; } @@ -563,6 +574,7 @@ const clips = rc.map((c) => ({ filename: c.media.filename, subfolder: c.media.subfolder || "", type: c.media.type || "output", in: +c.inSec.toFixed(3), dur: +c.durationSec.toFixed(3), + fx: c.fx, fps: c.fps, w: c.w, h: c.h, transition: c.transition, tdur: +(c.tdur || 0).toFixed(3), })); set({ gen: { state: "running", promptId: null, media: [], msg: `Stitching ${clips.length} clip(s)…` } }); try { diff --git a/movie_editor/server.py b/movie_editor/server.py index 8e6b99d7..83c27f53 100644 --- a/movie_editor/server.py +++ b/movie_editor/server.py @@ -154,6 +154,79 @@ def _run_sampler_inputs(target: Project, scene_count: int) -> dict: return samp +_XFADE_MAP = { + "crossfade": "fade", "fadeblack": "fadeblack", + "wipeleft": "wipeleft", "wiperight": "wiperight", "dissolve": "dissolve", +} + + +def _build_render_filter(clips: list) -> str: + """ffmpeg filter_complex for the final stitch. + + Per clip: apply blur / fade in-out / Ken-Burns zoom (scale eval=frame so the scale + animates over the clip; zoom-in crops the overflow, zoom-out pads into the box). Then + fold the clips left-to-right: a seam with a video_transition uses xfade (overlap, so the + total shortens by the transition length) + acrossfade for audio; otherwise a hard-cut + concat. All pure pixel ops — nothing re-encodes to latent. Returns labels [vout]/[aout]. + """ + n = len(clips) + parts: list[str] = [] + + for i, c in enumerate(clips): + fx = c.get("fx") or {} + dur = float(c.get("dur") or 0) or 0.0 + w = int(c.get("w") or 0) or None + h = int(c.get("h") or 0) or None + vf: list[str] = [] + zoom = fx.get("zoom") + if zoom in ("in", "out") and dur > 0 and w and h: + # animate the scale across the clip via scale's per-frame eval, relative to box. + if zoom == "in": + vf.append(f"scale=w='{w}*(1+0.20*t/{dur:.3f})':h='{h}*(1+0.20*t/{dur:.3f})':eval=frame") + vf.append(f"crop={w}:{h}:(iw-{w})/2:(ih-{h})/2") + else: # out -> shrink and pad into the box + vf.append(f"scale=w='{w}*(1-0.20*t/{dur:.3f})':h='{h}*(1-0.20*t/{dur:.3f})':eval=frame") + vf.append(f"pad={w}:{h}:({w}-iw)/2:({h}-ih)/2:black") + blur = float(fx.get("blur") or 0) + if blur > 0: + vf.append(f"gblur=sigma={blur * 20:.2f}") + fi = float(fx.get("fade_in") or 0) + if fi > 0: + vf.append(f"fade=t=in:st=0:d={fi:.3f}") + fo = float(fx.get("fade_out") or 0) + if fo > 0 and dur > 0: + vf.append(f"fade=t=out:st={max(0.0, dur - fo):.3f}:d={fo:.3f}") + vf.append("format=yuv420p") + vf.append("setsar=1") + parts.append(f"[{i}:v:0]{','.join(vf)}[v{i}]") + parts.append(f"[{i}:a:0]aformat=sample_fmts=fltp:channel_layouts=stereo[a{i}]") + + # Fold the processed streams together. + acc_v, acc_a = "[v0]", "[a0]" + acc_dur = float(clips[0].get("dur") or 0) or 0.0 + for i in range(1, n): + prev = clips[i - 1] + trans = (prev.get("transition") or "").strip() + td = float(prev.get("tdur") or 0) + dur_i = float(clips[i].get("dur") or 0) or 0.0 + if trans in _XFADE_MAP and td > 0 and acc_dur > td: + off = max(0.0, acc_dur - td) + ov, oa = f"[vx{i}]", f"[ax{i}]" + parts.append(f"{acc_v}[v{i}]xfade=transition={_XFADE_MAP[trans]}:duration={td:.3f}:offset={off:.3f}{ov}") + parts.append(f"{acc_a}[a{i}]acrossfade=d={td:.3f}{oa}") + acc_v, acc_a, acc_dur = ov, oa, acc_dur + dur_i - td + else: + ov, oa = f"[vc{i}]", f"[ac{i}]" + parts.append(f"{acc_v}[v{i}]concat=n=2:v=1:a=0{ov}") + parts.append(f"{acc_a}[a{i}]concat=n=2:v=0:a=1{oa}") + acc_v, acc_a, acc_dur = ov, oa, acc_dur + dur_i + + # Final relabel to [vout]/[aout]. + parts.append(f"{acc_v}null[vout]") + parts.append(f"{acc_a}anull[aout]") + return ";".join(parts) + + def _segment(p: Project, scene_ids: list) -> Project: """Project clone holding only `scene_ids` (in their project order). Used to generate one chain run: its first scene supplies the i2v anchor, the rest are @@ -586,8 +659,10 @@ if web is not None and PromptServer is not None: # The final render is ephemeral (temp dir) — persist it via the Export Save dialog. out_name = f"funpack_final_{int(_time.time())}.mp4" out_path = os.path.join(tempdir, out_name) - # Per-clip in/out via -ss/-t before each input, concatenated with the concat filter - # (re-encode; keeps the LTXAV audio). This honours splits and deletions. + # Per-clip in/out via -ss/-t before each input, then a filter graph that applies + # per-clip video effects (blur/fade/zoom — pure pixel ops, never re-encode to + # latent) and folds clips together with xfade/concat seam transitions. Re-encode + # keeps the LTXAV audio. cmd = [ff, "-y"] for c, pth in zip(clips, paths): inn, dur = c.get("in"), c.get("dur") @@ -597,9 +672,9 @@ if web is not None and PromptServer is not None: cmd += ["-t", f"{float(dur):.3f}"] cmd += ["-i", pth] n = len(clips) - filt = "".join(f"[{i}:v:0][{i}:a:0]" for i in range(n)) + f"concat=n={n}:v=1:a=1[v][a]" + filt = _build_render_filter(clips) cmd += [ - "-filter_complex", filt, "-map", "[v]", "-map", "[a]", + "-filter_complex", filt, "-map", "[vout]", "-map", "[aout]", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart", out_path, ]