diff --git a/app/legacy/store.js b/app/legacy/store.js index 74acb57d..7368e8cf 100644 --- a/app/legacy/store.js +++ b/app/legacy/store.js @@ -3450,7 +3450,7 @@ if (!pid || _upscaling.has(key)) return; _upscaling.add(key); set({ notice: `Upscaling with ${model}…` }); - let notice = ""; + let notice = null; // null: leave whatever else was said meanwhile try { const promptId = await API.queueUpscale(media, model); let misses = 0; @@ -3473,7 +3473,7 @@ notice = `Upscale failed: ${e && e.message ? e.message : e}`; } finally { _upscaling.delete(key); - set({ notice }); + set({ notice: notice !== null ? notice : (state.notice && state.notice.startsWith("Upscaling with") ? "" : state.notice) }); } } function upscaleRender(sceneId) { @@ -4670,6 +4670,7 @@ }); const name = (state.project.name || "montage").replace(/[^\w.-]+/g, "_") + `_final_${_stamp()}.mp4`; await _saveBlobAs(API.resultUrl(state.project.id, r.media), name); + if (r.warnings && r.warnings.length) set({ notice: `Rendered, but: ${r.warnings.join(" ")}` }); } catch (e) { set({ gen: { state: "error", promptId: null, media: [], msg: "Render failed: " + _friendlyGenError(e.message) } }); } diff --git a/app/legacy/timeline.js b/app/legacy/timeline.js index 10da977e..6da1dbd1 100644 --- a/app/legacy/timeline.js +++ b/app/legacy/timeline.js @@ -234,7 +234,7 @@ // and shared across every scene in the run, so a filmstrip per clip meant N full-file // downloads with preload=auto fighting the player pool over Chrome's 6-per-origin cap. const segUrl = API.previewSegmentUrl && st.project.id - ? API.previewSegmentUrl(st.project.id, scene.id, { media: r.media, renderIn: r.inSec || 0, dur: durSec }) + ? API.previewSegmentUrl(st.project.id, scene.id, { media: r.media, renderIn: r.inSec || 0, srcIn: scene.source_in || 0, dur: durSec }) : null; const url = segUrl || API.resultUrl(st.project.id, r.media); _fsEnqueue(() => new Promise((finish) => { @@ -1335,7 +1335,7 @@ // trimmed segment is this scene's span only, so no inSec slice is needed to paint it. const API = window.MovieEditorAPI; const segUrl = API.previewSegmentUrl - ? API.previewSegmentUrl(st.project.id, scene.id, { media: r.media, renderIn: r.inSec || 0, dur: durSec }) + ? API.previewSegmentUrl(st.project.id, scene.id, { media: r.media, renderIn: r.inSec || 0, srcIn: scene.source_in || 0, dur: durSec }) : null; _attachWaveform(canvas, `scene-aud:${scene.id}:${inSec.toFixed(3)}:${durSec.toFixed(3)}`, segUrl || API.resultUrl(st.project.id, r.media), segUrl ? { width: w, diff --git a/app/legacy/view_bus.js b/app/legacy/view_bus.js index 947175bf..ad112548 100644 --- a/app/legacy/view_bus.js +++ b/app/legacy/view_bus.js @@ -139,7 +139,7 @@ renders: st.sceneRenders, // The Upscale button: shown by the mode setting, disabled while that render upscales. upscale: [window.Store?.getEditorSetting?.("upscaleMode"), - st.selectedSceneId ? window.Store?.isUpscaling?.(st.selectedSceneId) : false], + (st.project?.scenes || []).map((s) => !!window.Store?.isUpscaling?.(s.id))], ratings: (st.project?.scenes || []).map((s) => [s.id, s.rating, s.source?.type]), conditioning: st.project?.conditioning_slot, // The Best-FaceID warning chip lives in this zone, so everything its message diff --git a/modules/output/render/files.py b/modules/output/render/files.py index 82091048..d7322009 100644 --- a/modules/output/render/files.py +++ b/modules/output/render/files.py @@ -132,7 +132,7 @@ def remux_faststart(src: str, out: str) -> None: run([ffmpeg(), "-y", "-i", src, "-c", "copy", "-movflags", "+faststart", out]) -def trim(src: str, out: str, start=None, dur=None, *, fast=False, reverse=False) -> None: +def trim(src: str, out: str, start=None, dur=None, *, fast=False, reverse=False, fps=None) -> None: """Cut [start, start+dur) out of `src` into a seekable h264/aac mp4. `fast` is for previews, where the latency of a scrub across a clip boundary matters @@ -145,6 +145,8 @@ def trim(src: str, out: str, start=None, dur=None, *, fast=False, reverse=False) cmd += ["-i", src] if reverse: cmd += ["-vf", "reverse", "-af", "areverse"] # after the trim: only the clip is buffered + if fps: + cmd += ["-r", str(fps)] # parts to be joined must share one rate cmd += ["-c:v", "libx264", "-pix_fmt", "yuv420p"] if fast: cmd += ["-preset", "veryfast"] @@ -180,6 +182,18 @@ def duration(path: str) -> float | None: return 0.0 if proc.returncode == 0 else None +def frame_rate(path: str) -> str | None: + """The first video stream's rate as ffprobe spells it ("25/1"), or None.""" + probe = shutil.which("ffprobe") + if not probe: + return None + proc = subprocess.run([probe, "-v", "error", "-select_streams", "v:0", "-show_entries", + "stream=r_frame_rate", "-of", "default=nw=1:nk=1", path], + capture_output=True, text=True) + out = proc.stdout.strip() + return out if out and out != "0/0" else None + + def add_silence(src: str, out: str) -> None: """Copy `src` with a silent stereo track added, so it can be joined to clips that have sound.""" run([ffmpeg(), "-y", "-i", src, "-f", "lavfi", "-i", f"anullsrc=r={AUDIO_RATE}:cl=stereo", diff --git a/modules/output/render/serving.py b/modules/output/render/serving.py index 17b0961d..5e733780 100644 --- a/modules/output/render/serving.py +++ b/modules/output/render/serving.py @@ -200,7 +200,7 @@ def ghost_clip(q) -> dict | None: if not q.get("filename") or q.get("dur") is None: return None try: - start, dur = float(q.get("render_in") or 0), float(q["dur"]) + start, dur = float(q.get("render_in") or 0) + float(q.get("src_in") or 0), float(q["dur"]) except (TypeError, ValueError): return None if start != start or dur != dur: @@ -245,6 +245,11 @@ async def segment(request, project) -> "web.StreamResponse": f"{projects.safe_part(scene_id)}_{int(time.time() * 1000)}.mp4") await asyncio.to_thread(files.trim, src, out, clip.get("in"), clip.get("dur"), fast=True, reverse=reverse) + got = files.duration(out) + if got is not None and got <= 0: # a window past the end trims to a file with no picture + os.remove(out) + return web.json_response({"detail": "This clip's window starts past the end of its render: " + "generate it again."}, status=400) except files.ClipError as exc: return web.json_response({"detail": str(exc)}, status=503) _segments.put(key, (None, out)) diff --git a/modules/output/render/stitch.py b/modules/output/render/stitch.py index 674ed332..7139b88d 100644 --- a/modules/output/render/stitch.py +++ b/modules/output/render/stitch.py @@ -59,6 +59,7 @@ def build_filter(clips: list[dict], tracks: list[dict] | None = None, *, keep_or raise RenderError("There is nothing to render: no clips, no overlays and no audio.") cw = int(_f(clips[0].get("w")) or 768) ch = int(_f(clips[0].get("h")) or 768) + cw, ch = cw - cw % 2, ch - ch % 2 # yuv420p cannot be encoded at an odd size fps = _f(clips[0].get("fps")) or 25.0 for i, c in enumerate(clips): dur = _f(c.get("dur")) @@ -131,6 +132,8 @@ def _clip_audio(i: int, clip: dict, dur: float) -> str: vol = _f(clip.get("volume"), 1.0) if abs(vol - 1.0) > 1e-3: af += f",volume={max(0.0, vol):.3f}" + if dur > 0: # exactly the picture's length: a short sound must not pull later clips early + af += f",apad,atrim=0:{dur:.3f},asetpts=PTS-STARTPTS" return f"[{i}:a:0]{af}[a{i}]" @@ -224,12 +227,12 @@ def render(project, clips: list[dict]) -> dict: raise RenderError(str(exc)) from exc if clips: cw, ch = int(_f(clips[0].get("w")) or cw), int(_f(clips[0].get("h")) or ch) - cw, ch = cw - cw % 2, ch - ch % 2 # yuv420p cannot be encoded at an odd size fps = _f(clips[0].get("fps")) or fps for c, p in zip(clips, paths): c["has_audio"] = files.has_audio(p) blank_canvas = None else: + cw, ch = cw - cw % 2, ch - ch % 2 blank_canvas = {"w": cw, "h": ch, "fps": fps, "dur": graphics_duration(project)} try: ffmpeg = files.ffmpeg() @@ -263,6 +266,11 @@ def render(project, clips: list[dict]) -> dict: drawn, pictures = overlays.prepare(project.overlay_tracks, project.overlay_lanes, tempdir=tempdir, image_path=image_path) + warnings = [] + if len(drawn) < len(project.overlay_tracks): + warnings.append(f"{len(project.overlay_tracks) - len(drawn)} overlay(s) left out: their picture is gone from the media library.") + if len(tracks) < len(project.audio_tracks): + warnings.append(f"{len(project.audio_tracks) - len(tracks)} audio lane(s) left out: their file is gone.") for path in pictures: cmd += ["-i", path] graph, has_audio = build_filter(clips, tracks, keep_original=keep, base_input=base, blank=blank_canvas) @@ -282,7 +290,7 @@ def render(project, clips: list[dict]) -> dict: except files.ClipError as exc: raise RenderError(f"ffmpeg could not make the render: {exc}") from exc return {"media": {"filename": name, "subfolder": "", "type": "temp", "kind": "videos"}, - "clips": len(clips) or 1} + "clips": len(clips) or 1, "warnings": warnings} def concat(clips: list[dict]) -> dict: @@ -293,10 +301,14 @@ def concat(clips: list[dict]) -> dict: raise RenderError("There is nothing to export.") stamp = _stamp() parts = [] + fps = None try: for i, c in enumerate(clips): out = files.temp_file(f"funpack_seg_{stamp}_{i}.mp4") - files.trim(files.clip_path(c), out, c.get("in"), c.get("dur")) + src = files.clip_path(c) + if fps is None: + fps = files.frame_rate(src) # the first clip's rate: the joiner keeps one + files.trim(src, out, c.get("in"), c.get("dur"), fps=fps) parts.append(out) got, want = files.duration(out), _f(c.get("dur")) if got is not None and (got <= 0 or (want > 0 and got < want - max(0.5, 0.1 * want))): diff --git a/modules/output/render/tests/test_serving.py b/modules/output/render/tests/test_serving.py index 78144ebc..a6bf0205 100644 --- a/modules/output/render/tests/test_serving.py +++ b/modules/output/render/tests/test_serving.py @@ -182,3 +182,24 @@ def test_the_url_window_wins_over_a_project_that_has_not_been_saved_yet(comfy, s # a different head trim is a different segment, not the cached one other = get(server, f"{BASE}/projects/{proj.id}/preview-segment/s1?dur=2.5&src_in=0")[2] assert other != body + + +@needs_ffmpeg +def test_a_ghost_whose_scene_is_not_saved_yet_still_gets_its_in_point(comfy, server): + make_clip(comfy.output / "a.mp4", 4.0) + proj = _project(comfy) + url = (f"{BASE}/projects/{proj.id}/preview-segment/new?filename=a.mp4&subfolder=&type=output" + "&render_in=0&src_in=2&dur=1.5") + status, _h, a = get(server, url) + status2, _h, b = get(server, url.replace("src_in=2", "src_in=0")) + assert status == status2 == 200 and a != b + + +@needs_ffmpeg +def test_a_window_past_the_end_of_the_render_is_refused_not_served_empty(comfy, server): + make_clip(comfy.output / "a.mp4", 1.0) + proj = _project(comfy) + url = (f"{BASE}/projects/{proj.id}/preview-segment/gone?filename=a.mp4&subfolder=&type=output" + "&render_in=5&dur=1") + status, _h, body = get(server, url) + assert status == 400 and b"past the end" in body diff --git a/modules/output/render/tests/test_stitch.py b/modules/output/render/tests/test_stitch.py index 8e0b4e47..2a522b7c 100644 --- a/modules/output/render/tests/test_stitch.py +++ b/modules/output/render/tests/test_stitch.py @@ -215,3 +215,51 @@ def test_two_exports_in_one_second_do_not_share_a_file(comfy): first = stitch.concat([{"filename": "a.mp4", "dur": 1.0}])["media"]["filename"] second = stitch.concat([{"filename": "a.mp4", "dur": 1.0}])["media"]["filename"] assert first != second and (comfy.temp / first).is_file() and (comfy.temp / second).is_file() + + +def _audio_seconds(path): + import subprocess + out = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "a:0", "-show_entries", + "stream=duration", "-of", "default=nw=1:nk=1", str(path)], + capture_output=True, text=True).stdout.split() + return float(out[0]) + + +@needs_ffmpeg +def test_a_clip_whose_sound_is_shorter_than_its_picture_does_not_pull_later_sound_early(comfy): + import subprocess + short = comfy.output / "short.mp4" + subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "testsrc=duration=2:size=160x120:rate=25", + "-f", "lavfi", "-i", "sine=frequency=440:duration=1.5", "-c:v", "libx264", + "-pix_fmt", "yuv420p", "-c:a", "aac", str(short)], check=True, capture_output=True) + out = stitch.render(projects.Project(width=160, height=120), + [clip("short.mp4", dur=2.0), clip("short.mp4", dur=2.0), clip("short.mp4", dur=2.0)]) + assert _audio_seconds(comfy.temp / out["media"]["filename"]) == pytest.approx(6.0, abs=0.15) + + +@needs_ffmpeg +def test_an_odd_canvas_renders(comfy): + make_clip(comfy.output / "a.mp4", 1.0, size="160x120") + out = stitch.render(projects.Project(width=161, height=121), [clip("a.mp4", dur=1.0, w=161, h=121)]) + assert probe(comfy.temp / out["media"]["filename"])[1:3] == (160, 120) + + +@needs_ffmpeg +def test_export_of_clips_with_different_frame_rates_keeps_picture_and_sound_together(comfy): + make_clip(comfy.output / "a.mp4", 2.0, rate=25) + make_clip(comfy.output / "b.mp4", 2.0, rate=30) + out = stitch.concat([{"filename": "a.mp4", "dur": 2.0}, {"filename": "b.mp4", "dur": 2.0}]) + path = comfy.temp / out["media"]["filename"] + assert probe(path)[0] == pytest.approx(4.0, abs=0.25) + assert _audio_seconds(path) == pytest.approx(4.0, abs=0.25) + + +@needs_ffmpeg +def test_a_render_says_what_it_left_out(comfy): + make_clip(comfy.output / "a.mp4", 1.0) + proj = projects.Project(width=160, height=120, + overlay_tracks=[{"id": "o1", "kind": "image", "media_ref": "aaaaaaaaaaaa", + "start_sec": 0, "duration_sec": 1}], + audio_tracks=[{"id": "t1", "media_ref": "bbbbbbbbbbbb", "start_sec": 0}]) + out = stitch.render(proj, [clip("a.mp4", dur=1.0)]) + assert len(out["warnings"]) == 2 and "overlay" in out["warnings"][0] and "audio" in out["warnings"][1]