diff --git a/app/legacy/api.js b/app/legacy/api.js index f79f91af..8133fe38 100644 --- a/app/legacy/api.js +++ b/app/legacy/api.js @@ -159,7 +159,11 @@ xhr.onload = () => { let body = {}; try { body = JSON.parse(xhr.responseText || "{}"); } catch (_) { /* not JSON */ } - if (xhr.status >= 200 && xhr.status < 300) resolve(body.media && body.media[0] ? body.media[0] : body); + if (xhr.status >= 200 && xhr.status < 300) { + if (body.media && body.media[0]) resolve(body.media[0]); + else if (body.problems && body.problems.length) reject(new Error(body.problems.join("; "))); + else resolve(body); + } else reject(new Error((body.problems && body.problems.join("; ")) || body.detail || xhr.statusText || `HTTP ${xhr.status}`)); }; xhr.onerror = () => reject(new Error("Network error during upload")); @@ -263,9 +267,19 @@ }, // null while it is still queued or running, else {error} or {videos}. upscaleResult: async (promptId) => { - const res = await fetch(`/history/${encodeURIComponent(promptId)}`); - const entry = res.ok ? (await res.json())[promptId] : null; - if (!entry) return null; + let entry = null; + try { + const res = await fetch(`/history/${encodeURIComponent(promptId)}`); + entry = res.ok ? (await res.json())[promptId] : null; + } catch (_) { return { retry: true }; } // a dropped reply is not a failed job + if (!entry) { + // Not in history: still queued/running, or gone (dequeued, ComfyUI restarted). + try { + const q = await (await fetch("/queue")).json(); + const live = [...(q.queue_running || []), ...(q.queue_pending || [])].some((r) => r[1] === promptId); + return live ? null : { gone: true }; + } catch (_) { return { retry: true }; } + } const st = entry.status || {}; if (st.status_str === "error") { const msg = (st.messages || []).find((m) => m[0] === "execution_error"); @@ -369,6 +383,7 @@ // timeline trim must produce a different URL. Anything else that changes the bytes // (reverse) must be in the URL too, for the same reason. if (spec.dur != null) q.set("dur", String(spec.dur)); + if (spec.srcIn != null) q.set("src_in", String(spec.srcIn)); if (spec.reverse) q.set("rev", "1"); u += "?" + q.toString(); } diff --git a/app/legacy/player.js b/app/legacy/player.js index 6f50a1bf..79e092ba 100644 --- a/app/legacy/player.js +++ b/app/legacy/player.js @@ -177,7 +177,7 @@ const sourceIn = sc?.source_in || 0; const renderIn = Math.max(0, (c.inSec || 0) - sourceIn); c.streamUrl = API.previewSegmentUrl(pid, c.sceneId, { - media: c.media, renderIn, dur: c.durationSec, reverse: wantsReverse, + media: c.media, renderIn, srcIn: sourceIn, dur: c.durationSec, reverse: wantsReverse, }); } } diff --git a/app/legacy/store.js b/app/legacy/store.js index 1329e056..74acb57d 100644 --- a/app/legacy/store.js +++ b/app/legacy/store.js @@ -57,6 +57,8 @@ // {kind:"section"|"node", id, label}. Node ids are project-scoped, which is why these // ride with the project rather than living in the browser alone. pinnedButtons: null, + // Post-render upscale (Settings ▸ Editor): follows the project to a new rental. + upscaleMode: "never", upscaleModel: "", }; function _loadEditorSettings() { try { @@ -3438,7 +3440,7 @@ } async function upscaleMedia(media) { const model = getEditorSetting("upscaleModel"); - if (!media || !media.filename) return; + if (!media || !media.filename || media.subfolder === "funpack_upscaled") return; // never upscale twice if (!model) { set({ notice: "Upscale: pick a model in Settings ▸ Editor ▸ Upscale finished renders." }); return; @@ -3451,15 +3453,20 @@ let notice = ""; try { const promptId = await API.queueUpscale(media, model); + let misses = 0; for (;;) { await new Promise((res) => setTimeout(res, 2000)); const r = await API.upscaleResult(promptId); - if (!r) continue; + if (!r) { misses = 0; continue; } + // A job neither queued nor in history is gone; a reply that fails to arrive is retried a while. + if (r.retry && ++misses < 30) continue; + if (r.retry) throw new Error("lost contact with ComfyUI"); + if (r.gone && ++misses < 3) continue; + if (r.gone) throw new Error("the job left ComfyUI's queue without finishing (interrupted or ComfyUI restarted)"); if (r.error) throw new Error(r.error); const out = { ...r.videos[0], kind: "videos" }; - if (state.project && state.project.id === pid && !_swapRenderMedia(media, out)) { - notice = "Upscaled, but that render was replaced meanwhile, so nothing was swapped."; - } + if (!state.project || state.project.id !== pid) notice = `Upscaled ${media.filename} in another project: it is in output/funpack_upscaled.`; + else if (!_swapRenderMedia(media, out)) notice = "Upscaled, but that render was replaced meanwhile, so nothing was swapped."; break; } } catch (e) { @@ -4750,8 +4757,11 @@ // state.mediaUpload: null when idle, else { current, total, name, loaded, size } -- the // drop zone had no way to show this at all before, so a batch of photos/a big video // uploading over a slow connection just looked like nothing was happening. + let _uploadBatches = 0; async function uploadMedia(files) { const list = [...files]; + const failed = []; + _uploadBatches++; for (let i = 0; i < list.length; i++) { const f = list[i]; // A full notify() here (once per file) is what builds the "uploading" drop zone in the @@ -4762,9 +4772,10 @@ await API.uploadMedia(f, (loaded, total) => { updateMediaUploadProgress({ loaded, size: total || f.size }); }); - } catch (e) { console.error("upload failed", e); } + } catch (e) { console.error("upload failed", e); failed.push(`${f.name}: ${e && e.message ? e.message : e}`); } } - state.mediaUpload = null; + if (--_uploadBatches === 0) state.mediaUpload = null; // a second batch drawn meanwhile keeps its bar + if (failed.length) set({ notice: `Not uploaded — ${failed.join(" · ")}` }); await loadMedia(); } async function deleteMedia(id) { diff --git a/app/legacy/tour.js b/app/legacy/tour.js index bcd5c8d3..0843c103 100644 --- a/app/legacy/tour.js +++ b/app/legacy/tour.js @@ -429,7 +429,7 @@ function patchStore(Store) { const blocked = [ - "generate", "generateMontage", "generateSelected", "renderFinal", "exportSelected", + "generate", "generateMontage", "generateSelected", "renderFinal", "upscaleRender", "exportSelected", "saveSelectedToMediaBin", "commit", "newProject", "loadProject", "deleteProject", "importProject", "downloadProject", "uploadMedia", "deleteMedia", "deleteMediaMany", "interrupt", "resetStudioSession", "syncFromPreview", "saveClipToMediaBin", @@ -462,6 +462,7 @@ generateMontage: "Generate", generateSelected: "Generate Selected", renderFinal: "Render", + upscaleRender: "Upscale", exportSelected: "Export", saveSelectedToMediaBin: "Save to media bin", commit: "Save project", diff --git a/app/legacy/view_bus.js b/app/legacy/view_bus.js index 36a9d197..947175bf 100644 --- a/app/legacy/view_bus.js +++ b/app/legacy/view_bus.js @@ -137,6 +137,9 @@ // button next to Auto Montage. ratingSel: st.selectedSceneId, renders: st.sceneRenders, + // The Upscale button: shown by the mode setting, disabled while that render upscales. + upscale: [window.Store?.getEditorSetting?.("upscaleMode"), + st.selectedSceneId ? window.Store?.isUpscaling?.(st.selectedSceneId) : false], ratings: (st.project?.scenes || []).map((s) => [s.id, s.rating, s.source?.type]), conditioning: st.project?.conditioning_slot, // The Best-FaceID warning chip lives in this zone, so everything its message diff --git a/core/media.py b/core/media.py index d5ff4b52..01800dd4 100644 --- a/core/media.py +++ b/core/media.py @@ -218,7 +218,8 @@ def thumb_path_for(mid: str) -> Path | None: try: if it["kind"] == "image": from PIL import Image - img = Image.open(src) + from PIL import ImageOps + img = ImageOps.exif_transpose(Image.open(src)) # the browser shows it turned upright img.thumbnail((THUMB_MAX_DIM, THUMB_MAX_DIM)) # in place, keeps aspect, never upscales (img if img.mode in ("RGB", "L") else img.convert("RGB")).save(tmp, "JPEG", quality=82) else: diff --git a/core/tests/test_media.py b/core/tests/test_media.py index bb1f19fa..c61fffdc 100644 --- a/core/tests/test_media.py +++ b/core/tests/test_media.py @@ -299,3 +299,15 @@ def test_rename_and_thumb_over_http(server): status, jpg = _request(server, "GET", f"/funpack/api/media/{mid}/thumb") assert status == 200 and jpg["raw"][:2] == b"\xff\xd8" assert _request(server, "GET", "/funpack/api/media/aaaaaaaaaaaa/thumb")[0] == 404 + + +def test_a_photo_thumbnail_is_turned_upright_like_the_browser_shows_it(store): + from PIL import Image + import io + buf = io.BytesIO() + img = Image.new("RGB", (400, 200), (10, 200, 10)) + exif = img.getexif() + exif[274] = 6 # rotate 90 on display + img.save(buf, "JPEG", exif=exif) + entry = media.save_upload("p.jpg", buf.getvalue()) + assert Image.open(media.thumb_path_for(entry["id"])).size == (160, 320) diff --git a/modules/output/render/files.py b/modules/output/render/files.py index da771cc6..82091048 100644 --- a/modules/output/render/files.py +++ b/modules/output/render/files.py @@ -24,6 +24,8 @@ from ..._core import media ISOBMFF = (".mp4", ".m4v", ".mov") # containers where moov/mdat order applies +AUDIO_RATE = 48000 # every exported part is made at this rate, so parts can be joined + FFMPEG_MISSING = "ffmpeg was not found on PATH: install it to preview, export or render clips." @@ -146,7 +148,7 @@ def trim(src: str, out: str, start=None, dur=None, *, fast=False, reverse=False) cmd += ["-c:v", "libx264", "-pix_fmt", "yuv420p"] if fast: cmd += ["-preset", "veryfast"] - cmd += ["-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart", out] + cmd += ["-c:a", "aac", "-b:a", "192k", "-ar", str(AUDIO_RATE), "-ac", "2", "-movflags", "+faststart", out] run(cmd) @@ -161,6 +163,30 @@ def has_audio(path: str) -> bool: return "Audio:" in (proc.stderr or "") +def duration(path: str) -> float | None: + """Seconds of picture in `path`, or None when it has none (a window past the end of the source + trims to a file with no streams at all) or ffprobe is not here to ask.""" + probe = shutil.which("ffprobe") + if not probe: + return None + proc = subprocess.run([probe, "-v", "error", "-select_streams", "v:0", "-show_entries", + "stream=duration:format=duration", "-of", "default=nw=1:nk=1", path], + capture_output=True, text=True) + for line in proc.stdout.split(): + try: + return float(line) + except ValueError: + continue + return 0.0 if proc.returncode == 0 else None + + +def add_silence(src: str, out: str) -> None: + """Copy `src` with a silent stereo track added, so it can be joined to clips that have sound.""" + run([ffmpeg(), "-y", "-i", src, "-f", "lavfi", "-i", f"anullsrc=r={AUDIO_RATE}:cl=stereo", + "-shortest", "-map", "0:v:0", "-map", "1:a:0", "-c:v", "copy", "-c:a", "aac", "-b:a", "192k", + "-movflags", "+faststart", out]) + + def temp_file(name: str) -> str: base = comfy_dir("temp") if not base: diff --git a/modules/output/render/serving.py b/modules/output/render/serving.py index 24f354e1..17b0961d 100644 --- a/modules/output/render/serving.py +++ b/modules/output/render/serving.py @@ -141,8 +141,12 @@ async def result(request) -> "web.StreamResponse": # --- preview segments --------------------------------------------------------- -def scene_clip(project, scene_id: str, render: dict | None = None) -> dict: - """The trim a scene's preview segment must show: the same window the export uses.""" +def scene_clip(project, scene_id: str, render: dict | None = None, window: dict | None = None) -> dict: + """The trim a scene's preview segment must show: the same window the export uses. + + `window` ({dur, src_in} from the URL) wins over the saved project: the editor autosaves + seconds after a trim, and a segment is cached for an hour under its URL, so the bytes + must follow what the URL says, never a project that has not caught up.""" scene = next((s for s in project.scenes if s.id == scene_id), None) if scene is None or scene.excluded: raise KeyError(f"scene {scene_id}") @@ -152,9 +156,15 @@ def scene_clip(project, scene_id: str, render: dict | None = None) -> dict: raise KeyError(f"no render for {scene_id}") fps = float(scene.eff_fps(project) or 25) dur = float(scene.source_dur) if scene.source_dur is not None else scene.eff_frames(project) / fps + src_in = float(scene.source_in or 0) + window = window or {} + if window.get("dur") is not None and window["dur"] > 0: + dur = window["dur"] + if window.get("src_in") is not None and window["src_in"] >= 0: + src_in = window["src_in"] return {"filename": media["filename"], "subfolder": media.get("subfolder") or "", "type": media.get("type") or "output", - "in": float(scene.source_in or 0) + _f(render.get("inSec")), "dur": dur, "fps": fps} + "in": src_in + _f(render.get("inSec")), "dur": dur, "fps": fps} def _f(value, default=0.0): @@ -165,6 +175,17 @@ def _f(value, default=0.0): return default if v != v else v +def query_window(q) -> dict: + """The trim the player asked for; a missing or junk value reads as "use the project's".""" + def num(key): + try: + v = float(q.get(key)) + except (TypeError, ValueError): + return None + return v if v == v and v not in (float("inf"), float("-inf")) else None + return {"dur": num("dur"), "src_in": num("src_in")} + + def query_render(q) -> dict | None: """A render named in the query string overrides the scene's saved one (a clip just made).""" if not q.get("filename"): @@ -192,7 +213,7 @@ async def segment(request, project) -> "web.StreamResponse": q = request.query scene_id = request.match_info["scene_id"] try: - clip = scene_clip(project, scene_id, query_render(q)) + clip = scene_clip(project, scene_id, query_render(q), query_window(q)) except KeyError as exc: clip = ghost_clip(q) if clip is None: diff --git a/modules/output/render/stitch.py b/modules/output/render/stitch.py index 0a95427e..674ed332 100644 --- a/modules/output/render/stitch.py +++ b/modules/output/render/stitch.py @@ -13,6 +13,7 @@ follows the same fold, then extra tracks are delayed to their start and mixed in from __future__ import annotations +import itertools import os import time @@ -197,6 +198,14 @@ def has_graphics(project) -> bool: return bool(project.audio_tracks) or any(_f(ov.get("duration_sec")) > 0 for ov in project.overlay_tracks) +_stamp_n = itertools.count() + + +def _stamp() -> str: + """Unique per job: two jobs in one second must not write the same file.""" + return f"{time.time_ns()}_{next(_stamp_n)}" + + def _clip_spec_ok(c) -> bool: return isinstance(c, dict) and (c.get("bin_media_ref") or c.get("filename")) @@ -215,6 +224,7 @@ def render(project, clips: list[dict]) -> dict: raise RenderError(str(exc)) from exc if clips: cw, ch = int(_f(clips[0].get("w")) or cw), int(_f(clips[0].get("h")) or ch) + cw, ch = cw - cw % 2, ch - ch % 2 # yuv420p cannot be encoded at an odd size fps = _f(clips[0].get("fps")) or fps for c, p in zip(clips, paths): c["has_audio"] = files.has_audio(p) @@ -264,7 +274,7 @@ def render(project, clips: list[dict]) -> dict: cmd += ["-filter_complex", ";".join(lines), "-map", "[vout]"] if has_audio: cmd += ["-map", "[aout]", "-c:a", "aac", "-b:a", "192k"] - name = f"funpack_final_{int(time.time())}.mp4" + name = f"funpack_final_{_stamp()}.mp4" out = os.path.join(tempdir, name) cmd += ["-c:v", "libx264", "-pix_fmt", "yuv420p", "-movflags", "+faststart", out] try: @@ -281,13 +291,27 @@ def concat(clips: list[dict]) -> dict: clips = [c for c in clips if _clip_spec_ok(c)] if isinstance(clips, list) else [] if not clips: raise RenderError("There is nothing to export.") - stamp = int(time.time()) + stamp = _stamp() + parts = [] try: - parts = [] for i, c in enumerate(clips): out = files.temp_file(f"funpack_seg_{stamp}_{i}.mp4") files.trim(files.clip_path(c), out, c.get("in"), c.get("dur")) parts.append(out) + got, want = files.duration(out), _f(c.get("dur")) + if got is not None and (got <= 0 or (want > 0 and got < want - max(0.5, 0.1 * want))): + raise files.ClipError( + f"Clip {i + 1} has {got:.1f}s of picture where the timeline expects {want:.1f}s: " + f"its render is shorter than the window cut from it (or the window starts past " + f"its end). Generate it again, then export.") + sound = [files.has_audio(p) for p in parts] + if any(sound) and not all(sound) and len(parts) > 1: # the joiner keeps only the first part's streams + for k, p in enumerate(parts): + if not sound[k]: + fixed = files.temp_file(f"funpack_seg_{stamp}_{k}_s.mp4") + files.add_silence(p, fixed) + os.remove(p) + parts[k] = fixed name = f"funpack_export_{stamp}.mp4" out = files.temp_file(name) if len(parts) == 1: @@ -308,5 +332,10 @@ def concat(clips: list[dict]) -> dict: except OSError: pass except files.ClipError as exc: + for p in parts: + try: + os.remove(p) + except OSError: + pass raise RenderError(str(exc)) from exc return {"media": {"filename": name, "subfolder": "", "type": "temp", "kind": "videos"}, "clips": len(clips)} diff --git a/modules/output/render/tests/test_serving.py b/modules/output/render/tests/test_serving.py index 0291a7f6..78144ebc 100644 --- a/modules/output/render/tests/test_serving.py +++ b/modules/output/render/tests/test_serving.py @@ -169,3 +169,16 @@ def test_the_cache_deletes_the_oldest_files_past_its_cap(comfy, monkeypatch): paths.append(p) cache.put(i, (None, str(p))) assert not paths[0].exists() and paths[1].exists() and paths[2].exists() and len(cache) == 2 + + +@needs_ffmpeg +def test_the_url_window_wins_over_a_project_that_has_not_been_saved_yet(comfy, server): + moov_at_end(comfy.output / "a.mp4", 4.0) + proj = _project(comfy, source_in=0.0, frames=25, frames_mode="timeline") # saved: 1 s + status, _h, body = get(server, f"{BASE}/projects/{proj.id}/preview-segment/s1?dur=2.5&src_in=1") + assert status == 200 + (comfy.root / "seg.mp4").write_bytes(body) + assert probe(comfy.root / "seg.mp4")[0] == pytest.approx(2.5, abs=0.2) + # a different head trim is a different segment, not the cached one + other = get(server, f"{BASE}/projects/{proj.id}/preview-segment/s1?dur=2.5&src_in=0")[2] + assert other != body diff --git a/modules/output/render/tests/test_stitch.py b/modules/output/render/tests/test_stitch.py index bce16251..8e0b4e47 100644 --- a/modules/output/render/tests/test_stitch.py +++ b/modules/output/render/tests/test_stitch.py @@ -189,3 +189,29 @@ def test_a_separated_audio_lane_plays_the_clips_own_sound_from_its_file(comfy): {"id": "s", "kind": "separated", "scene_id": "sc1", "start_sec": 0}]) out = stitch.render(proj, [clip("a.mp4", dur=2.0, scene_id="sc1")]) assert probe(comfy.temp / out["media"]["filename"])[3] is True + + +@needs_ffmpeg +def test_export_keeps_the_sound_when_the_first_clip_is_silent(comfy): + make_clip(comfy.output / "quiet.mp4", 1.0, audio=False) + make_clip(comfy.output / "loud.mp4", 1.0, audio=True) + out = stitch.concat([{"filename": "quiet.mp4", "dur": 1.0}, {"filename": "loud.mp4", "dur": 1.0}]) + assert files.has_audio(str(comfy.temp / out["media"]["filename"])) + + +@needs_ffmpeg +def test_export_of_a_window_past_the_end_says_so_and_leaves_nothing(comfy): + make_clip(comfy.output / "a.mp4", 1.0) + with pytest.raises(stitch.RenderError, match="shorter than the window|past"): + stitch.concat([{"filename": "a.mp4", "in": 5, "dur": 1.0}]) + with pytest.raises(stitch.RenderError, match="Clip 1"): + stitch.concat([{"filename": "a.mp4", "in": 0, "dur": 3.0}]) + assert not os.listdir(comfy.temp) + + +@needs_ffmpeg +def test_two_exports_in_one_second_do_not_share_a_file(comfy): + make_clip(comfy.output / "a.mp4", 1.0) + first = stitch.concat([{"filename": "a.mp4", "dur": 1.0}])["media"]["filename"] + second = stitch.concat([{"filename": "a.mp4", "dur": 1.0}])["media"]["filename"] + assert first != second and (comfy.temp / first).is_file() and (comfy.temp / second).is_file() diff --git a/modules/output/render/upscale.py b/modules/output/render/upscale.py index 05f41a78..1bf7e5f4 100644 --- a/modules/output/render/upscale.py +++ b/modules/output/render/upscale.py @@ -14,7 +14,8 @@ import subprocess from . import files -CHUNK = 8 # frames per upscale call +CHUNK = 8 # frames per upscale call, at most +CHUNK_PIXELS = 2_000_000 # ...and at most this many input pixels per call: a big upscale of 1080p is gigabytes per frame SUBFOLDER = "funpack_upscaled" @@ -51,6 +52,7 @@ def upscale_file(src, dst, upscale, interrupted=lambda: None): import torch w, h, fps, has_sound = probe(src) frame = w * h * 3 + chunk = max(1, min(CHUNK, CHUNK_PIXELS // max(1, w * h))) ffmpeg = files.ffmpeg() dec = subprocess.Popen([ffmpeg, "-v", "error", "-i", src, "-f", "rawvideo", "-pix_fmt", "rgb24", "-"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) @@ -58,7 +60,7 @@ def upscale_file(src, dst, upscale, interrupted=lambda: None): try: while True: interrupted() - raw = _read(dec.stdout, frame * CHUNK) + raw = _read(dec.stdout, frame * chunk) n = len(raw) // frame if n == 0: break