From 0d2b6779beb7cc54b280b5ea9c68873cbe68ab08 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 18:58:03 +0000 Subject: [PATCH] feat: show a clip's frame count and rate in the training pickers and tags step Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01MpnF7GwNGMbngU11wjym7P --- app/ui/features/training/data.js | 10 +++++++--- app/ui/features/training/index.js | 12 ++++++++++-- app/ui/features/training/state.js | 5 +++++ app/ui/features/training/tags.js | 5 +++-- app/ui/features/training/training.test.js | 12 ++++++++++++ app/ui/shell/api.js | 1 + modules/training/lora/__init__.py | 17 ++++++++++++++++- modules/training/lora/media_io.py | 15 +++++++++++++++ .../training/lora/tests/test_sets_and_tags.py | 7 +++++++ 9 files changed, 76 insertions(+), 8 deletions(-) diff --git a/app/ui/features/training/data.js b/app/ui/features/training/data.js index b7296c61..cd5a0308 100644 --- a/app/ui/features/training/data.js +++ b/app/ui/features/training/data.js @@ -1,7 +1,7 @@ // Step 1, Training data: pick or make a set, then put media in it from any mix of sources. // Skippable: a set that already has what you want goes straight on to tags or to training. import { composer as c } from "../../composer/composer.js"; -import { LABEL_WORDS, stillOf } from "./state.js"; +import { LABEL_WORDS, clipWords, srcKey, stillOf } from "./state.js"; export const RATED = [ { value: "liked", label: "Liked results" }, @@ -59,13 +59,16 @@ export function dataStep({ app, session, go, ui }) { const picked = ui.picked; const pickedNote = c.text.sm({ text: `${picked.size} picked` }); const usable = ui.bin.filter((m) => m.kind === "image" || m.kind === "video"); - const gallery = c.gallery.adaptive({ id: "training-bin", cols: 3, items: usable.map((m) => ({ id: m.id, label: m.name, thumb: `/funpack/api/media/${encodeURIComponent(m.id)}/thumb`, badge: m.kind })), selection: [...picked], + const binSrc = (m) => ({ kind: "bin", id: m.id }); + const binItems = () => usable.map((m) => ({ id: m.id, label: m.name, thumb: `/funpack/api/media/${encodeURIComponent(m.id)}/thumb`, badge: m.kind === "video" ? clipWords(ui.clips[srcKey(binSrc(m))]) || m.kind : m.kind })); + const gallery = c.gallery.adaptive({ id: "training-bin", cols: 3, items: binItems(), selection: [...picked], empty: "Your Bin has no pictures or clips yet", onActivate: (item) => { if (picked.has(item.id)) picked.delete(item.id); else picked.add(item.id); gallery.setValue([...picked]); pickedNote.setText(`${picked.size} picked`); addBin.node.disabled = !picked.size; } }); + ui.needClips(usable.filter((m) => m.kind === "video").map(binSrc), () => { gallery.setItems(binItems()); gallery.setValue([...picked]); }); const addBin = c.button.sm({ label: "Add picked", tone: "primary", disabled: !picked.size || !open, onClick: async () => { const chosen = usable.filter((m) => picked.has(m.id)); await added(chosen.map(binEntry), "the Bin"); @@ -91,7 +94,8 @@ export function dataStep({ app, session, go, ui }) { function showRated() { const all = shownRated(); for (const k of [...ratedPicked]) if (!all.some((r) => keyOf(r) === k)) ratedPicked.delete(k); - ratedGallery.setItems(all.map((r) => ({ id: keyOf(r), label: r.media.filename, thumb: stillOf({ src: { kind: "output", ...r.media }, media_kind: r.media.filename.match(/\.(mp4|webm|mov|mkv|avi)$/i) ? "video" : "image" }), badge: r.label }))); + ratedGallery.setItems(all.map((r) => ({ id: keyOf(r), label: r.media.filename, thumb: stillOf({ src: { kind: "output", ...r.media }, media_kind: r.media.filename.match(/\.(mp4|webm|mov|mkv|avi)$/i) ? "video" : "image" }), badge: [r.label, clipWords(ui.clips[keyOf(r)])].filter(Boolean).join(" · ") }))); + ui.needClips(all.filter((r) => /\.(mp4|webm|mov|mkv|avi)$/i.test(r.media.filename)).map((r) => ({ kind: "output", ...r.media })), () => showRated()); ratedGallery.setValue([...ratedPicked]); const n = all.length, p = ratedPicked.size; ratedNote.setText(n ? `${n} rated result${n > 1 ? "s" : ""} found in your projects. ${p ? `${p} picked: only those are added.` : "None picked: all are added."}` : "None found yet. Rate results in your projects and they appear here."); diff --git a/app/ui/features/training/index.js b/app/ui/features/training/index.js index f353607e..88f3522e 100644 --- a/app/ui/features/training/index.js +++ b/app/ui/features/training/index.js @@ -5,7 +5,7 @@ import { dataStep } from "./data.js"; import { applyTagging, tagsStep } from "./tags.js"; import { monitor, settingsOf, trainStep } from "./train.js"; import { resultsStep } from "./results.js"; -import { makeSession } from "./state.js"; +import { makeSession, srcKey } from "./state.js"; const TABS = [{ value: "data", label: "1 Data" }, { value: "tags", label: "2 Tags" }, { value: "train", label: "3 Train" }, { value: "results", label: "Results" }]; const POLL = 1000; @@ -26,12 +26,20 @@ export default { const api = app.api; const session = makeSession(api); const ui = { - tab: "data", picked: new Set(), ratedPicked: new Set(), bin: [], rated: { results: [], counts: {} }, ratedKind: "liked", include: ["main"], selected: new Set(), filter: "all", tag: "", limit: 48, + tab: "data", clips: {}, picked: new Set(), ratedPicked: new Set(), bin: [], rated: { results: [], counts: {} }, ratedKind: "liked", include: ["main"], selected: new Set(), filter: "all", tag: "", limit: 48, tagger: { id: "wd_vit_v3", list: [], job: null, general: 0.35, limit: 0, characters: false, mode: "add", scope: "untagged" }, advanced: false, runs: [], chosenRun: "", record: null, run: { shown: false, active: false, state: "idle", loss: [], smooth: [], step: 0, total: 0, elapsed: 0, message: "", error: "", lora: "", before: "" }, monitorBox: null, redraw: () => draw(), pollTaggers: () => pollTaggers(true), loadRecord: (id) => loadRecord(id), loadRuns: () => loadRuns(), start: () => start(), stop: () => stop(), + /** Ask for the frame counts of clips not known yet; `then` runs when new numbers arrived. */ + async needClips(srcs, then) { + const fresh = srcs.filter((x) => !(srcKey(x) in ui.clips)); + if (!fresh.length) return; + fresh.forEach((x) => { ui.clips[srcKey(x)] = null; }); + try { Object.assign(ui.clips, (await api.trainClips(fresh)).clips || {}); } catch { fresh.forEach((x) => { delete ui.clips[srcKey(x)]; }); return; } + if (then) then(); + }, }; let win = null, body = null, timers = [], offSession = null, tagPoll = null, runPoll = null; diff --git a/app/ui/features/training/state.js b/app/ui/features/training/state.js index 2574c464..e6d5a625 100644 --- a/app/ui/features/training/state.js +++ b/app/ui/features/training/state.js @@ -15,6 +15,11 @@ export function stillOf(item) { return item.media_kind === "video" ? `/funpack/api/m/render/poster?${q}` : `/view?${q}`; } +/** The key a source has on the server (store.src_key). */ +export const srcKey = (src) => (src.kind === "bin" ? `bin:${src.id}` : `${src.type || "output"}:${src.subfolder || ""}/${src.filename}`); +/** "124 frames, 24 fps" for a clip whose numbers are known, else "". */ +export const clipWords = (info) => (info ? `${info.frames} f · ${info.fps} fps` : ""); + export function makeSession(api) { const listeners = new Set(); const s = { name: "", doc: null, sets: [], error: "", saving: false, state: null }; diff --git a/app/ui/features/training/tags.js b/app/ui/features/training/tags.js index 71a36403..5152e80e 100644 --- a/app/ui/features/training/tags.js +++ b/app/ui/features/training/tags.js @@ -1,6 +1,6 @@ // Step 2, Tags: every item with its caption beside its picture, bulk edits, a tag cloud, and automatic tagging. import { composer as c } from "../../composer/composer.js"; -import { LABEL_TONES, LABEL_WORDS, stillOf } from "./state.js"; +import { LABEL_TONES, LABEL_WORDS, clipWords, srcKey, stillOf } from "./state.js"; import { addTags, bulk, frequency, joinTags, splitTags } from "./captions.js"; const PAGE = 48; @@ -28,10 +28,11 @@ export function tagsStep({ app, session, go, ui, peek }) { const cards = list.slice(0, ui.limit).map((item) => { const card = c.captioned.cell({ title: item.name || item.src.filename || item.src.id, src: stillOf(item), kind: item.media_kind, caption: item.caption, - badge: LABEL_WORDS[item.label], tone: LABEL_TONES[item.label], selected: sel.has(item.id), + badge: [LABEL_WORDS[item.label], clipWords(ui.clips[srcKey(item.src)])].filter(Boolean).join(" · "), tone: LABEL_TONES[item.label], selected: sel.has(item.id), onChange: (text) => session.edit(() => { item.caption = text; }), onSelect: (on) => { if (on) sel.add(item.id); else sel.delete(item.id); sync(); }, onOpen: () => peek(item) }); + if (item.media_kind === "video") ui.needClips([item.src], () => card.setBadge([LABEL_WORDS[item.label], clipWords(ui.clips[srcKey(item.src)])].filter(Boolean).join(" · "))); return card; }); const grid = cards.length ? c.grid.cards({ children: cards }) : c.hint.default({ text: ui.filter === "untagged" ? "Everything has tags." : "Nothing to show." }); diff --git a/app/ui/features/training/training.test.js b/app/ui/features/training/training.test.js index ec53460f..d5cf6c00 100644 --- a/app/ui/features/training/training.test.js +++ b/app/ui/features/training/training.test.js @@ -21,6 +21,7 @@ function fake({ enabled = true } = {}) { trainSets: async () => ({ sets: [{ name: "cats", count: 2 }] }), trainSet: async () => structuredClone(doc), trainSaveSet: async (name, d) => { calls.push(["save", name, d]); return d; }, + trainClips: async () => ({ clips: { "output:/v.mp4": { frames: 124, fps: 24 } } }), trainAdd: async () => ({ set: doc, added: 0 }), trainRated: async () => ({ results: [], counts: { liked: 3 } }), trainTaggers: async () => ({ taggers: [{ id: "wd_vit_v3", title: "WD ViT v3", state: "absent", hint: "", done: 0, total: 0 }] }), @@ -273,3 +274,14 @@ test("rated results are shown as pictures, and only the picked ones are added", assert.equal(added[0][0].src.filename, "y.png"); off(); }); + +test("Tags step: a clip's frame count and rate show beside its label", async () => { + const f = fake(); + const { button, off } = await open(f); + button.click(); + await wait(60); + click("Next: tags").click(); + await wait(100); + assert.ok(text().includes("124 f · 24 fps")); + off(); +}); diff --git a/app/ui/shell/api.js b/app/ui/shell/api.js index f8d44cbe..ea4f648a 100644 --- a/app/ui/shell/api.js +++ b/app/ui/shell/api.js @@ -87,6 +87,7 @@ export const api = { trainDeleteSet: (name) => call("DELETE", `${TRAIN}/sets/${encodeURIComponent(name)}`), trainAdd: (name, entries) => call("POST", `${TRAIN}/sets/${encodeURIComponent(name)}/add`, { entries }), trainBulk: (name, body) => call("POST", `${TRAIN}/sets/${encodeURIComponent(name)}/bulk`, body), + trainClips: (srcs) => call("POST", `${TRAIN}/clips`, { srcs }), trainRated: (include) => call("GET", `${TRAIN}/rated${include && include.length ? `?include=${encodeURIComponent(include.join(","))}` : ""}`), trainTaggers: () => call("GET", `${TRAIN}/taggers`), trainDownloadTagger: (tagger) => call("POST", `${TRAIN}/taggers/download`, { tagger }), diff --git a/modules/training/lora/__init__.py b/modules/training/lora/__init__.py index ff25e8b0..5a640bec 100644 --- a/modules/training/lora/__init__.py +++ b/modules/training/lora/__init__.py @@ -9,7 +9,7 @@ H3 trains picture and sound together; the repairs its latent and its rotary op n used when this module asks. UNTESTED on real weights or hardware: proven only on tiny stand-in models on CPU. """ -from . import captions, jobs, plan, rated, store, taggers, train +from . import captions, jobs, media_io, plan, rated, store, taggers, train from .nodes import FunPackTrainLora ID = "training_lora" @@ -117,6 +117,21 @@ def routes(table, base, web): counts = {label: sum(1 for r in found if r["label"] == label) for label in rated.LABEL_TITLES} return web.json_response({"results": found, "counts": counts}) + @table.post(base + "/clips") + async def _clips(req): + """Frame count and rate of video sources, by source key: what the pickers show beside a clip.""" + body = await req.json() + out, roots = {}, train.roots() + for raw in (body.get("srcs") or [])[:500] if isinstance(body, dict) else []: + src = store.clean_src(raw) + if src is None: + continue + path = store.resolve(src, roots, train.bin_path) + info = media_io.clip_info(path) if path and store.kind_of(path.name) == "video" else None + if info: + out[store.src_key(src)] = info + return web.json_response({"clips": out}) + # --- automatic tagging @table.get(base + "/taggers") async def _taggers(_req): diff --git a/modules/training/lora/media_io.py b/modules/training/lora/media_io.py index 01ba0937..18775396 100644 --- a/modules/training/lora/media_io.py +++ b/modules/training/lora/media_io.py @@ -58,6 +58,21 @@ def size_of(path, kind: str) -> tuple[int, int]: return int(s.codec_context.width), int(s.codec_context.height) +def clip_info(path) -> dict | None: + """{"frames", "fps"} of a video file at its own rate, or None when it cannot be read.""" + try: + import av + with av.open(str(path)) as box: + s = box.streams.video[0] + src = float(s.average_rate or s.guessed_rate or 24.0) + total = int(s.frames or 0) + if not total and s.duration and s.time_base: + total = int(float(s.duration * s.time_base) * src) + except Exception: # noqa: BLE001 -- an unreadable file is just left without a number + return None + return {"frames": total, "fps": round(src, 3)} if total else None + + def frames_at(path, fps: float) -> int: """How many frames a clip gives when read at `fps` (0 = its own rate).""" import av diff --git a/modules/training/lora/tests/test_sets_and_tags.py b/modules/training/lora/tests/test_sets_and_tags.py index 33abd3a6..56ff7442 100644 --- a/modules/training/lora/tests/test_sets_and_tags.py +++ b/modules/training/lora/tests/test_sets_and_tags.py @@ -135,3 +135,10 @@ def test_every_clip_is_read_for_as_many_frames_as_the_shortest_one_gives(tmp_pat train._fit_frames({"fps": 0, "frames": 120}, {**h3, "fit": lambda a: 0}, items) with pytest.raises(ValueError, match="s.mp4"): media_io.clip_frames(short, 80, 24, "start", 64, 48) + + +def test_clip_info_is_none_for_a_file_that_is_not_a_video(tmp_path): + bad = tmp_path / "x.mp4" + bad.write_bytes(b"not a video") + assert media_io.clip_info(bad) is None + assert media_io.clip_info(tmp_path / "missing.mp4") is None