feat: show a clip's frame count and rate in the training pickers and tags step

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MpnF7GwNGMbngU11wjym7P

app/ui/features/training/data.js

@@ -1,7 +1,7 @@
 // Step 1, Training data: pick or make a set, then put media in it from any mix of sources.
 // Skippable: a set that already has what you want goes straight on to tags or to training.
 import { composer as c } from "../../composer/composer.js";
-import { LABEL_WORDS, stillOf } from "./state.js";
+import { LABEL_WORDS, clipWords, srcKey, stillOf } from "./state.js";
 
 export const RATED = [
   { value: "liked", label: "Liked results" },
@@ -59,13 +59,16 @@ export function dataStep({ app, session, go, ui }) {
   const picked = ui.picked;
   const pickedNote = c.text.sm({ text: `${picked.size} picked` });
   const usable = ui.bin.filter((m) => m.kind === "image" || m.kind === "video");
-  const gallery = c.gallery.adaptive({ id: "training-bin", cols: 3, items: usable.map((m) => ({ id: m.id, label: m.name, thumb: `/funpack/api/media/${encodeURIComponent(m.id)}/thumb`, badge: m.kind })), selection: [...picked],
+  const binSrc = (m) => ({ kind: "bin", id: m.id });
+  const binItems = () => usable.map((m) => ({ id: m.id, label: m.name, thumb: `/funpack/api/media/${encodeURIComponent(m.id)}/thumb`, badge: m.kind === "video" ? clipWords(ui.clips[srcKey(binSrc(m))]) || m.kind : m.kind }));
+  const gallery = c.gallery.adaptive({ id: "training-bin", cols: 3, items: binItems(), selection: [...picked],
     empty: "Your Bin has no pictures or clips yet", onActivate: (item) => {
       if (picked.has(item.id)) picked.delete(item.id); else picked.add(item.id);
       gallery.setValue([...picked]);
       pickedNote.setText(`${picked.size} picked`);
       addBin.node.disabled = !picked.size;
     } });
+  ui.needClips(usable.filter((m) => m.kind === "video").map(binSrc), () => { gallery.setItems(binItems()); gallery.setValue([...picked]); });
   const addBin = c.button.sm({ label: "Add picked", tone: "primary", disabled: !picked.size || !open, onClick: async () => {
     const chosen = usable.filter((m) => picked.has(m.id));
     await added(chosen.map(binEntry), "the Bin");
@@ -91,7 +94,8 @@ export function dataStep({ app, session, go, ui }) {
   function showRated() {
     const all = shownRated();
     for (const k of [...ratedPicked]) if (!all.some((r) => keyOf(r) === k)) ratedPicked.delete(k);
-    ratedGallery.setItems(all.map((r) => ({ id: keyOf(r), label: r.media.filename, thumb: stillOf({ src: { kind: "output", ...r.media }, media_kind: r.media.filename.match(/\.(mp4|webm|mov|mkv|avi)$/i) ? "video" : "image" }), badge: r.label })));
+    ratedGallery.setItems(all.map((r) => ({ id: keyOf(r), label: r.media.filename, thumb: stillOf({ src: { kind: "output", ...r.media }, media_kind: r.media.filename.match(/\.(mp4|webm|mov|mkv|avi)$/i) ? "video" : "image" }), badge: [r.label, clipWords(ui.clips[keyOf(r)])].filter(Boolean).join(" · ") })));
+    ui.needClips(all.filter((r) => /\.(mp4|webm|mov|mkv|avi)$/i.test(r.media.filename)).map((r) => ({ kind: "output", ...r.media })), () => showRated());
     ratedGallery.setValue([...ratedPicked]);
     const n = all.length, p = ratedPicked.size;
     ratedNote.setText(n ? `${n} rated result${n > 1 ? "s" : ""} found in your projects. ${p ? `${p} picked: only those are added.` : "None picked: all are added."}` : "None found yet. Rate results in your projects and they appear here.");

app/ui/features/training/index.js

@@ -5,7 +5,7 @@ import { dataStep } from "./data.js";
 import { applyTagging, tagsStep } from "./tags.js";
 import { monitor, settingsOf, trainStep } from "./train.js";
 import { resultsStep } from "./results.js";
-import { makeSession } from "./state.js";
+import { makeSession, srcKey } from "./state.js";
 
 const TABS = [{ value: "data", label: "1  Data" }, { value: "tags", label: "2  Tags" }, { value: "train", label: "3  Train" }, { value: "results", label: "Results" }];
 const POLL = 1000;
@@ -26,12 +26,20 @@ export default {
     const api = app.api;
     const session = makeSession(api);
     const ui = {
-      tab: "data", picked: new Set(), ratedPicked: new Set(), bin: [], rated: { results: [], counts: {} }, ratedKind: "liked", include: ["main"], selected: new Set(), filter: "all", tag: "", limit: 48,
+      tab: "data", clips: {}, picked: new Set(), ratedPicked: new Set(), bin: [], rated: { results: [], counts: {} }, ratedKind: "liked", include: ["main"], selected: new Set(), filter: "all", tag: "", limit: 48,
       tagger: { id: "wd_vit_v3", list: [], job: null, general: 0.35, limit: 0, characters: false, mode: "add", scope: "untagged" },
       advanced: false, runs: [], chosenRun: "", record: null,
       run: { shown: false, active: false, state: "idle", loss: [], smooth: [], step: 0, total: 0, elapsed: 0, message: "", error: "", lora: "", before: "" },
       monitorBox: null, redraw: () => draw(), pollTaggers: () => pollTaggers(true), loadRecord: (id) => loadRecord(id), loadRuns: () => loadRuns(),
       start: () => start(), stop: () => stop(),
+      /** Ask for the frame counts of clips not known yet; `then` runs when new numbers arrived. */
+      async needClips(srcs, then) {
+        const fresh = srcs.filter((x) => !(srcKey(x) in ui.clips));
+        if (!fresh.length) return;
+        fresh.forEach((x) => { ui.clips[srcKey(x)] = null; });
+        try { Object.assign(ui.clips, (await api.trainClips(fresh)).clips || {}); } catch { fresh.forEach((x) => { delete ui.clips[srcKey(x)]; }); return; }
+        if (then) then();
+      },
     };
     let win = null, body = null, timers = [], offSession = null, tagPoll = null, runPoll = null;
 

app/ui/features/training/state.js

@@ -15,6 +15,11 @@ export function stillOf(item) {
   return item.media_kind === "video" ? `/funpack/api/m/render/poster?${q}` : `/view?${q}`;
 }
 
+/** The key a source has on the server (store.src_key). */
+export const srcKey = (src) => (src.kind === "bin" ? `bin:${src.id}` : `${src.type || "output"}:${src.subfolder || ""}/${src.filename}`);
+/** "124 frames, 24 fps" for a clip whose numbers are known, else "". */
+export const clipWords = (info) => (info ? `${info.frames} f · ${info.fps} fps` : "");
+
 export function makeSession(api) {
   const listeners = new Set();
   const s = { name: "", doc: null, sets: [], error: "", saving: false, state: null };

app/ui/features/training/tags.js

@@ -1,6 +1,6 @@
 // Step 2, Tags: every item with its caption beside its picture, bulk edits, a tag cloud, and automatic tagging.
 import { composer as c } from "../../composer/composer.js";
-import { LABEL_TONES, LABEL_WORDS, stillOf } from "./state.js";
+import { LABEL_TONES, LABEL_WORDS, clipWords, srcKey, stillOf } from "./state.js";
 import { addTags, bulk, frequency, joinTags, splitTags } from "./captions.js";
 
 const PAGE = 48;
@@ -28,10 +28,11 @@ export function tagsStep({ app, session, go, ui, peek }) {
 
   const cards = list.slice(0, ui.limit).map((item) => {
     const card = c.captioned.cell({ title: item.name || item.src.filename || item.src.id, src: stillOf(item), kind: item.media_kind, caption: item.caption,
-      badge: LABEL_WORDS[item.label], tone: LABEL_TONES[item.label], selected: sel.has(item.id),
+      badge: [LABEL_WORDS[item.label], clipWords(ui.clips[srcKey(item.src)])].filter(Boolean).join(" · "), tone: LABEL_TONES[item.label], selected: sel.has(item.id),
       onChange: (text) => session.edit(() => { item.caption = text; }),
       onSelect: (on) => { if (on) sel.add(item.id); else sel.delete(item.id); sync(); },
       onOpen: () => peek(item) });
+    if (item.media_kind === "video") ui.needClips([item.src], () => card.setBadge([LABEL_WORDS[item.label], clipWords(ui.clips[srcKey(item.src)])].filter(Boolean).join(" · ")));
     return card;
   });
   const grid = cards.length ? c.grid.cards({ children: cards }) : c.hint.default({ text: ui.filter === "untagged" ? "Everything has tags." : "Nothing to show." });

app/ui/features/training/training.test.js

@@ -21,6 +21,7 @@ function fake({ enabled = true } = {}) {
     trainSets: async () => ({ sets: [{ name: "cats", count: 2 }] }),
     trainSet: async () => structuredClone(doc),
     trainSaveSet: async (name, d) => { calls.push(["save", name, d]); return d; },
+    trainClips: async () => ({ clips: { "output:/v.mp4": { frames: 124, fps: 24 } } }),
     trainAdd: async () => ({ set: doc, added: 0 }),
     trainRated: async () => ({ results: [], counts: { liked: 3 } }),
     trainTaggers: async () => ({ taggers: [{ id: "wd_vit_v3", title: "WD ViT v3", state: "absent", hint: "", done: 0, total: 0 }] }),
@@ -273,3 +274,14 @@ test("rated results are shown as pictures, and only the picked ones are added",
   assert.equal(added[0][0].src.filename, "y.png");
   off();
 });
+
+test("Tags step: a clip's frame count and rate show beside its label", async () => {
+  const f = fake();
+  const { button, off } = await open(f);
+  button.click();
+  await wait(60);
+  click("Next: tags").click();
+  await wait(100);
+  assert.ok(text().includes("124 f · 24 fps"));
+  off();
+});

app/ui/shell/api.js

@@ -87,6 +87,7 @@ export const api = {
   trainDeleteSet: (name) => call("DELETE", `${TRAIN}/sets/${encodeURIComponent(name)}`),
   trainAdd: (name, entries) => call("POST", `${TRAIN}/sets/${encodeURIComponent(name)}/add`, { entries }),
   trainBulk: (name, body) => call("POST", `${TRAIN}/sets/${encodeURIComponent(name)}/bulk`, body),
+  trainClips: (srcs) => call("POST", `${TRAIN}/clips`, { srcs }),
   trainRated: (include) => call("GET", `${TRAIN}/rated${include && include.length ? `?include=${encodeURIComponent(include.join(","))}` : ""}`),
   trainTaggers: () => call("GET", `${TRAIN}/taggers`),
   trainDownloadTagger: (tagger) => call("POST", `${TRAIN}/taggers/download`, { tagger }),

modules/training/lora/__init__.py

@@ -9,7 +9,7 @@ H3 trains picture and sound together; the repairs its latent and its rotary op n
 used when this module asks. UNTESTED on real weights or hardware: proven only on tiny stand-in models on CPU.
 """
 
-from . import captions, jobs, plan, rated, store, taggers, train
+from . import captions, jobs, media_io, plan, rated, store, taggers, train
 from .nodes import FunPackTrainLora
 
 ID = "training_lora"
@@ -117,6 +117,21 @@ def routes(table, base, web):
         counts = {label: sum(1 for r in found if r["label"] == label) for label in rated.LABEL_TITLES}
         return web.json_response({"results": found, "counts": counts})
 
+    @table.post(base + "/clips")
+    async def _clips(req):
+        """Frame count and rate of video sources, by source key: what the pickers show beside a clip."""
+        body = await req.json()
+        out, roots = {}, train.roots()
+        for raw in (body.get("srcs") or [])[:500] if isinstance(body, dict) else []:
+            src = store.clean_src(raw)
+            if src is None:
+                continue
+            path = store.resolve(src, roots, train.bin_path)
+            info = media_io.clip_info(path) if path and store.kind_of(path.name) == "video" else None
+            if info:
+                out[store.src_key(src)] = info
+        return web.json_response({"clips": out})
+
     # --- automatic tagging
     @table.get(base + "/taggers")
     async def _taggers(_req):

modules/training/lora/media_io.py

@@ -58,6 +58,21 @@ def size_of(path, kind: str) -> tuple[int, int]:
         return int(s.codec_context.width), int(s.codec_context.height)
 
 
+def clip_info(path) -> dict | None:
+    """{"frames", "fps"} of a video file at its own rate, or None when it cannot be read."""
+    try:
+        import av
+        with av.open(str(path)) as box:
+            s = box.streams.video[0]
+            src = float(s.average_rate or s.guessed_rate or 24.0)
+            total = int(s.frames or 0)
+            if not total and s.duration and s.time_base:
+                total = int(float(s.duration * s.time_base) * src)
+    except Exception:                                     # noqa: BLE001 -- an unreadable file is just left without a number
+        return None
+    return {"frames": total, "fps": round(src, 3)} if total else None
+
+
 def frames_at(path, fps: float) -> int:
     """How many frames a clip gives when read at `fps` (0 = its own rate)."""
     import av

modules/training/lora/tests/test_sets_and_tags.py

@@ -135,3 +135,10 @@ def test_every_clip_is_read_for_as_many_frames_as_the_shortest_one_gives(tmp_pat
         train._fit_frames({"fps": 0, "frames": 120}, {**h3, "fit": lambda a: 0}, items)
     with pytest.raises(ValueError, match="s.mp4"):
         media_io.clip_frames(short, 80, 24, "start", 64, 48)
+
+
+def test_clip_info_is_none_for_a_file_that_is_not_a_video(tmp_path):
+    bad = tmp_path / "x.mp4"
+    bad.write_bytes(b"not a video")
+    assert media_io.clip_info(bad) is None
+    assert media_io.clip_info(tmp_path / "missing.mp4") is None