feat: wire per-scene reference-image picking into the real generate flow

Fills in the placeholder left in _buildQueueInputs: v5's real per-scene fields (core/projects.py's Scene.source_image/.references) already round-trip through api.js's getProject/saveProject today -- it's a raw whole-object PUT/GET, and the server's own Scene.from_dict()/asdict() extract and return them regardless of what else v4's project shape carries. There was just no UI to set them and nothing reading them at generate time. Added: - store.js: setSourceImage/setReferences, thin patchScene wrappers (same pattern as the existing setSourceTrim). - inspector.js: a "Reference media" section on the scene panel, right after v4's own i2v anchor/Source field, using the same window.MediaPicker thumbnail-grid component already used throughout this file for image picks -- a single Resolution-source picker plus an add/remove list for References. - generate_bridge.js: exposes reference_wiring.js's wireReferences() (untouched, already tested, no DOM dependency). - store.js's _buildQueueInputs: sets assets.source_image from the run's first/anchor scene, and calls wireReferences() for its references -- same "first scene stands for the whole run" convention its prompt-text branch already uses for a multi-scene (whole-timeline) run. An unwired reference (no matching slot in the current pipeline) surfaces as a real notice in the timeline header, not a silent drop. Live-verified against the dev server end to end: picked a real reference image from the Media bin, watched it persist server-side (GET /funpack/api/projects/:id showed it in the saved file), then clicked Generate and confirmed the pipeline check ran with it -- the current default pipeline has no reference slot, and the app correctly said so ("1 reference could not be wired into this pipeline and will not be used.") rather than silently dropping it. 691/692 JS tests and 548/548 Python tests pass (the one JS failure is the pre-existing, unrelated projects.test.js import-race flake). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-15 14:46:57 +03:00
parent 68df7ba866
commit 1abf136e03
3 changed files with 104 additions and 6 deletions
@@ -13,6 +13,7 @@ import { createRun, viewUrl, DONE, FAILED, CANCELLED } from "../shell/run.js";
 import { clientId, connect, queuedFor, finishedFor } from "../shell/client.js";
 import { wire, waitForTerminal } from "../shell/session.js";
 import { check } from "../shell/pipeline.js";
+import { wireReferences } from "../shell/reference_wiring.js";

 const PS = window.PipelineState;
 const id = clientId();
@@ -112,6 +113,7 @@ window.GenerateBridge = {
  waitForTerminal: () => waitForTerminal(run),
  ready: session.ready,
  slotForRole,
+  wireReferences,
  /**
   * Queue one run. `inputs` is {slotId: {inputName: value}}. `sceneIds`, when
   * the run covers more than one scene (no chain sampler to split it into
@@ -287,6 +287,55 @@
    });
  }

+  // v5's own per-scene reference fields (core/projects.py's Scene.source_image/
+  // .references) -- a different thing from v4's i2v anchor/guides above, which
+  // feed the sampler's own conditioning. These are read at generate time by
+  // whatever the pipeline's own "assets.source_image"/"assets.reference_N"
+  // roles ask for (see store.js's _buildQueueInputs / app/shell/reference_wiring.js),
+  // which today means H3's reference-to-video slot. Shared with the gen-unit's
+  // root scene, same as the i2v anchor above ("shared with the root clip").
+  function renderReferenceMedia(st, scene, parent) {
+    parent = parent || body;
+    if (!window.MediaPicker) return;
+    const tag = el("div", "insp-tag"); tag.textContent = "Reference media (v5 pipeline)"; parent.append(tag);
+
+    const srcPick = window.MediaPicker.create({
+      value: scene.source_image || null,
+      mediaBin: st.mediaBin,
+      filter: (m) => m.kind === "image",
+      noneLabel: "— choose resolution source —",
+      onChange: (mediaId) => S.setSourceImage(scene.id, mediaId),
+    });
+    parent.append(field("Resolution source", srcPick));
+    parent.append(el("div", "insp-hint",
+      "Sets this scene's aspect ratio for generation — the project's own Width/Height "
+      + "set the actual resolution, this image's pixels are not used."));
+
+    const refs = scene.references || [];
+    if (refs.length) {
+      const list = el("div", "insp-block");
+      refs.forEach((mediaId, i) => {
+        const row = el("div", "fields-row ref-row");
+        const asset = (st.mediaBin || []).find((m) => m.id === mediaId);
+        row.append(el("span", "insp-hint", asset ? asset.name : `${mediaId} (missing)`));
+        const rm = el("button", "btn ghost tiny danger", "✕");
+        rm.title = "Remove this reference";
+        rm.onclick = () => S.setReferences(scene.id, refs.filter((_, j) => j !== i));
+        row.append(rm);
+        list.append(row);
+      });
+      parent.append(list);
+    }
+    const addPick = window.MediaPicker.create({
+      value: null,
+      mediaBin: st.mediaBin,
+      filter: (m) => m.kind === "image",
+      noneLabel: "+ Add reference",
+      onChange: (mediaId) => { if (mediaId) S.setReferences(scene.id, [...refs, mediaId]); },
+    });
+    parent.append(field("References", addPick));
+  }
+
  function renderScene(st, scene) {
    const root = S.genUnitRoot(S.genUnitId(scene)) || scene;
    const unitScenes = (st.project.scenes || []).filter((s) => S.genUnitId(s) === S.genUnitId(scene))
@@ -345,6 +394,7 @@
      if ((root.source?.type) === "generated_frame") renderGeneratedFrameSource(st, root);
      if ((root.source?.type) === "v2v") renderVideoSource(st, root, body, "V2V source video");
    }
+    renderReferenceMedia(st, root, body);

    const effFrames = effOf(scene, "frames"), effFps = effOf(scene, "fps") || 1;
    const planDur = effFrames / effFps;
@@ -2870,6 +2870,22 @@
    if (Object.keys(p).length) patchScene(id, p);
  }

+  // A media library id, picked to size this scene's generation -- v5's real
+  // core/projects.py Scene field, not a v4 concept (v4's "drop a picture on
+  // the timeline" fed a resolution node the same way, through source.media_ref
+  // instead). api.js round-trips this already, whole-object-passthrough, so
+  // nothing here needs its own adapter -- it just needed a setter and a place
+  // to show it (see inspector.js's "Reference media" section).
+  function setSourceImage(id, mediaId) { patchScene(id, { source_image: mediaId || null }); }
+
+  // Media library ids, in the order they were added -- reference images for
+  // whatever the pipeline's own reference input wants them for (see
+  // app/shell/reference_wiring.js's wireReferences(), which _buildQueueInputs
+  // in the generate path below calls with this list at queue time).
+  function setReferences(id, mediaIds) {
+    patchScene(id, { references: Array.isArray(mediaIds) ? mediaIds.filter(Boolean) : [] });
+  }
+
  const ENGINE_PRESETS = {
    draft: {
      label: "Fast draft",
@@ -3536,11 +3552,16 @@
  // project.video's width/height, wherever the pipeline asks for them, by
  // role.input name -- same convention app/boot.js's syncVideo() uses.
  //
-  // Reference-image wiring (assets.source_image/reference_N, app/boot.js's
-  // wireReferences()) is NOT built here yet: v4's legacy project/scene shape
-  // doesn't round-trip source_image/references through api.js's
-  // getProject/saveProject adapter today, so there is nothing yet to wire —
-  // a placeholder left for the next pass, not a silent gap.
+  // Reference-image wiring (assets.source_image/reference_N -- H3's own
+  // reference-to-video slot today) reads the FIRST scene in this run's own
+  // source_image/references (core/projects.py's real per-scene fields --
+  // api.js's getProject/saveProject already round-trip them whole-object-
+  // passthrough, nothing v4-specific to adapt). One run can cover several
+  // scenes at once (H3's whole-timeline-in-one-call shape, no chain sampler
+  // to split it up), and there is no per-scene reference slot to give each
+  // one its own set -- the run's first/anchor scene's picks stand for the
+  // whole run, the same way its text already does for a single-scene run
+  // below.
  async function _buildQueueInputs(targetSceneIds) {
    const raw = {};
    const GB = window.GenerateBridge;
@@ -3555,6 +3576,31 @@
      }
    }

+    const primary = scene(targetSceneIds[0]);
+    if (primary) {
+      const source = GB.slotForRole("assets.source_image");
+      const reservedSlotIds = [];
+      if (source) {
+        raw[source.slot.id] = { ...(raw[source.slot.id] || {}), media_id: primary.source_image || "" };
+        reservedSlotIds.push(source.slot.id);
+      }
+      const { overrides, unwired } = GB.wireReferences(
+        primary.references || [], window.PipelineState.slots() || [], reservedSlotIds);
+      for (const [slotId, fields] of Object.entries(overrides)) {
+        raw[slotId] = { ...(raw[slotId] || {}), ...fields };
+      }
+      // Two different causes look the same from here -- fewer reference
+      // slots than references picked, or a malformed preset pointing two
+      // references at the same destination -- but both are true either way:
+      // the reference did not make it into this run (see wireReferences's
+      // own comment for why the wording doesn't try to guess which).
+      if (unwired > 0) {
+        state.notice = unwired === 1
+          ? "1 reference could not be wired into this pipeline and will not be used."
+          : `${unwired} references could not be wired into this pipeline and will not be used.`;
+      }
+    }
+
    const found = GB.slotForRole("generation.prompt");
    if (!found) return raw;
    const text = targetSceneIds.length > 1
@@ -4782,7 +4828,7 @@
    bringOverlayToFront, sendOverlayToBack, bringOverlayForward, sendOverlayBackward,
    addImageOverlay, addTextOverlay, updateOverlayTrack, removeOverlayTrack, removeSelectedOverlay,
    isOverlayAudioTrack, isSeparatedAudioTrack,
-    resizeScene, setSceneGapAfter, splitScene, autoMontage, hasPlayableRender, snapFrames, snapFramesFloor, snapFramesCeil, sceneEffFrames, sceneEffFps, scenesOverridingProjectFrames, useProjectFramesEverywhere, setSourceTrim, trimSceneLeft, slipScene,
+    resizeScene, setSceneGapAfter, splitScene, autoMontage, hasPlayableRender, snapFrames, snapFramesFloor, snapFramesCeil, sceneEffFrames, sceneEffFps, scenesOverridingProjectFrames, useProjectFramesEverywhere, setSourceTrim, setSourceImage, setReferences, trimSceneLeft, slipScene,
    applyEnginePreset, ENGINE_PRESETS, undo, redo,
    refreshPreview, syncFromPreview, applyGlobalPromptQuiet, scheduleGlobalPromptApply, globalPromptApplyPending, buildGlobalPromptFromTimeline, syncGlobalPromptFromTimeline, generate: _clockedGenerate, generateMontage: _clockedMontage, generateSelected: _clockedSelected, genElapsed, selectedSceneCount, renderFinal, exportSelected, saveSelectedToMediaBin, saveClipToMediaBin, clipSaveableToMediaBin,
    runComboSweep, parseComboSweepConfig, rateComboResult, discardComboResult, interrupt, loadModels, loadImageTargets, setModelInput, setModelBypass, setModelLink, clearNotice,