diff --git a/movie_editor/frontend/composer.js b/movie_editor/frontend/composer.js index 189f8ef0..ec48cc59 100644 --- a/movie_editor/frontend/composer.js +++ b/movie_editor/frontend/composer.js @@ -217,8 +217,40 @@ // Variables ($name) — collapsible, same toggle pattern as Settings ▸ Built-in pipeline. wrap.append(composeVariables()); + // Segmented detailing targets — only while the Chain Sampler feature is enabled + // (Engine ▸ Chain Sampler ▸ Experimental), so prompt-craft and "what gets detailed" + // live side by side without adding a permanently-visible field. + const detailBlock = composeDetailTargets(st); + if (detailBlock) wrap.append(detailBlock); return wrap; } + + // ── segmented detailing targets (shown only when the feature is on) ───────────── + function composeDetailTargets(st) { + const p = st.project; + if (!p) return null; + const PC = window.PipelineCaps; + if (PC && PC.usesChainSampler && !PC.usesChainSampler(st)) return null; + const si = p.sampler_inputs || {}; + if (!si.segmented_detailing) return null; + const box = el("div", "compose-detail"); + const row = el("div", "compose-head"); + row.append(el("div", "lib-form-title", "Segmented detailing")); + box.append(row); + const input = el("input", "lib-in"); + input.type = "text"; + input.placeholder = "hands, feet"; + input.value = si.detail_targets != null ? String(si.detail_targets) : "hands"; + // Quiet while typing (no repaint under the caret), commit on blur/Enter — + // same contract as the Engine Settings text knob, same store key, so the + // two fields can never disagree. + input.oninput = () => S.setSamplerInput("detail_targets", input.value); + input.onchange = () => S.setSamplerInputNow("detail_targets", input.value); + box.append(input); + box.append(el("div", "insp-hint", + "Comma-separated regions to detail (CLIPSeg text queries — plain words work best). Strength and the upsampler model live in Engine ▸ Chain Sampler ▸ Experimental.")); + return box; + } let composeTextarea = null; let varsOpen = false; // Variables panel expanded? let varHintEl = null; // undeclared / cycle hint line under the prompt @@ -834,6 +866,12 @@ const fp = JSON.stringify({ s: st.shortcuts?.length, t: st.transitions?.length, pid: st.project?.id, c: (st.shortcutCategories || []).reduce((n, x) => n + 1 + (x.sub_categories?.length || 0), 0), + // Segmented detailing block visibility: toggling it in Engine Settings must + // repaint an open Composer or the targets field appears/disappears only on the + // next unrelated repaint ("store right, UI stale"). The targets TEXT is not + // fingerprinted — typing goes through quiet patches and the field itself is + // the source of those edits. + sd: !!(st.project?.sampler_inputs?.segmented_detailing), }); if (fp === lastFp) return; lastFp = fp; diff --git a/movie_editor/frontend/engine_settings.js b/movie_editor/frontend/engine_settings.js index a2c81538..29b39564 100644 --- a/movie_editor/frontend/engine_settings.js +++ b/movie_editor/frontend/engine_settings.js @@ -196,6 +196,12 @@ hint: "EXPERIMENTAL speed: the near-pure-noise early steps carry almost no signal, so the transformer output barely changes across them. Computes it once at the top of the plateau and reuses it for the rest, skipping ~3-4 of 8 transformer passes. Deterministic given seed (safe in Batch Training) but an approximation — A/B it before trusting on finals. Much of wall-clock time is outside the sampler, so total speedup is smaller than the forward count suggests. UNVALIDATED LIVE." }, { name: "plateau_cache_threshold", label: "Plateau threshold (sigma)", kind: "float", default: 0.975, min: 0.5, max: 0.999, step: 0.005, dependsOn: "plateau_cache", hint: "Steps with sigma at or above this count as the reusable plateau. Higher = fewer steps cached (safer); lower = more cached (faster, more approximation). 0.975 catches the documented noise plateau while leaving structure formation fully computed." }, + { name: "segmented_detailing", label: "Segmented detailing (region refine)", kind: "bool", default: false, + hint: "EXPERIMENTAL ADetailer-for-video: after each scene renders, CLIPSeg finds the regions named below (hands, feet, …), cuts them out of the latent as a tube, refines them at 2× working resolution through Lightricks' latent upsampler (3 extra steps on the crop only), and pastes them back through a feathered silhouette. Final resolution never changes. Cost ≈ 4 × region area × 3 steps (hands ~+15%); regions over 35% of the frame are refused. Needs the detail upsampler model below. UNVALIDATED LIVE." }, + { name: "detail_targets", label: "Detail targets", kind: "text", default: "hands", dependsOn: "segmented_detailing", placeholder: "hands, feet", + hint: "Comma-separated regions to detail, in plain words. Each becomes a CLIPSeg text query — malformed anatomy still matches its name. Also editable from Composer ▸ Compose while detailing is on." }, + { name: "detail_strength", label: "Detail strength", kind: "float", default: 1.0, min: 0, max: 1, step: 0.05, dependsOn: "segmented_detailing", + hint: "Blend of the refined region at paste-back. 1.0 = full replacement inside the silhouette; 0 disables the pass." }, ]; const SAMPLER_KNOB_MAP = Object.fromEntries(SAMPLER_KNOBS.map((k) => [k.name, k])); @@ -271,6 +277,14 @@ ctrl = el("select"); ctrl.dataset.k = "si-" + k.name; (k.choices || []).forEach((c) => { const o = el("option", null, c); o.value = c; if (c === val) o.selected = true; ctrl.append(o); }); ctrl.onchange = () => S.setSamplerInputNow(k.name, ctrl.value); + } else if (k.kind === "text") { + ctrl = el("input"); ctrl.type = "text"; + ctrl.value = val != null ? String(val) : ""; + if (k.placeholder) ctrl.placeholder = k.placeholder; + ctrl.dataset.k = "si-" + k.name; + // Quiet while typing (no repaint under the caret), commit on blur/Enter. + ctrl.oninput = () => S.setSamplerInput(k.name, ctrl.value); + ctrl.onchange = () => S.setSamplerInputNow(k.name, ctrl.value); } else { ctrl = el("input"); ctrl.type = "number"; if (k.step != null) ctrl.step = String(k.step); @@ -300,7 +314,7 @@ chain_timing: ["frame_overlap", "transition_duration", "use_same_seed"], chain_guidance: ["cfg", "embed_guidance", "embed_guidance_source", "embed_guidance_strength", "score_slider", "score_slider_strength", "taste_nearest_prompt", "output_guidance", "output_guidance_strength", "dynashift", "dynashift_strength", "dynashift_threshold"], chain_decode: ["decode_noise_scale", "decode_timestep", "decode_tile_size"], - chain_experimental: ["plateau_cache", "plateau_cache_threshold", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"], + chain_experimental: ["plateau_cache", "plateau_cache_threshold", "segmented_detailing", "detail_targets", "detail_strength", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"], }; function countChainView(p, id) { @@ -576,6 +590,46 @@ renderKnobList(g, st, CHAIN_VIEW_KNOBS[id]); } + // ── Segmented detailing: the upsampler model list is live (models/latent_upscale_models + // on the server), so like the ArcFace projector below it can't be a static SAMPLER_KNOBS + // combo. The chain sampler node's own spec already carries the choices (with "None"). + let _detailUpsamplerChoices = null; + + async function ensureDetailUpsamplerChoices() { + if (_detailUpsamplerChoices) return _detailUpsamplerChoices; + try { + const spec = await API.nodeSpec("FunPackLTXAVSceneChainSampler"); + const w = (spec?.inputs || []).find((i) => i.name === "detail_upsampler"); + _detailUpsamplerChoices = (w && w.choices) || ["None"]; + } catch (_) { + _detailUpsamplerChoices = ["None"]; + } + render(); + return _detailUpsamplerChoices; + } + + function renderDetailUpsampler(pane, si) { + if (!si.segmented_detailing) return; + const g = group(pane, "Segmented detailing: upsampler model"); + if (!_detailUpsamplerChoices) { + g.append(hintEl("Loading upsampler list…")); + ensureDetailUpsamplerChoices(); + return; + } + const sel = el("select"); sel.dataset.k = "si-detail_upsampler"; + _detailUpsamplerChoices.forEach((c) => { + const o = el("option", null, c); o.value = c; + if (c === (si.detail_upsampler || "None")) o.selected = true; + sel.append(o); + }); + sel.onchange = () => S.setSamplerInputNow("detail_upsampler", sel.value); + g.append(field("Latent upsampler", sel, + "The LTX 2.3 spatial upsampler from models/latent_upscale_models (the official two-stage workflows use the same file). Detailing no-ops while this is None.")); + if ((si.detail_upsampler || "None") === "None" && _detailUpsamplerChoices.length <= 1) { + g.append(hintEl("No models found in models/latent_upscale_models on the server — download the LTX 2.3 latent upsampler there, then reopen this pane.")); + } + } + // ── Best-FaceID ArcFace projector: the one identity-transfer field that needs a // live (server-fetched) choice list, so it can't live in the static SAMPLER_KNOBS combo // shape. Reuses the same loras folder listing LoraLoaderModelOnly exposes. @@ -598,6 +652,7 @@ renderChainKnobsView(pane, st, "chain_experimental", "Experimental", "Research techniques — off by default; expect quality/overhead trade-offs."); const si = st.project.sampler_inputs || {}; + renderDetailUpsampler(pane, si); if (!si.identity_transfer_enabled) return; const g = group(pane, "Best-FaceID: ArcFace projector"); if (!_loraChoices) {