From b6841cbbd72d26cde9ce36674fce20a0630b05a7 Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:07:08 +0300 Subject: [PATCH] fix: the detail row now says what the A/B actually was MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every row read "no repeat → no repeat" because the label only ever tracked one knob — the block-repeat spec — so a comparison where anything else changed, or nothing did, described itself as unchanged and told you nothing about what it compared. sample() now snapshots its scalar arguments (locals() at the top IS the argument list, so no name list to go stale), and each row keeps the DIFF against the previous run. The row titles itself: h3_block_repeat: — → 40 sharper — +204% detail, on existing edges nothing changed stable — two identical runs match, so a real A/B can be trusted seed 111 → 222 different seed — not comparable The "nothing changed" case is the valuable one and was previously unreachable: two identical runs measure how much generation varies on its own. If that comes back unstable, no A/B on the setup means anything, and the row says so instead of letting every later comparison quietly inherit the noise. --- detail_probe.py | 19 +++++++++-- movie_editor/frontend/settings_window.js | 30 ++++++++++++++++- samplers.py | 10 +++++- tests/test_detail_probe.py | 41 ++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 5 deletions(-) diff --git a/detail_probe.py b/detail_probe.py index 8bc47ed0..2ca66118 100644 --- a/detail_probe.py +++ b/detail_probe.py @@ -145,7 +145,7 @@ def compare(before, after): } -def record(refinement_key, video_latent, label="", seed=None): +def record(refinement_key, video_latent, label="", seed=None, settings=None): """Score this run against the previous one on the same key, then become the new previous. -> the comparison dict (with "label_before"/"label_after"), or None when there was nothing to compare against yet. @@ -155,7 +155,14 @@ def record(refinement_key, video_latent, label="", seed=None): two different generations, so this is not an A/B at all" (different seed -- R2V pins the subject, not the sample) or "same seed, and the change moved the shot rather than its detail", which is a real result. Nothing in the three numbers can separate those; the - seed can, so it is stored rather than guessed at.""" + seed can, so it is stored rather than guessed at. + + `settings` is this run's scalar widget values. The row keeps the DIFF against the + previous run's, so it can say what the A/B actually was rather than trusting a label to + have named the right variable. An empty diff is not a failure: two identical runs measure + the instrument's own noise floor, which is what makes every other row interpretable. + (`seed` stays an explicit argument as well -- it is the one field the readout branches on, + so it should not depend on the settings capture having worked.)""" if not refinement_key or not isinstance(video_latent, torch.Tensor): return None try: @@ -171,6 +178,11 @@ def record(refinement_key, video_latent, label="", seed=None): if row is not None: row["label_before"] = str(prev.get("label") or "") row["label_after"] = str(label or "") + prev_set = prev.get("settings") or {} + cur_set = settings or {} + keys = set(prev_set) | set(cur_set) + row["changed"] = {k: [prev_set.get(k), cur_set.get(k)] for k in sorted(keys) + if prev_set.get(k) != cur_set.get(k)} row["seed_before"] = prev.get("seed") row["seed_after"] = seed row["same_seed"] = (seed is not None and prev.get("seed") is not None @@ -178,7 +190,8 @@ def record(refinement_key, video_latent, label="", seed=None): row["stamp"] = time.strftime("%Y-%m-%d_%H-%M-%S") data["rows"] = (data["rows"] + [row])[-MAX_ROWS:] data["previous"] = {"latent": lat, "label": str(label or ""), - "seed": None if seed is None else int(seed)} + "seed": None if seed is None else int(seed), + "settings": dict(settings or {})} _save(refinement_key, data) return row except Exception as e: # noqa: BLE001 diff --git a/movie_editor/frontend/settings_window.js b/movie_editor/frontend/settings_window.js index e262fa24..df5e2dc7 100644 --- a/movie_editor/frontend/settings_window.js +++ b/movie_editor/frontend/settings_window.js @@ -412,6 +412,16 @@ // so the seed decides. Different seeds = two different generations and the comparison // never was an A/B (a reference pins the subject, not the sample). Same seed = the // change moved the shot rather than its detail, which is a RESULT, not a bad setup. + const onlySeed = r.changed && !Object.keys(r.changed).filter((k) => k !== "seed").length; + if (onlySeed && r.same_seed) { + // Nothing differed at all: this is the instrument's noise floor for this setup. + return r.structure > 0.98 && Math.abs(r.detail - 1) < 0.02 + ? `stable — two identical runs match (structure ${r.structure.toFixed(3)}), so a real ` + + "A/B can be trusted" + : `two identical runs already differ this much (structure ${r.structure.toFixed(2)}, ` + + `detail x${r.detail.toFixed(2)}) — generation is not reproducible here, so no A/B ` + + "on this setup means anything"; + } if (r.structure < 0.85) { return r.same_seed ? `the shot itself moved (structure ${r.structure.toFixed(2)}) — this changed the ` @@ -428,6 +438,24 @@ : `+${gain.toFixed(0)}% detail but spread evenly — reads as grain, not sharpening`) + caveat; } + // What this pair actually was. The probe cannot know which knob you consider the + // variable, so it diffs every scalar setting between the two runs and names the result. + // No difference is not a wasted row: two identical runs measure how much the generation + // varies on its own, and without that number none of the others mean anything. + function dpChangeSummary(r) { + const ch = r.changed; + if (!ch) return `${r.label_before || "?"} → ${r.label_after || "?"}`; // pre-diff rows + const keys = Object.keys(ch).filter((k) => k !== "seed"); + if (!keys.length) { + return r.same_seed ? "nothing changed — repeatability check" + : "only the seed changed — a different sample"; + } + return keys.slice(0, 3).map((k) => { + const show = (v) => (v === "" || v == null ? "—" : String(v)); + return `${k}: ${show(ch[k][0])} → ${show(ch[k][1])}`; + }).join(" · ") + (keys.length > 3 ? ` · +${keys.length - 3} more` : ""); + } + function detailProbeRows(key) { const box = el("div", "sw-stack"); const paint = () => { @@ -456,7 +484,7 @@ // Newest first: the run you just made is the one you are asking about. dpRows.slice().reverse().forEach((r) => { tbl.append(infoRow( - `${r.label_before || "?"} → ${r.label_after || "?"}`, + dpChangeSummary(r), dpVerdict(r), r.same_seed !== false && r.structure >= 0.85 && r.detail > 1.02 && r.edge_aligned > 0.35)); diff --git a/samplers.py b/samplers.py index c4acad73..50b912f9 100644 --- a/samplers.py +++ b/samplers.py @@ -7258,6 +7258,14 @@ class FunPackLTXAVSceneChainSampler: # New generation: per-run log suppression starts over, so a failure that also # happened last run is reported again rather than deduped away forever. _log.begin_run() + # Every scalar widget value this run was called with, captured here because at this + # point locals() IS the argument list -- nothing else has been assigned yet. The + # detail probe diffs two runs' copies of this to say what actually CHANGED between + # them, instead of relying on a hand-picked label that only ever tracked one knob. + # Scalars only: models, tensors and conditioning are not settings, and a curated + # name list would go stale the first time a widget is added. + _run_settings = {k: v for k, v in locals().items() + if isinstance(v, (bool, int, float, str)) and k != "self"} self._is_h3 = self._set_stream_axes(model) if self._is_h3: # H3's rating-driven mechanisms are h3_phrase_emphasis (Studio), h3_repr_steering, @@ -8668,7 +8676,7 @@ class FunPackLTXAVSceneChainSampler: _dp.record(refinement_key_input, _snap, label=(f"repeat {h3_block_repeat}x{int(h3_block_repeat_times) + 1}" if self._parse_block_spec(h3_block_repeat) else "no repeat"), - seed=seed) + seed=seed, settings=_run_settings) # Trajectory probe: same run/rating pairing as the two above — the rating that scores # THIS run appends these per-bucket descriptors to the measurement log. diff --git a/tests/test_detail_probe.py b/tests/test_detail_probe.py index eaa9b027..8d93a3cf 100644 --- a/tests/test_detail_probe.py +++ b/tests/test_detail_probe.py @@ -297,3 +297,44 @@ def test_the_seed_rides_along_to_the_next_pairing(): dp.record("k", _edged(), label="b", seed=7) row = dp.record("k", _edged(), label="c", seed=9) assert row["seed_before"] == 7 and row["seed_after"] == 9 and row["same_seed"] is False + + +# --- what the A/B actually WAS ---------------------------------------------------- +# +# The probe cannot know which knob the user considers the variable, and a hand-picked label +# only ever tracked one of them (every row read "no repeat -> no repeat" while the real +# difference was elsewhere, or nowhere). So it diffs the run's scalar settings instead and +# the row names the difference itself. + +def test_the_row_names_what_changed_between_the_two_runs(): + dp.record("k", _edged(), seed=1, settings={"seed": 1, "h3_block_repeat": "", "cfg": 1.0}) + row = dp.record("k", _edged(), seed=1, settings={"seed": 1, "h3_block_repeat": "40", "cfg": 1.0}) + assert row["changed"] == {"h3_block_repeat": ["", "40"]} + + +def test_identical_settings_report_no_change_which_is_the_noise_floor(): + """Two identical runs are not a wasted comparison -- they measure how much the generation + varies on its own, and every other row is uninterpretable without that number.""" + cfg = {"seed": 5, "h3_block_repeat": ""} + dp.record("k", _edged(), seed=5, settings=cfg) + row = dp.record("k", _edged(), seed=5, settings=dict(cfg)) + assert row["changed"] == {} + assert row["same_seed"] is True + + +def test_a_new_or_removed_setting_counts_as_changed(): + dp.record("k", _edged(), settings={"a": 1}) + row = dp.record("k", _edged(), settings={"a": 1, "b": 2}) + assert row["changed"] == {"b": [None, 2]} + + +def test_several_changes_are_all_reported(): + dp.record("k", _edged(), settings={"a": 1, "b": 1, "c": 1}) + row = dp.record("k", _edged(), settings={"a": 2, "b": 1, "c": 3}) + assert row["changed"] == {"a": [1, 2], "c": [1, 3]} + + +def test_no_settings_at_all_is_an_empty_diff_not_a_crash(): + dp.record("k", _edged()) + row = dp.record("k", _edged()) + assert row["changed"] == {}