diff --git a/detail_probe.py b/detail_probe.py index 8bc47ed0..2ca66118 100644 --- a/detail_probe.py +++ b/detail_probe.py @@ -145,7 +145,7 @@ def compare(before, after): } -def record(refinement_key, video_latent, label="", seed=None): +def record(refinement_key, video_latent, label="", seed=None, settings=None): """Score this run against the previous one on the same key, then become the new previous. -> the comparison dict (with "label_before"/"label_after"), or None when there was nothing to compare against yet. @@ -155,7 +155,14 @@ def record(refinement_key, video_latent, label="", seed=None): two different generations, so this is not an A/B at all" (different seed -- R2V pins the subject, not the sample) or "same seed, and the change moved the shot rather than its detail", which is a real result. Nothing in the three numbers can separate those; the - seed can, so it is stored rather than guessed at.""" + seed can, so it is stored rather than guessed at. + + `settings` is this run's scalar widget values. The row keeps the DIFF against the + previous run's, so it can say what the A/B actually was rather than trusting a label to + have named the right variable. An empty diff is not a failure: two identical runs measure + the instrument's own noise floor, which is what makes every other row interpretable. + (`seed` stays an explicit argument as well -- it is the one field the readout branches on, + so it should not depend on the settings capture having worked.)""" if not refinement_key or not isinstance(video_latent, torch.Tensor): return None try: @@ -171,6 +178,11 @@ def record(refinement_key, video_latent, label="", seed=None): if row is not None: row["label_before"] = str(prev.get("label") or "") row["label_after"] = str(label or "") + prev_set = prev.get("settings") or {} + cur_set = settings or {} + keys = set(prev_set) | set(cur_set) + row["changed"] = {k: [prev_set.get(k), cur_set.get(k)] for k in sorted(keys) + if prev_set.get(k) != cur_set.get(k)} row["seed_before"] = prev.get("seed") row["seed_after"] = seed row["same_seed"] = (seed is not None and prev.get("seed") is not None @@ -178,7 +190,8 @@ def record(refinement_key, video_latent, label="", seed=None): row["stamp"] = time.strftime("%Y-%m-%d_%H-%M-%S") data["rows"] = (data["rows"] + [row])[-MAX_ROWS:] data["previous"] = {"latent": lat, "label": str(label or ""), - "seed": None if seed is None else int(seed)} + "seed": None if seed is None else int(seed), + "settings": dict(settings or {})} _save(refinement_key, data) return row except Exception as e: # noqa: BLE001 diff --git a/movie_editor/frontend/settings_window.js b/movie_editor/frontend/settings_window.js index e262fa24..df5e2dc7 100644 --- a/movie_editor/frontend/settings_window.js +++ b/movie_editor/frontend/settings_window.js @@ -412,6 +412,16 @@ // so the seed decides. Different seeds = two different generations and the comparison // never was an A/B (a reference pins the subject, not the sample). Same seed = the // change moved the shot rather than its detail, which is a RESULT, not a bad setup. + const onlySeed = r.changed && !Object.keys(r.changed).filter((k) => k !== "seed").length; + if (onlySeed && r.same_seed) { + // Nothing differed at all: this is the instrument's noise floor for this setup. + return r.structure > 0.98 && Math.abs(r.detail - 1) < 0.02 + ? `stable — two identical runs match (structure ${r.structure.toFixed(3)}), so a real ` + + "A/B can be trusted" + : `two identical runs already differ this much (structure ${r.structure.toFixed(2)}, ` + + `detail x${r.detail.toFixed(2)}) — generation is not reproducible here, so no A/B ` + + "on this setup means anything"; + } if (r.structure < 0.85) { return r.same_seed ? `the shot itself moved (structure ${r.structure.toFixed(2)}) — this changed the ` @@ -428,6 +438,24 @@ : `+${gain.toFixed(0)}% detail but spread evenly — reads as grain, not sharpening`) + caveat; } + // What this pair actually was. The probe cannot know which knob you consider the + // variable, so it diffs every scalar setting between the two runs and names the result. + // No difference is not a wasted row: two identical runs measure how much the generation + // varies on its own, and without that number none of the others mean anything. + function dpChangeSummary(r) { + const ch = r.changed; + if (!ch) return `${r.label_before || "?"} → ${r.label_after || "?"}`; // pre-diff rows + const keys = Object.keys(ch).filter((k) => k !== "seed"); + if (!keys.length) { + return r.same_seed ? "nothing changed — repeatability check" + : "only the seed changed — a different sample"; + } + return keys.slice(0, 3).map((k) => { + const show = (v) => (v === "" || v == null ? "—" : String(v)); + return `${k}: ${show(ch[k][0])} → ${show(ch[k][1])}`; + }).join(" · ") + (keys.length > 3 ? ` · +${keys.length - 3} more` : ""); + } + function detailProbeRows(key) { const box = el("div", "sw-stack"); const paint = () => { @@ -456,7 +484,7 @@ // Newest first: the run you just made is the one you are asking about. dpRows.slice().reverse().forEach((r) => { tbl.append(infoRow( - `${r.label_before || "?"} → ${r.label_after || "?"}`, + dpChangeSummary(r), dpVerdict(r), r.same_seed !== false && r.structure >= 0.85 && r.detail > 1.02 && r.edge_aligned > 0.35)); diff --git a/samplers.py b/samplers.py index c4acad73..50b912f9 100644 --- a/samplers.py +++ b/samplers.py @@ -7258,6 +7258,14 @@ class FunPackLTXAVSceneChainSampler: # New generation: per-run log suppression starts over, so a failure that also # happened last run is reported again rather than deduped away forever. _log.begin_run() + # Every scalar widget value this run was called with, captured here because at this + # point locals() IS the argument list -- nothing else has been assigned yet. The + # detail probe diffs two runs' copies of this to say what actually CHANGED between + # them, instead of relying on a hand-picked label that only ever tracked one knob. + # Scalars only: models, tensors and conditioning are not settings, and a curated + # name list would go stale the first time a widget is added. + _run_settings = {k: v for k, v in locals().items() + if isinstance(v, (bool, int, float, str)) and k != "self"} self._is_h3 = self._set_stream_axes(model) if self._is_h3: # H3's rating-driven mechanisms are h3_phrase_emphasis (Studio), h3_repr_steering, @@ -8668,7 +8676,7 @@ class FunPackLTXAVSceneChainSampler: _dp.record(refinement_key_input, _snap, label=(f"repeat {h3_block_repeat}x{int(h3_block_repeat_times) + 1}" if self._parse_block_spec(h3_block_repeat) else "no repeat"), - seed=seed) + seed=seed, settings=_run_settings) # Trajectory probe: same run/rating pairing as the two above — the rating that scores # THIS run appends these per-bucket descriptors to the measurement log. diff --git a/tests/test_detail_probe.py b/tests/test_detail_probe.py index eaa9b027..8d93a3cf 100644 --- a/tests/test_detail_probe.py +++ b/tests/test_detail_probe.py @@ -297,3 +297,44 @@ def test_the_seed_rides_along_to_the_next_pairing(): dp.record("k", _edged(), label="b", seed=7) row = dp.record("k", _edged(), label="c", seed=9) assert row["seed_before"] == 7 and row["seed_after"] == 9 and row["same_seed"] is False + + +# --- what the A/B actually WAS ---------------------------------------------------- +# +# The probe cannot know which knob the user considers the variable, and a hand-picked label +# only ever tracked one of them (every row read "no repeat -> no repeat" while the real +# difference was elsewhere, or nowhere). So it diffs the run's scalar settings instead and +# the row names the difference itself. + +def test_the_row_names_what_changed_between_the_two_runs(): + dp.record("k", _edged(), seed=1, settings={"seed": 1, "h3_block_repeat": "", "cfg": 1.0}) + row = dp.record("k", _edged(), seed=1, settings={"seed": 1, "h3_block_repeat": "40", "cfg": 1.0}) + assert row["changed"] == {"h3_block_repeat": ["", "40"]} + + +def test_identical_settings_report_no_change_which_is_the_noise_floor(): + """Two identical runs are not a wasted comparison -- they measure how much the generation + varies on its own, and every other row is uninterpretable without that number.""" + cfg = {"seed": 5, "h3_block_repeat": ""} + dp.record("k", _edged(), seed=5, settings=cfg) + row = dp.record("k", _edged(), seed=5, settings=dict(cfg)) + assert row["changed"] == {} + assert row["same_seed"] is True + + +def test_a_new_or_removed_setting_counts_as_changed(): + dp.record("k", _edged(), settings={"a": 1}) + row = dp.record("k", _edged(), settings={"a": 1, "b": 2}) + assert row["changed"] == {"b": [None, 2]} + + +def test_several_changes_are_all_reported(): + dp.record("k", _edged(), settings={"a": 1, "b": 1, "c": 1}) + row = dp.record("k", _edged(), settings={"a": 2, "b": 1, "c": 3}) + assert row["changed"] == {"a": [1, 2], "c": [1, 3]} + + +def test_no_settings_at_all_is_an_empty_diff_not_a_crash(): + dp.record("k", _edged()) + row = dp.record("k", _edged()) + assert row["changed"] == {}