diff --git a/block_influence.py b/block_influence.py index b57045c4..761531c6 100644 --- a/block_influence.py +++ b/block_influence.py @@ -120,6 +120,7 @@ def _load(refinement_key): data.setdefault("rows", []) data.setdefault("pending", None) data.setdefault("pending_novelty", None) + data.setdefault("pending_raw", None) return data @@ -131,7 +132,7 @@ def _save(refinement_key, data): os.replace(tmp, path) -def save_pending(refinement_key, profile, novelty=None): +def save_pending(refinement_key, profile, novelty=None, raw=None): """Store this run's per-block profile as the pending candidate the next rating scores. `profile` is {block_index: mean relative delta}; `novelty` is the optional companion {block_index: mean cosine with the PREVIOUS block's delta}. Overwritten every run, same @@ -143,6 +144,9 @@ def save_pending(refinement_key, profile, novelty=None): data["pending"] = {int(b): float(v) for b, v in profile.items()} data["pending_novelty"] = ({int(b): float(v) for b, v in novelty.items()} if novelty else None) + # Raw ||f_i|| alongside the ratio: see `share` in profile() for why the ratio alone is + # not comparable across depth. + data["pending_raw"] = ({int(b): float(v) for b, v in raw.items()} if raw else None) try: _save(refinement_key, data) except OSError as e: @@ -158,8 +162,10 @@ def commit(refinement_key, reward): data = _load(refinement_key) pending = data.get("pending") pending_novelty = data.get("pending_novelty") + pending_raw = data.get("pending_raw") data["pending"] = None data["pending_novelty"] = None + data["pending_raw"] = None if pending is None: try: _save(refinement_key, data) @@ -168,7 +174,7 @@ def commit(refinement_key, reward): return "no_pending" try: data["rows"].append({"profile": pending, "novelty": pending_novelty or {}, - "weight": float(reward)}) + "raw": pending_raw or {}, "weight": float(reward)}) except (TypeError, ValueError): try: _save(refinement_key, data) @@ -211,7 +217,12 @@ def profile(refinement_key): `mean_novelty` its average: ~1 means blocks mostly amplify their predecessor (little new per block), ~0 means each adds an orthogonal component nothing before it added, negative means blocks partly UNDO each other. Magnitude cannot tell those apart, which is why it - is recorded alongside rather than instead. `difference` is the rating-driven signal: blocks whose activity runs higher on + is recorded alongside rather than instead. + + `share` is each block's RAW movement ||f_i|| as a fraction of every block's, which is the + number to rank by. `overall` (the ratio ||f_i||/||x_i||) is not comparable across depth: + the residual stream grows as it goes, so early blocks score high because the denominator + is barely formed rather than because they move the picture more. `difference` is the rating-driven signal: blocks whose activity runs higher on runs you liked. `flatness` is the spread of `overall` as a fraction of its own mean (coefficient of variation): near 0 means every block moves the stream by the same amount and there is nothing to aim at; a large value means the depth profile is structured. @@ -225,7 +236,7 @@ def profile(refinement_key): if not rows or not blocks: return {"overall": {}, "liked": None, "disliked": None, "difference": None, "n_liked": 0, "n_disliked": 0, "flatness": None, - "novelty": {}, "mean_novelty": None} + "novelty": {}, "mean_novelty": None, "share": {}, "raw": {}} def _mean(subset): out = {} @@ -244,10 +255,22 @@ def profile(refinement_key): out[b] = sum(vals) / len(vals) return out + def _mean_raw(subset): + out = {} + for b in blocks: + vals = [float(r["raw"][b]) for r in subset + if isinstance(r.get("raw"), dict) and b in r["raw"]] + if vals: + out[b] = sum(vals) / len(vals) + return out + liked_rows = [r for r in rows if r["weight"] > 0] disliked_rows = [r for r in rows if r["weight"] < 0] overall = _mean(rows) novelty = _mean_novelty(rows) + raw = _mean_raw(rows) + total = sum(raw.values()) + share = {b: v / total for b, v in raw.items()} if total > 1e-12 else {} flatness = None if overall: vals = torch.tensor(list(overall.values()), dtype=torch.float32) @@ -261,5 +284,6 @@ def profile(refinement_key): return {"overall": overall, "liked": liked, "disliked": disliked, "difference": difference, "n_liked": len(liked_rows), "n_disliked": len(disliked_rows), "flatness": flatness, + "share": share, "raw": raw, "novelty": novelty, "mean_novelty": ( sum(novelty.values()) / len(novelty) if novelty else None)} diff --git a/movie_editor/frontend/settings_window.js b/movie_editor/frontend/settings_window.js index df5e2dc7..ce9ff6b5 100644 --- a/movie_editor/frontend/settings_window.js +++ b/movie_editor/frontend/settings_window.js @@ -540,15 +540,21 @@ const diff = (state && state.difference) || null; // Ranked by how much the block moves the picture, not by block number: the question is // which blocks do the work, and a 50-row list in index order buries that. - const entries = Object.entries(overall) - .map(([b, v]) => [Number(b), v]) - .sort((a, b) => b[1] - a[1]) + // Rank by SHARE, not by the ratio: ||f_i||/||x_i|| shrinks with depth by construction + // (the residual stream grows as it goes), so ranking on it just sorts the earliest + // blocks to the top regardless of how much they actually move the picture. + const share = (state && state.share) || {}; + const rankOn = Object.keys(share).length ? share : overall; + const entries = Object.keys(rankOn) + .map((b) => [Number(b), overall[b], share[b]]) + .sort((a, b) => (b[2] ?? b[1]) - (a[2] ?? a[1])) .slice(0, 12); const tbl = el("div", "sw-rows"); - entries.forEach(([block, v]) => { + entries.forEach(([block, v, sh]) => { const d = diff ? diff[String(block)] : null; const nov = state.novelty ? state.novelty[String(block)] : null; - let value = v.toPrecision(3); + let value = sh != null ? `${(sh * 100).toFixed(1)}% of the movement` + : (v == null ? "—" : v.toPrecision(3)); if (nov != null) value += ` · new ${nov >= 0 ? "+" : ""}${nov.toFixed(2)}`; if (d != null) value += ` · liked ${d >= 0 ? "+" : ""}${d.toFixed(4)}`; tbl.append(infoRow(`Block ${block}`, value, d == null ? null : d > 0)); @@ -620,7 +626,7 @@ const tbl = biProfileTable(biState); if (tbl) { box.append(el("div", "sw-rows-label", - "Busiest blocks — how much each moves the picture as it forms")); + "Busiest blocks — each block's share of all the movement in the stack")); box.append(tbl); } }; diff --git a/movie_editor/server.py b/movie_editor/server.py index 9b8faa4d..ed833f6c 100644 --- a/movie_editor/server.py +++ b/movie_editor/server.py @@ -2200,6 +2200,7 @@ if web is not None and PromptServer is not None: "overall": {str(b): v for b, v in (prof["overall"] or {}).items()}, "difference": ({str(b): v for b, v in prof["difference"].items()} if prof["difference"] else None), + "share": {str(b): v for b, v in (prof["share"] or {}).items()}, "novelty": {str(b): v for b, v in (prof["novelty"] or {}).items()}, "mean_novelty": prof["mean_novelty"], } @@ -2213,7 +2214,7 @@ if web is not None and PromptServer is not None: except Exception as e: return web.json_response({"key": key, "enabled": False, "runs": 0, "n_liked": 0, "n_disliked": 0, "flatness": None, - "overall": {}, "difference": None, "novelty": {}, + "overall": {}, "difference": None, "novelty": {}, "share": {}, "mean_novelty": None, "error": str(e)}) @routes.post(UI_PREFIX + "/api/block_influence") diff --git a/samplers.py b/samplers.py index 50b912f9..8dfe94ed 100644 --- a/samplers.py +++ b/samplers.py @@ -3792,10 +3792,12 @@ class FunPackLTXAVSceneChainSampler: except ImportError: import block_influence as _bi # One host sync for the whole run, here, instead of per block per step. + _bi_means = {b: torch.stack(v).mean(dim=0) + for b, v in _influence_capture[0].items() if v} _bi.save_pending( refinement_key, - {b: float(torch.stack(v).mean().item()) - for b, v in _influence_capture[0].items() if v}, + {b: float((m[0] / m[1]).item()) for b, m in _bi_means.items()}, + raw={b: float(m[0].item()) for b, m in _bi_means.items()}, novelty={b: float(torch.stack(v).mean().item()) for b, v in _influence_capture[1].items() if v}) elif _influence_on: @@ -6942,8 +6944,14 @@ class FunPackLTXAVSceneChainSampler: base = before.norm() # Kept as a 0-dim tensor: averaged and read once, after # sampling, instead of syncing the device 50x per step. + # BOTH norms, not their ratio. The residual stream grows with + # depth, so ||f_i||/||x_i|| shrinks with depth by construction -- + # early blocks score huge because the denominator is barely + # formed, not because they do more work. Storing the pair lets + # the readout normalise by something comparable across depth + # instead of baking the confound in here. capture_holder[0].setdefault(block, []).append( - delta.norm() / base.clamp(min=1e-8)) + torch.stack([delta.norm(), base.clamp(min=1e-8)])) if len(capture_holder) > 1: last, last_block = prev_delta if (last is not None and block > last_block diff --git a/tests/test_block_influence.py b/tests/test_block_influence.py index 375cca0c..4359336e 100644 --- a/tests/test_block_influence.py +++ b/tests/test_block_influence.py @@ -182,6 +182,13 @@ def _install(capture, n_blocks=2, max_rows=512): return patched.model_options["transformer_options"]["patches_replace"]["dit"] +def _ratio(entries): + """capture[0][block] holds [||delta||, ||before||] pairs -- the ratio is derived, not + stored, so the readout can normalise by something comparable across depth.""" + m = torch.stack(entries).mean(dim=0) + return float(m[0] / m[1]) + + def _call(hook, src, out, mod_segments): """Drive the hook the way minimax/model.py does.""" return hook({"img": src, "mod_segments": mod_segments}, @@ -197,7 +204,7 @@ def test_hook_records_the_relative_delta_on_video_rows(): out[:, 1] = 3.0 # delta has the same norm as the input res = _call(dit[("double_block", 0)], src, out, [(0, 4, 6)]) # tag 6 % 3 == 0 -> video assert torch.equal(res["img"], out) # measurement must not modify the stream - recorded = torch.stack(capture[0][0]).mean().item() + recorded = _ratio(capture[0][0]) assert recorded == pytest.approx(1.0, rel=1e-4) @@ -210,7 +217,7 @@ def test_hook_averages_across_steps(): out = src.clone() out[:, 1] = scale _call(dit[("double_block", 0)], src, out, [(0, 4, 6)]) - assert torch.stack(capture[0][0]).mean().item() == pytest.approx(2.0, rel=1e-4) + assert _ratio(capture[0][0]) == pytest.approx(2.0, rel=1e-4) def test_text_and_audio_rows_are_not_measured(): @@ -222,7 +229,7 @@ def test_text_and_audio_rows_are_not_measured(): out = src.clone() out[2:, 1] = 50.0 # rows 2-3 are text below _call(dit[("double_block", 0)], src, out, [(0, 2, 6), (2, 4, 7)]) - recorded = torch.stack(capture[0][0]).mean().item() + recorded = _ratio(capture[0][0]) assert recorded == pytest.approx(0.0, abs=1e-6) @@ -244,8 +251,8 @@ def test_each_block_accumulates_separately(): out[:, 1] = scale _call(dit[("double_block", block)], src, out, [(0, 4, 6)]) assert set(capture[0]) == {0, 2} # block 1 was never called - assert torch.stack(capture[0][0]).mean().item() == pytest.approx(1.0, rel=1e-4) - assert torch.stack(capture[0][2]).mean().item() == pytest.approx(4.0, rel=1e-4) + assert _ratio(capture[0][0]) == pytest.approx(1.0, rel=1e-4) + assert _ratio(capture[0][2]) == pytest.approx(4.0, rel=1e-4) def test_row_subsampling_bounds_the_reduction(): @@ -258,7 +265,7 @@ def test_row_subsampling_bounds_the_reduction(): out = src.clone() out[:, 1] = 2.0 _call(dit[("double_block", 0)], src, out, [(0, n, 6)]) - assert torch.stack(capture[0][0]).mean().item() == pytest.approx(2.0, rel=1e-3) + assert _ratio(capture[0][0]) == pytest.approx(2.0, rel=1e-3) def test_an_existing_patch_at_the_same_block_is_chained_not_replaced(): @@ -488,7 +495,7 @@ def test_a_block_that_writes_in_place_is_still_measured(): res = dit[("double_block", 0)]({"img": src, "mod_segments": [(0, 4, 6)]}, {"original_block": in_place}) - recorded = torch.stack(capture[0][0]).mean().item() + recorded = _ratio(capture[0][0]) assert recorded == pytest.approx(2.0, rel=1e-4) # |delta| = 2*|before| assert torch.equal(res["img"], src) @@ -501,4 +508,37 @@ def test_a_block_returning_a_new_tensor_is_unaffected_by_the_fix(): out = src.clone() out[:, 1] = 3.0 _call(dit[("double_block", 0)], src, out, [(0, 4, 6)]) - assert torch.stack(capture[0][0]).mean().item() == pytest.approx(3.0, rel=1e-4) + assert _ratio(capture[0][0]) == pytest.approx(3.0, rel=1e-4) + + +# --- share vs ratio: the depth confound ------------------------------------------- +# +# ||f_i||/||x_i|| is not comparable across depth. The residual stream GROWS as it goes, so +# an early block scores high because its denominator is barely formed, not because it moves +# the picture more. Live data made this concrete: block 1 read 6.01 and block 36 read 0.368, +# a 16x gap that is mostly the stream norm. `share` divides every block by the same total +# instead, so the ranking answers "who moved the picture" rather than "who ran early". + +def test_share_is_each_blocks_fraction_of_all_movement(): + bi.save_pending("k", {0: 9.0, 1: 1.0}, raw={0: 1.0, 1: 3.0}) + bi.commit("k", 1.0) + p = bi.profile("k") + assert p["share"][0] == pytest.approx(0.25) + assert p["share"][1] == pytest.approx(0.75) + # ...and the ratio still says the opposite, which is exactly the confound. + assert p["overall"][0] > p["overall"][1] + + +def test_share_is_absent_rather_than_wrong_without_raw_norms(): + bi.save_pending("k", {0: 0.5}) + bi.commit("k", 1.0) + assert bi.profile("k")["share"] == {} + + +def test_share_averages_across_runs_before_normalising(): + for raw in ({0: 1.0, 1: 1.0}, {0: 3.0, 1: 1.0}): + bi.save_pending("k", {0: 0.1, 1: 0.1}, raw=raw) + bi.commit("k", 1.0) + p = bi.profile("k") + assert p["raw"][0] == pytest.approx(2.0) # mean of 1 and 3 + assert p["share"][0] == pytest.approx(2.0 / 3.0)