diff --git a/block_influence.py b/block_influence.py index c27ac9f4..b57045c4 100644 --- a/block_influence.py +++ b/block_influence.py @@ -119,6 +119,7 @@ def _load(refinement_key): return {"rows": [], "pending": None} data.setdefault("rows", []) data.setdefault("pending", None) + data.setdefault("pending_novelty", None) return data @@ -130,15 +131,18 @@ def _save(refinement_key, data): os.replace(tmp, path) -def save_pending(refinement_key, profile): +def save_pending(refinement_key, profile, novelty=None): """Store this run's per-block profile as the pending candidate the next rating scores. - `profile` is {block_index: mean relative delta}. Overwritten every run, same reason - h3_repr_steering.save_pending overwrites: a pending capture must never survive to be - paired with a LATER run's rating.""" + `profile` is {block_index: mean relative delta}; `novelty` is the optional companion + {block_index: mean cosine with the PREVIOUS block's delta}. Overwritten every run, same + reason h3_repr_steering.save_pending overwrites: a pending capture must never survive to + be paired with a LATER run's rating.""" if not refinement_key or not profile: return data = _load(refinement_key) data["pending"] = {int(b): float(v) for b, v in profile.items()} + data["pending_novelty"] = ({int(b): float(v) for b, v in novelty.items()} + if novelty else None) try: _save(refinement_key, data) except OSError as e: @@ -153,7 +157,9 @@ def commit(refinement_key, reward): return "no_key" data = _load(refinement_key) pending = data.get("pending") + pending_novelty = data.get("pending_novelty") data["pending"] = None + data["pending_novelty"] = None if pending is None: try: _save(refinement_key, data) @@ -161,7 +167,8 @@ def commit(refinement_key, reward): pass return "no_pending" try: - data["rows"].append({"profile": pending, "weight": float(reward)}) + data["rows"].append({"profile": pending, "novelty": pending_novelty or {}, + "weight": float(reward)}) except (TypeError, ValueError): try: _save(refinement_key, data) @@ -200,7 +207,11 @@ def profile(refinement_key): "flatness": float|None} `overall` is the depth profile itself -- where this model does its work, regardless of - ratings. `difference` is the rating-driven signal: blocks whose activity runs higher on + ratings. `novelty` is the per-block cosine with the PREVIOUS block's delta, and + `mean_novelty` its average: ~1 means blocks mostly amplify their predecessor (little new + per block), ~0 means each adds an orthogonal component nothing before it added, negative + means blocks partly UNDO each other. Magnitude cannot tell those apart, which is why it + is recorded alongside rather than instead. `difference` is the rating-driven signal: blocks whose activity runs higher on runs you liked. `flatness` is the spread of `overall` as a fraction of its own mean (coefficient of variation): near 0 means every block moves the stream by the same amount and there is nothing to aim at; a large value means the depth profile is structured. @@ -213,7 +224,8 @@ def profile(refinement_key): blocks = _blocks_in(rows) if not rows or not blocks: return {"overall": {}, "liked": None, "disliked": None, "difference": None, - "n_liked": 0, "n_disliked": 0, "flatness": None} + "n_liked": 0, "n_disliked": 0, "flatness": None, + "novelty": {}, "mean_novelty": None} def _mean(subset): out = {} @@ -223,9 +235,19 @@ def profile(refinement_key): out[b] = sum(vals) / len(vals) return out + def _mean_novelty(subset): + out = {} + for b in blocks: + vals = [float(r["novelty"][b]) for r in subset + if isinstance(r.get("novelty"), dict) and b in r["novelty"]] + if vals: + out[b] = sum(vals) / len(vals) + return out + liked_rows = [r for r in rows if r["weight"] > 0] disliked_rows = [r for r in rows if r["weight"] < 0] overall = _mean(rows) + novelty = _mean_novelty(rows) flatness = None if overall: vals = torch.tensor(list(overall.values()), dtype=torch.float32) @@ -238,4 +260,6 @@ def profile(refinement_key): difference = {b: liked[b] - disliked[b] for b in liked if b in disliked} return {"overall": overall, "liked": liked, "disliked": disliked, "difference": difference, "n_liked": len(liked_rows), - "n_disliked": len(disliked_rows), "flatness": flatness} + "n_disliked": len(disliked_rows), "flatness": flatness, + "novelty": novelty, "mean_novelty": ( + sum(novelty.values()) / len(novelty) if novelty else None)} diff --git a/movie_editor/frontend/settings_window.js b/movie_editor/frontend/settings_window.js index 2121aab7..7b172466 100644 --- a/movie_editor/frontend/settings_window.js +++ b/movie_editor/frontend/settings_window.js @@ -416,6 +416,21 @@ return `Flatness ${f.toFixed(3)} · ${shape}`; } + // How much of each block's push is NEW, rather than more of what the previous block was + // already doing. Magnitude cannot tell those apart; this is the number that speaks to + // whether 50 blocks are doing 50 things or one thing 50 times. + function biNoveltyVerdict(state) { + const n = state && state.mean_novelty; + if (n == null) return null; + const shape = n > 0.6 ? "each block mostly amplifies the one before it — little new per " + + "block" + : n > 0.2 ? "partly new, partly a continuation of the previous block" + : n > -0.2 ? "each block adds something the one before it did not — near-independent " + + "contributions" + : "blocks partly undo each other, so the useful work is smaller than the sizes suggest"; + return `Novelty ${n >= 0 ? "+" : ""}${n.toFixed(3)} · ${shape}`; + } + function biProfileTable(state) { const overall = state && state.overall; if (!overall || !Object.keys(overall).length) return null; @@ -429,8 +444,10 @@ const tbl = el("div", "sw-rows"); entries.forEach(([block, v]) => { const d = diff ? diff[String(block)] : null; - const value = d == null ? v.toFixed(4) - : `${v.toFixed(4)} · liked ${d >= 0 ? "+" : ""}${d.toFixed(4)}`; + const nov = state.novelty ? state.novelty[String(block)] : null; + let value = v.toFixed(4); + if (nov != null) value += ` · new ${nov >= 0 ? "+" : ""}${nov.toFixed(2)}`; + if (d != null) value += ` · liked ${d >= 0 ? "+" : ""}${d.toFixed(4)}`; tbl.append(infoRow(`Block ${block}`, value, d == null ? null : d > 0)); }); return tbl; @@ -489,6 +506,8 @@ box.append(rows); box.append(el("div", "sw-hint", biVerdict(biState))); + const nv = biNoveltyVerdict(biState); + if (nv) box.append(el("div", "sw-hint", nv)); if (biState && biState.difference == null && runs > 0) { box.append(el("div", "sw-hint", `Needs ${biState.min_per_group}+ liked and ${biState.min_per_group}+ disliked ` diff --git a/movie_editor/server.py b/movie_editor/server.py index 4c22d8d4..6adb20fe 100644 --- a/movie_editor/server.py +++ b/movie_editor/server.py @@ -2200,6 +2200,8 @@ if web is not None and PromptServer is not None: "overall": {str(b): v for b, v in (prof["overall"] or {}).items()}, "difference": ({str(b): v for b, v in prof["difference"].items()} if prof["difference"] else None), + "novelty": {str(b): v for b, v in (prof["novelty"] or {}).items()}, + "mean_novelty": prof["mean_novelty"], } @routes.get(UI_PREFIX + "/api/block_influence") @@ -2211,7 +2213,8 @@ if web is not None and PromptServer is not None: except Exception as e: return web.json_response({"key": key, "enabled": False, "runs": 0, "n_liked": 0, "n_disliked": 0, "flatness": None, - "overall": {}, "difference": None, "error": str(e)}) + "overall": {}, "difference": None, "novelty": {}, + "mean_novelty": None, "error": str(e)}) @routes.post(UI_PREFIX + "/api/block_influence") async def _block_influence_toggle(req): diff --git a/samplers.py b/samplers.py index 5521e93b..69c1a097 100644 --- a/samplers.py +++ b/samplers.py @@ -3744,7 +3744,7 @@ class FunPackLTXAVSceneChainSampler: # Measurement first, steering second: whatever is installed later chains this probe # as its inner call, so the delta it records is the block's OWN, not one that already # contains another mechanism's injection. - _influence_capture = [{}] + _influence_capture = [{}, {}] # [0] relative magnitudes, [1] novelty cosines _influence_on = False if refinement_key: try: @@ -3778,9 +3778,12 @@ class FunPackLTXAVSceneChainSampler: except ImportError: import block_influence as _bi # One host sync for the whole run, here, instead of per block per step. - _bi.save_pending(refinement_key, { - b: float(torch.stack(v).mean().item()) - for b, v in _influence_capture[0].items() if v}) + _bi.save_pending( + refinement_key, + {b: float(torch.stack(v).mean().item()) + for b, v in _influence_capture[0].items() if v}, + novelty={b: float(torch.stack(v).mean().item()) + for b, v in _influence_capture[1].items() if v}) elif _influence_on: # An empty capture and a genuinely flat profile look identical downstream, # and this probe exists to tell those apart -- so it says which one happened @@ -6779,6 +6782,15 @@ class FunPackLTXAVSceneChainSampler: the number the block's OWN delta instead of one that includes another mechanism's injection (the same capture-before-injection ordering h3_repr_steering needed). + Also records NOVELTY: the cosine between this block's delta and the previous block's. + Magnitude alone cannot distinguish a block adding something nothing before it added + from a block pushing harder in the direction the last one already went. ~1 means it + is amplifying its predecessor (little new), ~0 means orthogonal (a genuinely new + component), negative means it is partly UNDOING the previous block -- which real + stacks do, and which would mean the effective work is far smaller than the sum of the + magnitudes suggests. `capture_holder[0]` collects the relative magnitudes, + `capture_holder[1]` the novelty cosines. + `n_blocks` over-covers deliberately: registering a patch for an index the model does not have is never called, so a shorter stack simply reports fewer blocks.""" try: @@ -6790,6 +6802,11 @@ class FunPackLTXAVSceneChainSampler: to = patched.model_options.get("transformer_options", {}).copy() patches_replace = dict(to.get("patches_replace", {})) dit_patches = dict(patches_replace.get("dit", {})) + # The previous block's delta, for the novelty cosine. Blocks fire in index order + # within a step, so a block index that does NOT exceed the last one seen means a + # new step began and there is no predecessor to compare against -- pairing across + # that boundary would compare the end of one step to the start of the next. + prev_delta = [None, -1] def _make_probe(block): inner = dit_patches.get(("double_block", block)) @@ -6818,6 +6835,14 @@ class FunPackLTXAVSceneChainSampler: # sampling, instead of syncing the device 50x per step. capture_holder[0].setdefault(block, []).append( delta.norm() / base.clamp(min=1e-8)) + if len(capture_holder) > 1: + last, last_block = prev_delta + if (last is not None and block > last_block + and last.shape == delta.shape): + capture_holder[1].setdefault(block, []).append( + torch.nn.functional.cosine_similarity( + delta.flatten(), last.flatten(), dim=0)) + prev_delta[0], prev_delta[1] = delta, block except Exception: # noqa: BLE001 pass # a probe never breaks a generation return {"img": out} diff --git a/tests/test_block_influence.py b/tests/test_block_influence.py index c3b4c74a..56b15573 100644 --- a/tests/test_block_influence.py +++ b/tests/test_block_influence.py @@ -350,3 +350,123 @@ def test_the_switch_does_not_gate_recording(_switch, monkeypatch, tmp_path): bi.save_pending("k", {0: 0.5}) assert bi.commit("k", 1.0) == "recorded" assert bi.profile("k")["overall"][0] == pytest.approx(0.5) + + +# --- novelty: what each block ADDS that the one before it didn't ------------------ +# +# Magnitude alone cannot tell "this block added something new" from "this block pushed +# harder in the direction the last one already went". The cosine with the previous block's +# delta is what separates those, and it is the only number here that speaks to whether the +# 50 blocks are doing 50 different things or one thing 50 times. + +def test_novelty_is_recorded_and_averaged(): + bi.save_pending("k", {1: 0.1, 2: 0.2}, novelty={2: 0.0}) + bi.commit("k", 1.0) + bi.save_pending("k", {1: 0.1, 2: 0.2}, novelty={2: 1.0}) + bi.commit("k", 1.0) + p = bi.profile("k") + assert p["novelty"][2] == pytest.approx(0.5) + assert p["mean_novelty"] == pytest.approx(0.5) + + +def test_novelty_is_optional_and_old_rows_stay_readable(): + """A row recorded before novelty existed carries none; it must not break the readout or + be counted as a 0.0 cosine (which would read as 'perfectly orthogonal').""" + bi.save_pending("k", {1: 0.5}) # no novelty + bi.commit("k", 1.0) + bi.save_pending("k", {1: 0.5}, novelty={1: 0.8}) + bi.commit("k", 1.0) + p = bi.profile("k") + assert p["novelty"][1] == pytest.approx(0.8) # averaged over the row that HAS it only + + +def test_no_novelty_anywhere_reports_none_not_zero(): + bi.save_pending("k", {1: 0.5}) + bi.commit("k", 1.0) + p = bi.profile("k") + assert p["novelty"] == {} + assert p["mean_novelty"] is None + + +def test_negative_novelty_survives_the_round_trip(): + """Blocks partly undoing each other is a real outcome and the most interesting one -- + it must not be clamped or dropped on the way through.""" + bi.save_pending("k", {3: 0.4}, novelty={3: -0.6}) + bi.commit("k", 1.0) + assert bi.profile("k")["novelty"][3] == pytest.approx(-0.6) + + +# --- the hook's novelty half ------------------------------------------------------ + +def _install2(capture, n_blocks=4, max_rows=512): + s = samplers.FunPackLTXAVSceneChainSampler() + patched = s._install_block_influence(_Model(), capture, n_blocks=n_blocks, + max_rows=max_rows) + return patched.model_options["transformer_options"]["patches_replace"]["dit"] + + +def _push(dit, block, src, delta_vec, segs=None): + out = src + delta_vec + return _call(dit[("double_block", block)], src, out, segs or [(0, 4, 6)]) + + +def test_hook_records_orthogonal_deltas_as_zero_novelty(): + capture = [{}, {}] + dit = _install2(capture) + src = torch.zeros(4, 8) + src[:, 0] = 1.0 + d0 = torch.zeros(4, 8); d0[:, 1] = 1.0 + d1 = torch.zeros(4, 8); d1[:, 2] = 1.0 # orthogonal to d0 + _push(dit, 0, src, d0) + _push(dit, 1, src, d1) + assert torch.stack(capture[1][1]).mean().item() == pytest.approx(0.0, abs=1e-5) + + +def test_hook_records_a_repeated_direction_as_novelty_one(): + capture = [{}, {}] + dit = _install2(capture) + src = torch.zeros(4, 8) + src[:, 0] = 1.0 + d = torch.zeros(4, 8); d[:, 1] = 1.0 + _push(dit, 0, src, d) + _push(dit, 1, src, d * 3.0) # same direction, bigger + assert torch.stack(capture[1][1]).mean().item() == pytest.approx(1.0, rel=1e-4) + + +def test_hook_records_an_undoing_block_as_negative_novelty(): + capture = [{}, {}] + dit = _install2(capture) + src = torch.zeros(4, 8) + src[:, 0] = 1.0 + d = torch.zeros(4, 8); d[:, 1] = 1.0 + _push(dit, 0, src, d) + _push(dit, 1, src, -d) + assert torch.stack(capture[1][1]).mean().item() == pytest.approx(-1.0, rel=1e-4) + + +def test_the_first_block_of_a_step_has_no_predecessor(): + """Block 0 of step 2 must not be compared against block 49 of step 1 -- that pairs the + end of one denoise step with the start of the next, which is not a depth relationship.""" + capture = [{}, {}] + dit = _install2(capture) + src = torch.zeros(4, 8) + src[:, 0] = 1.0 + d = torch.zeros(4, 8); d[:, 1] = 1.0 + for _step in range(2): # two steps, blocks 0..2 each + for b in range(3): + _push(dit, b, src, d) + assert 0 not in capture[1] # never a novelty entry for the first block + assert len(capture[1][1]) == 2 # one per step, not three + + +def test_novelty_is_skipped_when_the_holder_has_no_slot_for_it(): + """Back-compat: a caller passing a 1-slot holder still gets magnitudes and no crash.""" + capture = [{}] + dit = _install2(capture) + src = torch.zeros(4, 8) + src[:, 0] = 1.0 + d = torch.zeros(4, 8); d[:, 1] = 1.0 + _push(dit, 0, src, d) + _push(dit, 1, src, d) + assert set(capture[0]) == {0, 1} + assert len(capture) == 1