nodes/olivv-cs-ComfyUI-FunPack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: rank blocks by share of movement, not by a ratio depth decides

Browse Source 
Live data showed block 1 at 6.01 and block 0 at 1.83 against ~0.2-0.35
everywhere else, with block 1's liked/disliked gap (+5.82) as large as
its whole measurement. That is not "the first two blocks do 30x the
work" — ||f_i||/||x_i|| divides by a residual stream that GROWS with
depth, so early blocks score high because the denominator is barely
formed. Ranking on it just sorts the earliest blocks to the top, and a
high-variance early block fakes a large taste correlation at low n.

The hook now stores both norms instead of their ratio, and profile()
adds `share` — each block's raw ||f_i|| over every block's, one
denominator for the whole stack. The panel ranks and displays that
("2.3% of the movement"). The ratio is kept, since it is still the right
answer to "how much did this block move the stream relative to what the
stream was".

Checked against data shaped like the report: ratio ranks 0,1,2 (all
artifact), share ranks the blocks that actually move more.
This commit is contained in:
digital-garbage
2026-09-04 18:10:55 +03:00
parent b6841cbbd7
commit be5579f1f9
5 changed files with 101 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
block_influence.py
+28 -4
View File
@@ -120,6 +120,7 @@ def _load(refinement_key):
data.setdefault("rows", [])
data.setdefault("pending", None)
data.setdefault("pending_novelty", None)
data.setdefault("pending_raw", None)
return data
@@ -131,7 +132,7 @@ def _save(refinement_key, data):
os.replace(tmp, path)
def save_pending(refinement_key, profile, novelty=None):
def save_pending(refinement_key, profile, novelty=None, raw=None):
"""Store this run's per-block profile as the pending candidate the next rating scores.
`profile` is {block_index: mean relative delta}; `novelty` is the optional companion
{block_index: mean cosine with the PREVIOUS block's delta}. Overwritten every run, same
@@ -143,6 +144,9 @@ def save_pending(refinement_key, profile, novelty=None):
data["pending"] = {int(b): float(v) for b, v in profile.items()}
data["pending_novelty"] = ({int(b): float(v) for b, v in novelty.items()}
if novelty else None)
# Raw ||f_i|| alongside the ratio: see `share` in profile() for why the ratio alone is
# not comparable across depth.
data["pending_raw"] = ({int(b): float(v) for b, v in raw.items()} if raw else None)
try:
_save(refinement_key, data)
except OSError as e:
@@ -158,8 +162,10 @@ def commit(refinement_key, reward):
data = _load(refinement_key)
pending = data.get("pending")
pending_novelty = data.get("pending_novelty")
pending_raw = data.get("pending_raw")
data["pending"] = None
data["pending_novelty"] = None
data["pending_raw"] = None
if pending is None:
try:
_save(refinement_key, data)
@@ -168,7 +174,7 @@ def commit(refinement_key, reward):
return "no_pending"
try:
data["rows"].append({"profile": pending, "novelty": pending_novelty or {},
"weight": float(reward)})
"raw": pending_raw or {}, "weight": float(reward)})
except (TypeError, ValueError):
try:
_save(refinement_key, data)
@@ -211,7 +217,12 @@ def profile(refinement_key):
`mean_novelty` its average: ~1 means blocks mostly amplify their predecessor (little new
per block), ~0 means each adds an orthogonal component nothing before it added, negative
means blocks partly UNDO each other. Magnitude cannot tell those apart, which is why it
is recorded alongside rather than instead. `difference` is the rating-driven signal: blocks whose activity runs higher on
is recorded alongside rather than instead.
`share` is each block's RAW movement ||f_i|| as a fraction of every block's, which is the
number to rank by. `overall` (the ratio ||f_i||/||x_i||) is not comparable across depth:
the residual stream grows as it goes, so early blocks score high because the denominator
is barely formed rather than because they move the picture more. `difference` is the rating-driven signal: blocks whose activity runs higher on
runs you liked. `flatness` is the spread of `overall` as a fraction of its own mean
(coefficient of variation): near 0 means every block moves the stream by the same amount
and there is nothing to aim at; a large value means the depth profile is structured.
@@ -225,7 +236,7 @@ def profile(refinement_key):
if not rows or not blocks:
return {"overall": {}, "liked": None, "disliked": None, "difference": None,
"n_liked": 0, "n_disliked": 0, "flatness": None,
"novelty": {}, "mean_novelty": None}
"novelty": {}, "mean_novelty": None, "share": {}, "raw": {}}
def _mean(subset):
out = {}
@@ -244,10 +255,22 @@ def profile(refinement_key):
out[b] = sum(vals) / len(vals)
return out
def _mean_raw(subset):
out = {}
for b in blocks:
vals = [float(r["raw"][b]) for r in subset
if isinstance(r.get("raw"), dict) and b in r["raw"]]
if vals:
out[b] = sum(vals) / len(vals)
return out
liked_rows = [r for r in rows if r["weight"] > 0]
disliked_rows = [r for r in rows if r["weight"] < 0]
overall = _mean(rows)
novelty = _mean_novelty(rows)
raw = _mean_raw(rows)
total = sum(raw.values())
share = {b: v / total for b, v in raw.items()} if total > 1e-12 else {}
flatness = None
if overall:
vals = torch.tensor(list(overall.values()), dtype=torch.float32)
@@ -261,5 +284,6 @@ def profile(refinement_key):
return {"overall": overall, "liked": liked, "disliked": disliked,
"difference": difference, "n_liked": len(liked_rows),
"n_disliked": len(disliked_rows), "flatness": flatness,
"share": share, "raw": raw,
"novelty": novelty, "mean_novelty": (
sum(novelty.values()) / len(novelty) if novelty else None)}
movie_editor/frontend/settings_window.js
+12 -6
View File
@@ -540,15 +540,21 @@
const diff = (state && state.difference) || null;
// Ranked by how much the block moves the picture, not by block number: the question is
// which blocks do the work, and a 50-row list in index order buries that.
const entries = Object.entries(overall)
.map(([b, v]) => [Number(b), v])
.sort((a, b) => b[1] - a[1])
// Rank by SHARE, not by the ratio: ||f_i||/||x_i|| shrinks with depth by construction
// (the residual stream grows as it goes), so ranking on it just sorts the earliest
// blocks to the top regardless of how much they actually move the picture.
const share = (state && state.share) || {};
const rankOn = Object.keys(share).length ? share : overall;
const entries = Object.keys(rankOn)
.map((b) => [Number(b), overall[b], share[b]])
.sort((a, b) => (b[2] ?? b[1]) - (a[2] ?? a[1]))
.slice(0, 12);
const tbl = el("div", "sw-rows");
entries.forEach(([block, v]) => {
entries.forEach(([block, v, sh]) => {
const d = diff ? diff[String(block)] : null;
const nov = state.novelty ? state.novelty[String(block)] : null;
let value = v.toPrecision(3);
let value = sh != null ? `${(sh * 100).toFixed(1)}% of the movement`
: (v == null ? "—" : v.toPrecision(3));
if (nov != null) value += ` · new ${nov >= 0 ? "+" : ""}${nov.toFixed(2)}`;
if (d != null) value += ` · liked ${d >= 0 ? "+" : ""}${d.toFixed(4)}`;
tbl.append(infoRow(`Block ${block}`, value, d == null ? null : d > 0));
@@ -620,7 +626,7 @@
const tbl = biProfileTable(biState);
if (tbl) {
box.append(el("div", "sw-rows-label",
"Busiest blocks — how much each moves the picture as it forms"));
"Busiest blocks — each block's share of all the movement in the stack"));
box.append(tbl);
}
};
movie_editor/server.py
+2 -1
View File
@@ -2200,6 +2200,7 @@ if web is not None and PromptServer is not None:
"overall": {str(b): v for b, v in (prof["overall"] or {}).items()},
"difference": ({str(b): v for b, v in prof["difference"].items()}
if prof["difference"] else None),
"share": {str(b): v for b, v in (prof["share"] or {}).items()},
"novelty": {str(b): v for b, v in (prof["novelty"] or {}).items()},
"mean_novelty": prof["mean_novelty"],
}
@@ -2213,7 +2214,7 @@ if web is not None and PromptServer is not None:
except Exception as e:
return web.json_response({"key": key, "enabled": False, "runs": 0,
"n_liked": 0, "n_disliked": 0, "flatness": None,
"overall": {}, "difference": None, "novelty": {},
"overall": {}, "difference": None, "novelty": {}, "share": {},
"mean_novelty": None, "error": str(e)})
@routes.post(UI_PREFIX + "/api/block_influence")
samplers.py
+11 -3
View File
@@ -3792,10 +3792,12 @@ class FunPackLTXAVSceneChainSampler:
except ImportError:
import block_influence as _bi
# One host sync for the whole run, here, instead of per block per step.
_bi_means = {b: torch.stack(v).mean(dim=0)
for b, v in _influence_capture[0].items() if v}
_bi.save_pending(
refinement_key,
{b: float(torch.stack(v).mean().item())
for b, v in _influence_capture[0].items() if v},
{b: float((m[0] / m[1]).item()) for b, m in _bi_means.items()},
raw={b: float(m[0].item()) for b, m in _bi_means.items()},
novelty={b: float(torch.stack(v).mean().item())
for b, v in _influence_capture[1].items() if v})
elif _influence_on:
@@ -6942,8 +6944,14 @@ class FunPackLTXAVSceneChainSampler:
base = before.norm()
# Kept as a 0-dim tensor: averaged and read once, after
# sampling, instead of syncing the device 50x per step.
# BOTH norms, not their ratio. The residual stream grows with
# depth, so ||f_i||/||x_i|| shrinks with depth by construction --
# early blocks score huge because the denominator is barely
# formed, not because they do more work. Storing the pair lets
# the readout normalise by something comparable across depth
# instead of baking the confound in here.
capture_holder[0].setdefault(block, []).append(
delta.norm() / base.clamp(min=1e-8))
torch.stack([delta.norm(), base.clamp(min=1e-8)]))
if len(capture_holder) > 1:
last, last_block = prev_delta
if (last is not None and block > last_block
tests/test_block_influence.py
+48 -8
View File
@@ -182,6 +182,13 @@ def _install(capture, n_blocks=2, max_rows=512):
return patched.model_options["transformer_options"]["patches_replace"]["dit"]
def _ratio(entries):
"""capture[0][block] holds [||delta||, ||before||] pairs -- the ratio is derived, not
stored, so the readout can normalise by something comparable across depth."""
m = torch.stack(entries).mean(dim=0)
return float(m[0] / m[1])
def _call(hook, src, out, mod_segments):
"""Drive the hook the way minimax/model.py does."""
return hook({"img": src, "mod_segments": mod_segments},
@@ -197,7 +204,7 @@ def test_hook_records_the_relative_delta_on_video_rows():
out[:, 1] = 3.0 # delta has the same norm as the input
res = _call(dit[("double_block", 0)], src, out, [(0, 4, 6)]) # tag 6 % 3 == 0 -> video
assert torch.equal(res["img"], out) # measurement must not modify the stream
recorded = torch.stack(capture[0][0]).mean().item()
recorded = _ratio(capture[0][0])
assert recorded == pytest.approx(1.0, rel=1e-4)
@@ -210,7 +217,7 @@ def test_hook_averages_across_steps():
out = src.clone()
out[:, 1] = scale
_call(dit[("double_block", 0)], src, out, [(0, 4, 6)])
assert torch.stack(capture[0][0]).mean().item() == pytest.approx(2.0, rel=1e-4)
assert _ratio(capture[0][0]) == pytest.approx(2.0, rel=1e-4)
def test_text_and_audio_rows_are_not_measured():
@@ -222,7 +229,7 @@ def test_text_and_audio_rows_are_not_measured():
out = src.clone()
out[2:, 1] = 50.0 # rows 2-3 are text below
_call(dit[("double_block", 0)], src, out, [(0, 2, 6), (2, 4, 7)])
recorded = torch.stack(capture[0][0]).mean().item()
recorded = _ratio(capture[0][0])
assert recorded == pytest.approx(0.0, abs=1e-6)
@@ -244,8 +251,8 @@ def test_each_block_accumulates_separately():
out[:, 1] = scale
_call(dit[("double_block", block)], src, out, [(0, 4, 6)])
assert set(capture[0]) == {0, 2} # block 1 was never called
assert torch.stack(capture[0][0]).mean().item() == pytest.approx(1.0, rel=1e-4)
assert torch.stack(capture[0][2]).mean().item() == pytest.approx(4.0, rel=1e-4)
assert _ratio(capture[0][0]) == pytest.approx(1.0, rel=1e-4)
assert _ratio(capture[0][2]) == pytest.approx(4.0, rel=1e-4)
def test_row_subsampling_bounds_the_reduction():
@@ -258,7 +265,7 @@ def test_row_subsampling_bounds_the_reduction():
out = src.clone()
out[:, 1] = 2.0
_call(dit[("double_block", 0)], src, out, [(0, n, 6)])
assert torch.stack(capture[0][0]).mean().item() == pytest.approx(2.0, rel=1e-3)
assert _ratio(capture[0][0]) == pytest.approx(2.0, rel=1e-3)
def test_an_existing_patch_at_the_same_block_is_chained_not_replaced():
@@ -488,7 +495,7 @@ def test_a_block_that_writes_in_place_is_still_measured():
res = dit[("double_block", 0)]({"img": src, "mod_segments": [(0, 4, 6)]},
{"original_block": in_place})
recorded = torch.stack(capture[0][0]).mean().item()
recorded = _ratio(capture[0][0])
assert recorded == pytest.approx(2.0, rel=1e-4) # |delta| = 2*|before|
assert torch.equal(res["img"], src)
@@ -501,4 +508,37 @@ def test_a_block_returning_a_new_tensor_is_unaffected_by_the_fix():
out = src.clone()
out[:, 1] = 3.0
_call(dit[("double_block", 0)], src, out, [(0, 4, 6)])
assert torch.stack(capture[0][0]).mean().item() == pytest.approx(3.0, rel=1e-4)
assert _ratio(capture[0][0]) == pytest.approx(3.0, rel=1e-4)
# --- share vs ratio: the depth confound -------------------------------------------
#
# ||f_i||/||x_i|| is not comparable across depth. The residual stream GROWS as it goes, so
# an early block scores high because its denominator is barely formed, not because it moves
# the picture more. Live data made this concrete: block 1 read 6.01 and block 36 read 0.368,
# a 16x gap that is mostly the stream norm. `share` divides every block by the same total
# instead, so the ranking answers "who moved the picture" rather than "who ran early".
def test_share_is_each_blocks_fraction_of_all_movement():
bi.save_pending("k", {0: 9.0, 1: 1.0}, raw={0: 1.0, 1: 3.0})
bi.commit("k", 1.0)
p = bi.profile("k")
assert p["share"][0] == pytest.approx(0.25)
assert p["share"][1] == pytest.approx(0.75)
# ...and the ratio still says the opposite, which is exactly the confound.
assert p["overall"][0] > p["overall"][1]
def test_share_is_absent_rather_than_wrong_without_raw_norms():
bi.save_pending("k", {0: 0.5})
bi.commit("k", 1.0)
assert bi.profile("k")["share"] == {}
def test_share_averages_across_runs_before_normalising():
for raw in ({0: 1.0, 1: 1.0}, {0: 3.0, 1: 1.0}):
bi.save_pending("k", {0: 0.1, 1: 0.1}, raw=raw)
bi.commit("k", 1.0)
p = bi.profile("k")
assert p["raw"][0] == pytest.approx(2.0) # mean of 1 and 3
assert p["share"][0] == pytest.approx(2.0 / 3.0)