fix: compare what differs between runs, not what they share

Runs of one prompt share most of their content: on a real 21-run session, 86% of
each descriptor was the part every run had. Comparing raw vectors measured that
shared part, so every distance came out near zero and near equal — mean cosine
0.97, and no separation at ANY bucket including the late ones where steering
demonstrably works.

Centring first flips the same data from -0.002 (backwards) to +0.03 and p 0.83
to 0.11, consistent across all four buckets. Not significant at 5 good / 16 bad,
but pointing the right way and now limited by sample size rather than by the
statistic.

The descriptor was not at fault. adaptive_avg_pool1d over the flat latent keeps
512 REGIONAL means, not one global mean, and it distinguishes clips fine.

Also: the prompt id is quantised before hashing. Raw bytes answered "are these
bit-identical", so any value crossing a rounding boundary changed the id — one
prompt produced 20 distinct ids across 21 runs. Stable now for jitter of the
size a GPU produces; no exact hash of floats can do better.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tests/test_trajectory_probe.py

@@ -709,3 +709,58 @@ def test_a_row_with_nothing_in_it_is_not_a_run(monkeypatch, tmp_path):
     junk = [{}, {"rows": []}, {"rows": [{"bucket": 0}], "reward": "not a number"},
             "a string", None]
     assert probe.merge_rows("imported", junk) == (0, 0)
+
+
+# ---------------------------------------------------------------------------
+# Centring, and the hash
+# ---------------------------------------------------------------------------
+
+
+def test_signal_hidden_under_shared_content_is_found():
+    """Runs of one prompt share most of their content. Measured on a real session, 86% of
+    each descriptor was the part every run had; comparing raw vectors measured that and
+    reported no separation at any bucket, including the late control.
+
+    Here the shared part is 20x the difference — the regime the real data was in."""
+    shared = torch.ones(64) * 20.0
+    good = [shared + torch.full((64,), 1.0) + torch.randn(64) * 0.05 for _ in range(6)]
+    bad = [shared - torch.full((64,), 1.0) + torch.randn(64) * 0.05 for _ in range(6)]
+    labels = [1] * 6 + [0] * 6
+
+    result = probe.permutation_test(good + bad, labels, trials=400, seed=1)
+    assert result["separation"] > 0
+    assert result["p_value"] <= 0.05
+
+
+def test_centring_does_not_invent_a_split_in_noise():
+    """The other half: removing the shared part must not turn nothing into something."""
+    gen = torch.Generator().manual_seed(11)
+    vectors = [torch.ones(64) * 20.0 + torch.randn(64, generator=gen) for _ in range(12)]
+    result = probe.permutation_test(vectors, [1, 0] * 6, trials=400, seed=1)
+    assert result["p_value"] > 0.05
+
+
+def test_one_prompt_keeps_one_id_through_gpu_jitter():
+    """A real session produced 20 distinct ids across 21 generations of one prompt, and the
+    hash was a candidate cause: it compared raw bytes, so any value crossing a rounding
+    boundary changed the id.
+
+    Quantising fixes it for jitter of the size a GPU actually produces (~1e-6 relative).
+    It cannot be fixed for ALL jitter — with 768 values some will always sit near a
+    boundary, whatever the quantum — so this asserts the realistic range and not more.
+    """
+    gen = torch.Generator().manual_seed(3)
+    cond = torch.randn(1, 77, 768, generator=gen)
+    for scale in (1e-7, 1e-6, 1e-5):
+        jittered = cond + torch.randn(cond.shape, generator=gen) * scale
+        assert probe.prompt_hash(cond) == probe.prompt_hash(jittered), scale
+
+
+def test_a_different_prompt_still_gets_a_different_id():
+    """The rounding must not be so coarse that two prompts collide — that would hide a
+    confound instead of measuring it."""
+    gen = torch.Generator().manual_seed(4)
+    a = torch.randn(1, 77, 768, generator=gen)
+    b = torch.randn(1, 77, 768, generator=gen)
+    assert probe.prompt_hash(a) != probe.prompt_hash(b)
+    assert probe.prompt_hash(a) == probe.prompt_hash(a.clone())

trajectory_probe.py

@@ -415,6 +415,14 @@ def load_all_logs():
     return out
 
 
+#: How coarsely a conditioning is rounded before hashing. Raw bytes answer "are these
+#: bit-identical", which is not the question: measured on a real session, ONE prompt
+#: produced 20 distinct ids across 21 generations, because GPU jitter moves values across
+#: rounding boundaries. Two decimals is far coarser than that jitter and far finer than the
+#: gap between two different prompts.
+HASH_QUANTUM = 100.0
+
+
 def prompt_hash(conditioning):
     """A stable id for "the same prompt", so the analysis can ask whether a separation is
     really about ratings rather than about which prompt was running."""
@@ -425,7 +433,8 @@ def prompt_hash(conditioning):
         c = conditioning.detach().float()
         while c.dim() > 1:
             c = c.mean(dim=0)
-        return md5(c.to(torch.float16).cpu().numpy().tobytes()).hexdigest()[:16]
+        # +0.0 so a -0.0 bin, which round() can produce, hashes as the 0.0 it equals.
+        return md5((torch.round(c * HASH_QUANTUM) + 0.0).cpu().numpy().tobytes()).hexdigest()[:16]
     except Exception:
         return None
 
@@ -439,8 +448,18 @@ def prompt_hash(conditioning):
 
 
 def _unit(vectors):
+    """Unit vectors, CENTRED on what the set has in common.
+
+    Runs of one prompt share most of their content -- measured on a real session, 86% of
+    each descriptor was the part every run had, and the run-specific remainder was ~14%.
+    Comparing the raw vectors therefore measures the shared part, and every distance comes
+    out near zero and near equal: 21 rated runs gave a mean cosine of 0.97 and no
+    separation at ANY bucket, including the late ones where steering demonstrably works.
+    Subtracting the mean leaves only what differs between runs, which is the only place a
+    rating difference could ever have been.
+    """
     m = torch.stack([v.detach().float().reshape(-1) for v in vectors])
-    return torch.nn.functional.normalize(m, dim=-1)
+    return torch.nn.functional.normalize(m - m.mean(dim=0, keepdim=True), dim=-1)
 
 
 def separation_statistic(descriptors, labels, groups=None):