[evidence]: VSA training parity — step0 bitwise, band from measured VSA self-noise

2026-07-23 04:59:54 -07:00
parent c73ccc75ed
commit afe9c077be
10 changed files with 140 additions and 0 deletions
@@ -0,0 +1,36 @@
+{
+  "fastvideo_commit": "c459a1897899ffcec3be7765534d81000b9bb9c1",
+  "dataset": "wlsaidhi/crush-smol_processed_t2v",
+  "mode": "vsa",
+  "model": "FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
+  "seed": 42,
+  "steps": 5,
+  "num_latent_t": 8,
+  "config": "1gpu bs1 accum1 lr5e-5 wd1e-4 betas(0.9,0.999) clip1.0 uniform-t cfg_rate0 dit_fp32 mixed_bf16 flow-match target=noise-latents flash_attn",
+  "torch": "2.12.0+cu130",
+  "gpu": "NVIDIA GB200",
+  "losses": [
+    0.2221531718969345,
+    0.7356714606285095,
+    0.10640689730644226,
+    1.1848068237304688,
+    0.3357445001602173
+  ],
+  "self_noise_runs": [
+    [
+      0.2221531718969345,
+      0.735809326171875,
+      0.10631868988275528,
+      1.3559304475784302,
+      0.3470917344093323
+    ],
+    [
+      0.2221531718969345,
+      0.7356714606285095,
+      0.10640689730644226,
+      1.1848068237304688,
+      0.3357445001602173
+    ]
+  ],
+  "self_noise_max": 0.17112362384796143
+}
@@ -0,0 +1,87 @@
+[
+ {
+  "caption": "A large metal cylinder is seen pressing down on a pile of colorful candies, flattening them as if they were under a hydraulic press. The candies are crushed and broken into small pieces, creating a mess on the table.",
+  "latents_hash": "93ae01bc47c0817b",
+  "embeds_hash": "53f384dfed477e28",
+  "noise_hash": "e1f135db4ff97ff8",
+  "noisy_hash": "6a0cabf3c2e98794",
+  "timesteps": [
+   118.0
+  ],
+  "sigmas": [
+   0.1181640625
+  ],
+  "pred_hash": "49493b9aaad874ad",
+  "loss": 0.2221531718969345,
+  "grad_norm": 1.365469217300415,
+  "vsa_sparsity": 0.2
+ },
+ {
+  "caption": "The video shows a colorful sponge being flattened as if it were under a hydraulic press, with the sponge being compressed and eventually flattened into a thin layer.",
+  "latents_hash": "b9d5abc838e815da",
+  "embeds_hash": "82c3f45fec89f678",
+  "noise_hash": "ea2e0427b4584180",
+  "noisy_hash": "5ff0bac18dd8e80e",
+  "timesteps": [
+   85.0
+  ],
+  "sigmas": [
+   0.0849609375
+  ],
+  "pred_hash": "1a7294ecc5eff0d6",
+  "loss": 0.7356714606285095,
+  "grad_norm": 8.022859573364258,
+  "vsa_sparsity": 0.4
+ },
+ {
+  "caption": "The video shows a hydraulic press in action, flattening objects as if they were under a hydraulic press. The press is composed of a large, cylindrical metal cylinder with yellow and black stripes, and a metal base. The objects being flattened are two cylindrical blocks of cotton candy, one pink and one blue. The press is positioned on a metal table, and the background features a green wall with a yellow and red sign.",
+  "latents_hash": "840ddc5d78da4b7b",
+  "embeds_hash": "ffce0c1bf664fb3d",
+  "noise_hash": "2f512661cebc8a51",
+  "noisy_hash": "05b1746f31f8d83d",
+  "timesteps": [
+   618.0
+  ],
+  "sigmas": [
+   0.6171875
+  ],
+  "pred_hash": "802548648224bc30",
+  "loss": 0.10640689730644226,
+  "grad_norm": 1.5231013298034668,
+  "vsa_sparsity": 0.6000000000000001
+ },
+ {
+  "caption": "A watermelon wearing a helmet is crushed by a hydraulic press, causing it to flatten and burst open.",
+  "latents_hash": "c5d8005785df08ab",
+  "embeds_hash": "88eea058d58c26f7",
+  "noise_hash": "1022b2e68b615e5f",
+  "noisy_hash": "ee1750fb9fdb7094",
+  "timesteps": [
+   41.0
+  ],
+  "sigmas": [
+   0.041015625
+  ],
+  "pred_hash": "e5afd85258757532",
+  "loss": 1.1848068237304688,
+  "grad_norm": 16.469207763671875,
+  "vsa_sparsity": 0.8
+ },
+ {
+  "caption": "The video shows a large, industrial press flattening objects as if they were under a hydraulic press. The press is shown in action, compressing a pile of pink objects into a pile of crumbs. The press is large and metallic, with a yellow and black striped pattern on its side. The background is a green wall with a yellow warning sign.",
+  "latents_hash": "57045eb31ae1e81a",
+  "embeds_hash": "71cee36d5ca3083d",
+  "noise_hash": "cdc1cbc4a1fed9ac",
+  "noisy_hash": "53ada1e537af47ac",
+  "timesteps": [
+   610.0
+  ],
+  "sigmas": [
+   0.609375
+  ],
+  "pred_hash": "880902f3c1b8e159",
+  "loss": 0.3357445001602173,
+  "grad_norm": 8.711103439331055,
+  "vsa_sparsity": 0.8
+ }
+]
@@ -158,3 +158,4 @@
 {"ts": "2026-07-23T03:16:22Z", "gate": "anchor.fastwan-vsa-main", "status": "pass", "model_id": "fastwan-t2v-1.3b", "card_digest": "0bd5c7771e10ce44", "metrics": {"dit.vsa.t1000": 0.0, "dit.vsa.t757": 0.0, "dit.vsa.t522": 0.0, "text.e2e": 0.0, "text.probe": 0.0, "e2e.final": 0.0, "e2e.step1.x": 0.0, "e2e.step2.x": 0.0}, "tolerances": {"all": 0.0}, "env": {"python": "3.12.13", "machine": "aarch64", "commit": "0b332ff", "torch": "2.12.0+cu130", "cuda": "13.0", "gpu": "NVIDIA GB200", "diffusers": "0.39.0", "transformers": "5.13.0", "attention": {"policy": "AUTO", "available": ["FLASH_ATTN", "SDPA"]}}, "detail": "goldens c459a1897"}
 {"ts": "2026-07-23T11:20:04Z", "gate": "anchor.sfwan-main", "status": "pass", "model_id": "sfwan-t2v-1.3b", "card_digest": "9c9b4cc573606028", "metrics": {"text.e2e": 0.0, "e2e.final": 0.0, "fwd.hash_matches": 0.0}, "tolerances": {"all": 0.0}, "env": {"python": "3.12.13", "machine": "aarch64", "commit": "8b7285a", "torch": "2.12.0+cu130", "cuda": "13.0", "gpu": "NVIDIA GB200", "diffusers": "0.39.0", "transformers": "5.13.0", "attention": {"policy": "AUTO", "available": ["FLASH_ATTN", "SDPA"]}}, "detail": "goldens c459a1897"}
 {"ts": "2026-07-23T11:47:02Z", "gate": "anchor.train-finetune-main", "status": "pass", "model_id": "wan2.1-t2v-1.3b", "card_digest": "7171d8729ae72ae4", "metrics": {"noisy.step0": 0.0, "pred.step0": 0.0, "loss.step0": 0.0, "gnorm.step0": 7.675588130950928e-05, "loss.step1": 5.221366882324219e-05, "gnorm.step1": 0.05505657196044922, "loss.step2": 0.0007446706295013428, "gnorm.step2": 0.006015121936798096, "loss.step3": 0.0025970935821533203, "gnorm.step3": 0.032111406326293945, "loss.step4": 0.0013115331530570984, "gnorm.step4": 0.0047234296798706055, "params.w0": 0.0, "params.w5": 0.00029868653199637143}, "tolerances": {"step0": 0.0, "later": "main self-noise band (see manifest)"}, "env": {"python": "3.12.13", "machine": "aarch64", "commit": "e14afd7", "torch": "2.12.0+cu130", "cuda": "13.0", "gpu": "NVIDIA GB200", "diffusers": "0.39.0", "transformers": "5.13.0", "attention": {"policy": "AUTO", "available": ["FLASH_ATTN", "SDPA"]}}, "detail": "goldens c459a1897"}
+{"ts": "2026-07-23T11:58:27Z", "gate": "anchor.train-vsa-main", "status": "pass", "model_id": "fastwan-t2v-1.3b", "card_digest": "0bd5c7771e10ce44", "metrics": {"noisy.step0": 0.0, "pred.step0": 0.0, "loss.step0": 0.0, "gnorm.step0": 7.104873657226562e-05, "loss.step1": 0.00040608644485473633, "gnorm.step1": 0.014156341552734375, "loss.step2": 5.901604890823364e-05, "gnorm.step2": 0.18469977378845215, "loss.step3": 0.08010447025299072, "gnorm.step3": 0.9372882843017578, "loss.step4": 0.005944937467575073, "gnorm.step4": 0.15396595001220703, "params.w0": 0.0, "params.w5": 0.0010486775342362664}, "tolerances": {"step0": 0.0, "later": "main self-noise band (see manifest)"}, "env": {"python": "3.12.13", "machine": "aarch64", "commit": "c73ccc7", "torch": "2.12.0+cu130", "cuda": "13.0", "gpu": "NVIDIA GB200", "diffusers": "0.39.0", "transformers": "5.13.0", "attention": {"policy": "AUTO", "available": ["FLASH_ATTN", "SDPA"]}}, "detail": "goldens c459a1897"}
@@ -35,3 +35,19 @@ FSDP2-wrapped port via pred.step0).
 Next gates (see fastvideo2/train/PORT_NOTES.md): dataloader order parity,
 VSA training, DMD2 (legacy CFG convention), self-forcing (rollout via
 WanCausalDMDLoop), attn-QAT/QAD, DiffusionNFT+VideoAlign.
+
+## VSA training — added 2026-07-23
+
+Gate `anchor.train-vsa-main`: PASS. Config: FastWan2.1 checkpoint (carries
+deterministic `to_gate_compress` weights — the base-checkpoint variant
+random-inits them and needs an init-RNG gate later), VIDEO_SPARSE_ATTN,
+sparsity ramp 0.2→0.8 over 5 steps (rate 0.2, interval 1), otherwise the
+finetune config.
+
+step0 noisy/pred/loss **bitwise 0.0** — the vendored VSA kernel path
+(tile/untile + `fastvideo_kernel.video_sparse_attn` + per-step ramped
+metadata) is exact under training. Later steps sit WELL inside main's own
+measured self-noise: the block-sparse backward at 0.8 sparsity drifts main
+itself by 0.171 in per-step loss across reruns (vs flash's 3.7e-3); our
+worst diff is 0.080 — about half of main's own run-to-run spread.
+Band = 1.5x measured, from the goldens manifest.