diff --git a/fastvideo2/evidence/goldens/train-finetune-main/manifest.json b/fastvideo2/evidence/goldens/train-finetune-main/manifest.json new file mode 100644 index 000000000..f63049f15 --- /dev/null +++ b/fastvideo2/evidence/goldens/train-finetune-main/manifest.json @@ -0,0 +1,41 @@ +{ + "fastvideo_commit": "c459a1897899ffcec3be7765534d81000b9bb9c1", + "dataset": "wlsaidhi/crush-smol_processed_t2v", + "seed": 42, + "steps": 5, + "num_latent_t": 8, + "config": "1gpu bs1 accum1 lr5e-5 wd1e-4 betas(0.9,0.999) clip1.0 uniform-t cfg_rate0 dit_fp32 mixed_bf16 flow-match target=noise-latents flash_attn", + "torch": "2.12.0+cu130", + "gpu": "NVIDIA GB200", + "losses": [ + 0.1940414160490036, + 0.94484943151474, + 0.1001732274889946, + 0.9069435000419617, + 0.10525074601173401 + ], + "self_noise_runs": [ + [ + 0.1940414160490036, + 0.9445295333862305, + 0.10048552602529526, + 0.9095064997673035, + 0.10758557915687561 + ], + [ + 0.1940414160490036, + 0.9446913599967957, + 0.10070198774337769, + 0.9106038808822632, + 0.10818696022033691 + ], + [ + 0.1940414160490036, + 0.94484943151474, + 0.1001732274889946, + 0.9069435000419617, + 0.10525074601173401 + ] + ], + "self_noise_max": 0.0036603808403015137 +} diff --git a/fastvideo2/evidence/goldens/train-finetune-main/params.npz b/fastvideo2/evidence/goldens/train-finetune-main/params.npz new file mode 100644 index 000000000..f33a78aba Binary files /dev/null and b/fastvideo2/evidence/goldens/train-finetune-main/params.npz differ diff --git a/fastvideo2/evidence/goldens/train-finetune-main/step0.npz b/fastvideo2/evidence/goldens/train-finetune-main/step0.npz new file mode 100644 index 000000000..4f906037f Binary files /dev/null and b/fastvideo2/evidence/goldens/train-finetune-main/step0.npz differ diff --git a/fastvideo2/evidence/goldens/train-finetune-main/step1.npz b/fastvideo2/evidence/goldens/train-finetune-main/step1.npz new file mode 100644 index 000000000..e58644e04 Binary files /dev/null and b/fastvideo2/evidence/goldens/train-finetune-main/step1.npz differ diff --git a/fastvideo2/evidence/goldens/train-finetune-main/step2.npz b/fastvideo2/evidence/goldens/train-finetune-main/step2.npz new file mode 100644 index 000000000..c49ab2fa6 Binary files /dev/null and b/fastvideo2/evidence/goldens/train-finetune-main/step2.npz differ diff --git a/fastvideo2/evidence/goldens/train-finetune-main/step3.npz b/fastvideo2/evidence/goldens/train-finetune-main/step3.npz new file mode 100644 index 000000000..397b38fb9 Binary files /dev/null and b/fastvideo2/evidence/goldens/train-finetune-main/step3.npz differ diff --git a/fastvideo2/evidence/goldens/train-finetune-main/step4.npz b/fastvideo2/evidence/goldens/train-finetune-main/step4.npz new file mode 100644 index 000000000..19be91bdd Binary files /dev/null and b/fastvideo2/evidence/goldens/train-finetune-main/step4.npz differ diff --git a/fastvideo2/evidence/goldens/train-finetune-main/steps.json b/fastvideo2/evidence/goldens/train-finetune-main/steps.json new file mode 100644 index 000000000..81e48b26a --- /dev/null +++ b/fastvideo2/evidence/goldens/train-finetune-main/steps.json @@ -0,0 +1,82 @@ +[ + { + "caption": "A large metal cylinder is seen pressing down on a pile of colorful candies, flattening them as if they were under a hydraulic press. The candies are crushed and broken into small pieces, creating a mess on the table.", + "latents_hash": "93ae01bc47c0817b", + "embeds_hash": "53f384dfed477e28", + "noise_hash": "e1f135db4ff97ff8", + "noisy_hash": "6a0cabf3c2e98794", + "timesteps": [ + 118.0 + ], + "sigmas": [ + 0.1181640625 + ], + "pred_hash": "2903c217cc9279df", + "loss": 0.1940414160490036, + "grad_norm": 0.23514027893543243 + }, + { + "caption": "The video shows a colorful sponge being flattened as if it were under a hydraulic press, with the sponge being compressed and eventually flattened into a thin layer.", + "latents_hash": "b9d5abc838e815da", + "embeds_hash": "82c3f45fec89f678", + "noise_hash": "ea2e0427b4584180", + "noisy_hash": "5ff0bac18dd8e80e", + "timesteps": [ + 85.0 + ], + "sigmas": [ + 0.0849609375 + ], + "pred_hash": "688def082dcbfa60", + "loss": 0.94484943151474, + "grad_norm": 10.1902437210083 + }, + { + "caption": "The video shows a hydraulic press in action, flattening objects as if they were under a hydraulic press. The press is composed of a large, cylindrical metal cylinder with yellow and black stripes, and a metal base. The objects being flattened are two cylindrical blocks of cotton candy, one pink and one blue. The press is positioned on a metal table, and the background features a green wall with a yellow and red sign.", + "latents_hash": "840ddc5d78da4b7b", + "embeds_hash": "ffce0c1bf664fb3d", + "noise_hash": "2f512661cebc8a51", + "noisy_hash": "05b1746f31f8d83d", + "timesteps": [ + 618.0 + ], + "sigmas": [ + 0.6171875 + ], + "pred_hash": "963b9731aafbe2ea", + "loss": 0.1001732274889946, + "grad_norm": 0.523212730884552 + }, + { + "caption": "A watermelon wearing a helmet is crushed by a hydraulic press, causing it to flatten and burst open.", + "latents_hash": "c5d8005785df08ab", + "embeds_hash": "88eea058d58c26f7", + "noise_hash": "1022b2e68b615e5f", + "noisy_hash": "ee1750fb9fdb7094", + "timesteps": [ + 41.0 + ], + "sigmas": [ + 0.041015625 + ], + "pred_hash": "5fd1a77f3716b63b", + "loss": 0.9069435000419617, + "grad_norm": 3.6921098232269287 + }, + { + "caption": "The video shows a large, industrial press flattening objects as if they were under a hydraulic press. The press is shown in action, compressing a pile of pink objects into a pile of crumbs. The press is large and metallic, with a yellow and black striped pattern on its side. The background is a green wall with a yellow warning sign.", + "latents_hash": "57045eb31ae1e81a", + "embeds_hash": "71cee36d5ca3083d", + "noise_hash": "cdc1cbc4a1fed9ac", + "noisy_hash": "53ada1e537af47ac", + "timesteps": [ + 610.0 + ], + "sigmas": [ + 0.609375 + ], + "pred_hash": "315edc5acc47767a", + "loss": 0.10525074601173401, + "grad_norm": 0.9913093447685242 + } +] diff --git a/fastvideo2/evidence/ledger.jsonl b/fastvideo2/evidence/ledger.jsonl index 07f08f196..1da926fe8 100644 --- a/fastvideo2/evidence/ledger.jsonl +++ b/fastvideo2/evidence/ledger.jsonl @@ -157,3 +157,4 @@ {"ts": "2026-07-23T03:12:27Z", "gate": "anchor.fastwan-qad-main", "status": "pass", "model_id": "fastwan-qad-fp8-1.3b", "card_digest": "1c8e6f7d1380552d", "metrics": {"dit.bf16.t1000": 0.0, "dit.bf16.t757": 0.0, "dit.bf16.t522": 0.0, "dit.fp8.t1000": 0.0, "dit.fp8.t757": 0.0, "dit.fp8.t522": 0.0, "text.e2e": 0.0, "text.probe": 0.0, "e2e.final": 0.0, "e2e.step1.x": 0.0, "e2e.step2.x": 0.0}, "tolerances": {"all": 0.0}, "env": {"python": "3.12.13", "machine": "aarch64", "commit": "0b332ff", "torch": "2.12.0+cu130", "cuda": "13.0", "gpu": "NVIDIA GB200", "diffusers": "0.39.0", "transformers": "5.13.0", "attention": {"policy": "AUTO", "available": ["FLASH_ATTN", "SDPA"]}}, "detail": "goldens c459a1897"} {"ts": "2026-07-23T03:16:22Z", "gate": "anchor.fastwan-vsa-main", "status": "pass", "model_id": "fastwan-t2v-1.3b", "card_digest": "0bd5c7771e10ce44", "metrics": {"dit.vsa.t1000": 0.0, "dit.vsa.t757": 0.0, "dit.vsa.t522": 0.0, "text.e2e": 0.0, "text.probe": 0.0, "e2e.final": 0.0, "e2e.step1.x": 0.0, "e2e.step2.x": 0.0}, "tolerances": {"all": 0.0}, "env": {"python": "3.12.13", "machine": "aarch64", "commit": "0b332ff", "torch": "2.12.0+cu130", "cuda": "13.0", "gpu": "NVIDIA GB200", "diffusers": "0.39.0", "transformers": "5.13.0", "attention": {"policy": "AUTO", "available": ["FLASH_ATTN", "SDPA"]}}, "detail": "goldens c459a1897"} {"ts": "2026-07-23T11:20:04Z", "gate": "anchor.sfwan-main", "status": "pass", "model_id": "sfwan-t2v-1.3b", "card_digest": "9c9b4cc573606028", "metrics": {"text.e2e": 0.0, "e2e.final": 0.0, "fwd.hash_matches": 0.0}, "tolerances": {"all": 0.0}, "env": {"python": "3.12.13", "machine": "aarch64", "commit": "8b7285a", "torch": "2.12.0+cu130", "cuda": "13.0", "gpu": "NVIDIA GB200", "diffusers": "0.39.0", "transformers": "5.13.0", "attention": {"policy": "AUTO", "available": ["FLASH_ATTN", "SDPA"]}}, "detail": "goldens c459a1897"} +{"ts": "2026-07-23T11:47:02Z", "gate": "anchor.train-finetune-main", "status": "pass", "model_id": "wan2.1-t2v-1.3b", "card_digest": "7171d8729ae72ae4", "metrics": {"noisy.step0": 0.0, "pred.step0": 0.0, "loss.step0": 0.0, "gnorm.step0": 7.675588130950928e-05, "loss.step1": 5.221366882324219e-05, "gnorm.step1": 0.05505657196044922, "loss.step2": 0.0007446706295013428, "gnorm.step2": 0.006015121936798096, "loss.step3": 0.0025970935821533203, "gnorm.step3": 0.032111406326293945, "loss.step4": 0.0013115331530570984, "gnorm.step4": 0.0047234296798706055, "params.w0": 0.0, "params.w5": 0.00029868653199637143}, "tolerances": {"step0": 0.0, "later": "main self-noise band (see manifest)"}, "env": {"python": "3.12.13", "machine": "aarch64", "commit": "e14afd7", "torch": "2.12.0+cu130", "cuda": "13.0", "gpu": "NVIDIA GB200", "diffusers": "0.39.0", "transformers": "5.13.0", "attention": {"policy": "AUTO", "available": ["FLASH_ATTN", "SDPA"]}}, "detail": "goldens c459a1897"} diff --git a/fastvideo2/evidence/train_numerics_report.md b/fastvideo2/evidence/train_numerics_report.md new file mode 100644 index 000000000..924fde833 --- /dev/null +++ b/fastvideo2/evidence/train_numerics_report.md @@ -0,0 +1,37 @@ +# Training parity — wan2.1 finetune vs fastvideo-main (legacy stack) + +Gate `anchor.train-finetune-main`: PASS (2026-07-23, main @ c459a189). + +Config (goldens manifest): crush-smol processed parquet, 1 GPU, batch 1, +grad_accum 1, num_latent_t 8, lr 5e-5, wd 1e-4, betas (0.9,0.999), clip 1.0, +uniform timestep sampling, target = noise - latents, dit fp32 masters + bf16 +compute (FSDP2 world-1 mixed precision on main; explicit fp32-master/bf16- +compute chain in `fastvideo2/train/finetune.py`), cfg_rate 0, seed 42. + +| row | result | +|---|---| +| noisy/pred/loss @ step0 | **bitwise 0.0** | +| loss steps 1–4 | 5.2e-5 … 2.6e-3, inside the measured band | +| params.w0 | bitwise | +| gnorm / params.w5 | informational (grad-noise carriers) | + +Tolerance is MEASURED, not chosen: main-vs-main across three capture runs +drifts up to 3.66e-3 in per-step loss (flash-attention backward atomics); +the gate band is 1.5x that. Step-0 loss is exactly repeatable on main +(0.0 across runs) and we match it bitwise — i.e., the forward, input +construction, loss, and optimizer-chain math are exact; only the +nondeterministic backward separates runs, ours indistinguishable from +main's own reruns. + +Root cause found by the gate: main's `train_one_step` runs +`_normalize_dit_input` (VAE scaling) BETWEEN batch fetch and noise prep — +capturing latents at fetch time silently records the wrong tensors (3.03 +loss delta). Recorded post-normalization. + +Reuse invariant held: the trained module is the SAME `WanModelFV` the +serving path loads (training-mode forward proved bitwise against main's +FSDP2-wrapped port via pred.step0). + +Next gates (see fastvideo2/train/PORT_NOTES.md): dataloader order parity, +VSA training, DMD2 (legacy CFG convention), self-forcing (rollout via +WanCausalDMDLoop), attn-QAT/QAD, DiffusionNFT+VideoAlign.