From 9212f4f218dce416c66176c8f2188e73c2c1a41d Mon Sep 17 00:00:00 2001 From: William Lin Date: Tue, 25 Aug 2026 21:26:25 -0700 Subject: [PATCH] [ci] make GPU validation change-aware (#1747) --- .agents/skills/ci-runner/SKILL.md | 99 ++ .../skills/reseed-ssim-references/SKILL.md | 9 +- .agents/skills/seed-ssim-references/SKILL.md | 6 +- .buildkite/pipeline.yml | 976 ++++++++---------- .buildkite/scripts/lanes/api_server.sh | 5 + .buildkite/scripts/lanes/distillation_dmd.sh | 5 + .buildkite/scripts/lanes/dreamverse.sh | 87 ++ .buildkite/scripts/lanes/encoder.sh | 5 + .buildkite/scripts/lanes/eval.sh | 5 + .buildkite/scripts/lanes/golden_gate.sh | 35 + .buildkite/scripts/lanes/inference_lora.sh | 5 + .buildkite/scripts/lanes/inference_vmoba.sh | 5 + .buildkite/scripts/lanes/kernel_tests.sh | 5 + .buildkite/scripts/lanes/lora_extraction.sh | 5 + .buildkite/scripts/lanes/performance.sh | 52 + .buildkite/scripts/lanes/self_forcing.sh | 6 + .buildkite/scripts/lanes/ssim.sh | 40 + .buildkite/scripts/lanes/train_framework.sh | 5 + .buildkite/scripts/lanes/training.sh | 6 + .buildkite/scripts/lanes/training_lora.sh | 6 + .buildkite/scripts/lanes/training_vsa.sh | 6 + .buildkite/scripts/lanes/transformer.sh | 5 + .buildkite/scripts/lanes/vae.sh | 5 + .buildkite/scripts/pr_test.sh | 13 + .buildkite/scripts/unit_test.sh | 22 + .github/PULL_REQUEST_TEMPLATE.md | 4 +- .github/scripts/gate_full_suite.sh | 20 +- .github/scripts/plan_merge_ci.py | 570 ++++++++++ .github/scripts/test_gate_full_suite.sh | 2 +- .github/workflows/_template-build-image.yml | 1 + .github/workflows/ci-aggregate-status.yml | 64 +- .github/workflows/ci-scheduled-ssim.yml | 44 + .github/workflows/ci-slash-commands.yml | 56 +- .github/workflows/ci-trigger-full-suite.yml | 79 +- .github/workflows/community-welcome.yml | 8 +- .github/workflows/infra-build-image.yml | 27 + docker/Dockerfile | 38 +- docs/contributing/ci_architecture.md | 284 +++-- docs/contributing/performance_benchmarks.md | 18 +- docs/contributing/pull_requests.md | 18 +- docs/contributing/testing.md | 82 +- .../inference_schema_parity_inventory.yaml | 28 + examples/inference/basic/README.md | 8 +- fastvideo/tests/AGENTS.md | 18 +- .../test_attn_qat_infer_capability_gate.py | 38 +- .../tests/contract/test_ci_test_collection.py | 206 +++- .../tests/contract/test_merge_ci_plan.py | 145 +++ .../tests/contract/test_modal_fa4_policy.py | 40 +- .../tests/contract/test_ssim_ci_runner.py | 93 ++ fastvideo/tests/encoders/test_clip_encoder.py | 14 +- fastvideo/tests/encoders/test_hyt5_encoder.py | 4 +- .../tests/encoders/test_llama_encoder.py | 4 +- .../tests/encoders/test_qwen2_5_encoder.py | 4 +- .../tests/encoders/test_siglip_encoder.py | 4 +- fastvideo/tests/encoders/test_t5_encoder.py | 4 +- .../tests/golden_gate/test_kandinsky5.py | 10 +- .../lora/test_lora_inference_similarity.py | 4 +- fastvideo/tests/modal/kernel_build_cache.py | 14 +- fastvideo/tests/modal/pr_test.py | 40 +- fastvideo/tests/modal/ssim_test.py | 6 + .../tests/modal/test_kernel_build_cache.py | 57 +- fastvideo/tests/modal/test_pr_test.py | 60 +- fastvideo/tests/ssim/AGENTS.md | 10 +- fastvideo/tests/ssim/README.md | 22 +- fastvideo/tests/ssim/ci_runner.py | 289 ++++++ .../tests/train/methods/grad_norm_refs.json | 2 +- .../train/methods/grad_norm_regression.py | 12 +- .../methods/test_matrixgame2_finetune.py | 2 +- .../train/methods/test_wan_causal_dfsft.py | 2 +- .../tests/train/methods/test_wan_finetune.py | 2 +- .../models/test_wan_override_contract.py | 26 + .../training/VSA/test_training_loss_VSA.py | 6 +- .../tests/training/Vanilla/mfu_calculation.py | 8 +- .../training/Vanilla/test_training_loss.py | 11 +- .../training/distill/test_distill_dmd.py | 4 +- .../tests/training/lora/test_lora_training.py | 9 +- .../self-forcing/test_self_forcing.py | 4 +- fastvideo/tests/training/test_trackers.py | 32 + fastvideo/tests/transformers/test_cosmos.py | 4 +- .../tests/transformers/test_cosmos2_5.py | 4 +- .../transformers/test_hunyuangamecraft.py | 15 +- fastvideo/tests/transformers/test_hyworld.py | 13 +- fastvideo/tests/transformers/test_wanvideo.py | 4 +- fastvideo/tests/vaes/test_hunyuan15_vae.py | 4 +- fastvideo/tests/vaes/test_wan_vae.py | 4 +- fastvideo/train/models/hunyuan/hunyuan.py | 4 + fastvideo/train/models/longcat/longcat.py | 8 + .../train/models/matrixgame2/matrixgame2.py | 8 + fastvideo/training/trackers.py | 33 +- 89 files changed, 3139 insertions(+), 937 deletions(-) create mode 100644 .agents/skills/ci-runner/SKILL.md create mode 100755 .buildkite/scripts/lanes/api_server.sh create mode 100755 .buildkite/scripts/lanes/distillation_dmd.sh create mode 100755 .buildkite/scripts/lanes/dreamverse.sh create mode 100755 .buildkite/scripts/lanes/encoder.sh create mode 100755 .buildkite/scripts/lanes/eval.sh create mode 100755 .buildkite/scripts/lanes/golden_gate.sh create mode 100755 .buildkite/scripts/lanes/inference_lora.sh create mode 100755 .buildkite/scripts/lanes/inference_vmoba.sh create mode 100755 .buildkite/scripts/lanes/kernel_tests.sh create mode 100755 .buildkite/scripts/lanes/lora_extraction.sh create mode 100755 .buildkite/scripts/lanes/performance.sh create mode 100755 .buildkite/scripts/lanes/self_forcing.sh create mode 100755 .buildkite/scripts/lanes/ssim.sh create mode 100755 .buildkite/scripts/lanes/train_framework.sh create mode 100755 .buildkite/scripts/lanes/training.sh create mode 100755 .buildkite/scripts/lanes/training_lora.sh create mode 100755 .buildkite/scripts/lanes/training_vsa.sh create mode 100755 .buildkite/scripts/lanes/transformer.sh create mode 100755 .buildkite/scripts/lanes/vae.sh create mode 100644 .buildkite/scripts/unit_test.sh create mode 100644 .github/scripts/plan_merge_ci.py create mode 100644 .github/workflows/ci-scheduled-ssim.yml create mode 100644 fastvideo/tests/contract/test_merge_ci_plan.py create mode 100644 fastvideo/tests/contract/test_ssim_ci_runner.py create mode 100644 fastvideo/tests/ssim/ci_runner.py create mode 100644 fastvideo/tests/train/models/test_wan_override_contract.py diff --git a/.agents/skills/ci-runner/SKILL.md b/.agents/skills/ci-runner/SKILL.md new file mode 100644 index 000000000..258107ac6 --- /dev/null +++ b/.agents/skills/ci-runner/SKILL.md @@ -0,0 +1,99 @@ +--- +name: ci-runner +description: Work on FastVideo's Slurm-only, change-aware GPU CI lanes, static Buildkite graph, trusted ci-runner policy, lane scripts, and GB200 validation. +--- + +# Slinky Slurm CI lanes + +FastVideo's `ci-runner` Buildkite queue is the control plane for all active +GPU CI. A private host-owned dispatcher leases GPUs from the Slinky Slurm tray +and runs the immutable PR SHA inside an isolated Enroot container. Buildkite +pipeline upload and Slurm submission occur on the login plane; every test +payload executes on Slurm compute. + +The files under `fastvideo/tests/modal/` and `.buildkite/scripts/pr_test.sh` +are dormant rollback code. Never add an active Buildkite or slash-command +route to them. `pr_test.sh` must continue to reject Buildkite invocations. + +The private operator bundle is deliberately outside this repository because +it contains site paths and credentials. See +`docs/contributing/ci_architecture.md`; this skill covers the repository half +and the coordination contract with that bundle. + +## Invariants + +- `.buildkite/pipeline.yml` contains exactly one static step for every active + GPU lane. Each step pins a unique key and label, a 90-minute timeout, the + trusted `/opt/fastvideo-ci-runner/run-ci` command (`run-unit` is the one + compatibility wrapper), step-level internal `TEST_TYPE`, and + `queue: "ci-runner"`. +- Active CI contains no `pr_test.sh` command, Modal invocation, default queue, + Buildkite plugin, `soft_fail`, or job-controlled artifact glob. +- The six Fastcheck lanes use `:microscope:` labels. Full-Suite-only lanes use + `:test_tube:` or `:bar_chart:` so direct reruns update the right aggregate. +- SSIM and vanilla training request all four GPUs. Keep both in the + `fastvideo/slinky/whole-tray` Buildkite concurrency group with a limit of one + so the second job does not consume an agent or command timeout while waiting + for the same tray. +- `/test full` schedules all twenty lanes. `/merge`, `ready`, and new pushes to + ready PRs use the trusted base-branch planner in + `.github/scripts/plan_merge_ci.py`: automatic Fastcheck remains the universal + six-lane baseline, and the merge build adds only path-relevant integration + lanes. Unknown source/build paths fail closed to all fourteen additive lanes. + The trusted uploader still normalizes and validates the complete static graph + before Buildkite evaluates its plan conditions. +- Focused merge builds may pass allowlisted golden-gate and SSIM test basenames. + The private host validates the lane plan and basenames before staging them, + and the in-container scripts validate them again. Direct `/test ssim`, + explicit `/test full`, and the weekly main-branch schedule run the complete + SSIM matrix. +- The trusted uploader serves exactly three entry pipelines: + `pr-fastcheck` for automatic PR builds, `ci` for slash-command/ready-label + API builds, and `fastvideo-performance-lane` for the weekly schedule. Keep + incoming GitHub webhook processing disabled on `ci` so it cannot duplicate + `pr-fastcheck` on every PR update. +- Test payloads live in `.buildkite/scripts/unit_test.sh` or executable + `.buildkite/scripts/lanes/.sh`. Backend policy (GPU count, extras, + secrets, kernel build, artifacts) stays in the agent-owned lane table. +- Tests must preserve an inherited `MASTER_PORT`. Packed containers share the + tray network namespace, so the private runner assigns a distinct port range + per GPU lease and the SSIM scheduler assigns task offsets within its range. +- The ARM64 runner image includes the pinned FA4 CuTe overlay validated on + GB200. Keep SSIM at `FASTVIDEO_FA4=1` because its references were seeded with + FA4; keep lanes with FA2 baselines at `FASTVIDEO_FA4=0`. A runner image change + must revalidate both the FA4 import and an actual GB200 forward kernel. +- `fastvideo/tests/ssim/ci_runner.py` is the active four-GPU SSIM scheduler. + New SSIM files are discovered through `REQUIRED_GPUS` and + `*_MODEL_TO_PARAMS`; do not wire them through the dormant Modal scheduler. +- The host policy fail-closes unknown tuples. A repository-side lane change is + inert until the operator updates the private lane table and uploader policy + in the same rollout. + +## Adding or changing a lane + +1. Read the closest `AGENTS.md` and the domain-specific testing guide. +2. Add or update the executable lane payload under `.buildkite/scripts/`. + Keep it deterministic and free of host-specific paths or credential fetches. +3. Add the static pipeline step and canonical `/test ` mapping. Keep the + `-ci` alias only when compatibility requires it. +4. Add its source/test path ownership to `.github/scripts/plan_merge_ci.py`. + Prefer the narrowest correctness-preserving lane set; leave unknown paths + fail-closed. Extend `fastvideo/tests/contract/test_ci_test_collection.py`, + `test_merge_ci_plan.py`, and focused CPU-only scheduler/policy tests. +5. Coordinate the private lane row: GPU count (1-4), wall time, script, scope + pairs, step key, command, HF cache/token, tracking mode, extras, attention + backend policy, kernel policy, and artifact relay. Active training lanes + keep W&B offline and do not stage a W&B credential. +6. Update the trusted pipeline-uploader schema. A mismatch must reject the + pipeline rather than silently skip a lane. +7. Run `pre-commit run --files `, the planner's representative + diff matrix, contract tests, private driver tests, and a real GB200 canary. + Multi-GPU, hardware-reference, training, performance, and SSIM changes need + their own target-hardware evidence. + +## Rollback + +Rollback the Slurm routing/configuration change or pause the `ci-runner` queue. +Do not silently reactivate Modal. A manual Modal experiment requires the +explicit local opt-in documented in `ci_architecture.md`; returning it to +production CI needs a separate reviewed decision. diff --git a/.agents/skills/reseed-ssim-references/SKILL.md b/.agents/skills/reseed-ssim-references/SKILL.md index 99eaeb831..d5f48ab4f 100644 --- a/.agents/skills/reseed-ssim-references/SKILL.md +++ b/.agents/skills/reseed-ssim-references/SKILL.md @@ -1,6 +1,6 @@ --- name: reseed-ssim-references -description: Re-seed HF reference videos for a single existing SSIM test on Modal L40S. Always backs up current refs locally first, regenerates on Modal, pauses for the user to eyeball before-vs-after quality, then overwrites the targeted `` subtree on `FastVideo/ssim-reference-videos` with `--force`. Use when an intentional code change (model port fix, attention backend swap, kernel upgrade, hyperparameter change) has invalidated existing refs and they need to be regenerated. Pairs with `seed-ssim-references`, which is for first-time seeding only. +description: Re-seed HF reference videos for a single existing SSIM test on Modal L40S. Always backs up current refs locally first, regenerates on Modal, pauses for the user to eyeball before-vs-after quality, then overwrites the targeted model subtree on `FastVideo/ssim-reference-videos` with `--force`. Use when an intentional code change (model port fix, attention backend swap, kernel upgrade, hyperparameter change) has invalidated existing refs and they need to be regenerated. Pairs with `seed-ssim-references`, which is for first-time seeding only. --- # Re-seed SSIM Reference Videos @@ -13,7 +13,7 @@ on HF — the old refs are overwritten — so the skill always: 1. Confirms intent with a one-liner the user has to type. 2. Downloads the existing refs as a local, timestamped backup. -3. Regenerates on Modal L40S (same code path that CI uses). +3. Regenerates through the manual legacy Modal L40S maintenance path. 4. Pauses for a side-by-side eyeball of backup vs new mp4s. 5. Uploads with `--force`, scoped to the single `--model-id`. 6. Reminds the user to keep the backup until the PR lands. @@ -51,8 +51,9 @@ harder to recover from than failing closed. Hardcoded: -- Modal GPU: **L40S** (matches CI; re-seeding from another SKU produces refs - that L40S CI cannot match). +- Modal GPU: **L40S**. This is a manual reference-maintenance target, not the + active Slurm CI compute path; changing the SKU also changes the historical + `L40S_reference_videos` contract. - Quality tier: **`default`**. `full_quality` is a separate, deliberate operation. - HF repo: `FastVideo/ssim-reference-videos` (override via diff --git a/.agents/skills/seed-ssim-references/SKILL.md b/.agents/skills/seed-ssim-references/SKILL.md index 0b1a2f78a..ca2266ae1 100644 --- a/.agents/skills/seed-ssim-references/SKILL.md +++ b/.agents/skills/seed-ssim-references/SKILL.md @@ -35,7 +35,8 @@ The skill is run **manually**, once per new test. Before invoking it, the user has already sanity-tested the new test locally — it launches `VideoGenerator` and writes an artefact without crashing (the missing-reference assertion at the end is expected). The skill does not re-test locally; it goes straight -to Modal L40S (which is what CI uses). +to the manual legacy Modal L40S reference-maintenance target. Active CI runs +on the Slinky Slurm cluster and only consumes the resulting references. ## When to use @@ -61,7 +62,8 @@ Prompt the user for it if they didn't supply it. Everything else is fixed: -- Modal runner GPU: **L40S** (hardcoded in `fastvideo/tests/modal/ssim_test.py`). +- Modal maintenance GPU: **L40S** (hardcoded in + `fastvideo/tests/modal/ssim_test.py`; this is not the active CI compute path). - Device folder: `L40S_reference_videos`. - Quality tier: `default` (the tier CI runs). The `full_quality` tier is not seeded by this skill. diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index fd7d59371..bf2e4500f 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,7 +1,8 @@ env: IMAGE_VERSION: "py3.12-latest" BUILDKITE_CLEAN_CHECKOUT: true - # Buildkite only launches Modal; remote jobs initialize their own submodules. + # Slurm workers clone the immutable commit and initialize submodules inside + # their isolated container. The Buildkite login-plane checkout is a no-op. BUILDKITE_GIT_SUBMODULES: false notify: @@ -10,539 +11,458 @@ notify: if: build.env("TEST_SCOPE") == "fastcheck" || build.env("TEST_SCOPE") == null - github_commit_status: context: "full-suite-passed" - if: build.env("TEST_SCOPE") == "full" + if: build.env("TEST_SCOPE") == "full" || build.env("TEST_SCOPE") == "merge" - github_commit_status: context: "direct-test-completed" if: build.env("TEST_SCOPE") == "direct" + - github_commit_status: + context: "scheduled-ssim-passed" + if: build.env("TEST_SCOPE") == "scheduled" +# This is the complete active GPU CI surface. Every command is a trusted host +# dispatcher, and every test payload executes inside the Slinky Slurm tray. +# fastvideo/tests/modal remains available only for an explicit manual rollback; +# no active pipeline or slash-command route invokes it. steps: - # ============================================================ - # Direct test: triggered by /test slash command. - # Labels match fastcheck/full-suite counterparts so the GitHub - # check status overwrites the original failed check. - # Only ONE step executes per build (gated by TEST_TYPE). - # ============================================================ + - label: ":microscope: Encoder Tests" + key: "encoder" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,encoder,/) || + build.env("TEST_SCOPE") == "fastcheck" || + build.env("TEST_SCOPE") == null || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "encoder" || build.env("TEST_TYPE") == "encoder_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + env: + TEST_TYPE: "encoder_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + agents: + queue: "ci-runner" - # --- Fastcheck-scope direct tests --- - - label: ":microscope: Encoder Tests" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "encoder" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - agents: - queue: "default" - - label: ":microscope: VAE Tests" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "vae" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - agents: - queue: "default" - - label: ":microscope: Transformer Tests" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "transformer" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - agents: - queue: "default" - - label: ":microscope: Kernel Tests" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "kernel_tests" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - agents: - queue: "default" - - label: ":vertical_traffic_light: Golden-Gate Tests" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "golden_gate" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - agents: - queue: "default" - - label: ":microscope: Unit Tests" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "unit_test" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - agents: - queue: "default" - - label: ":microscope: DreamVerse App Tests" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "dreamverse_app" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - agents: - queue: "default" + - label: ":microscope: VAE Tests" + key: "vae" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,vae,/) || + build.env("TEST_SCOPE") == "fastcheck" || + build.env("TEST_SCOPE") == null || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "vae" || build.env("TEST_TYPE") == "vae_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + env: + TEST_TYPE: "vae_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + agents: + queue: "ci-runner" - # --- Full-suite-scope direct tests --- - - label: ":bar_chart: SSIM Tests" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "ssim" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - - exit_status: 1 - limit: 2 - agents: - queue: "default" - - label: ":test_tube: LoRA Inference Tests" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "inference_lora" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - agents: - queue: "default" - - label: ":test_tube: LoRA Extraction Tests" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "lora_extraction" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - agents: - queue: "default" - - label: ":test_tube: Training Tests" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "training" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - agents: - queue: "default" - - label: ":test_tube: Distillation DMD Tests" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "distillation_dmd" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - agents: - queue: "default" - - label: ":test_tube: Self-Forcing Tests" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "self_forcing" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - agents: - queue: "default" - - label: ":test_tube: LoRA Training Tests" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "training_lora" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - - exit_status: 1 - limit: 2 - agents: - queue: "default" - - label: ":test_tube: Training Tests VSA" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "training_vsa" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - - exit_status: 1 - limit: 2 - agents: - queue: "default" - - label: ":test_tube: Inference Tests VMoBA" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "inference_vmoba" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - agents: - queue: "default" - - label: ":test_tube: Performance Tests" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "performance" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - agents: - queue: "default" - - label: ":test_tube: API Server Tests" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "api_server" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - agents: - queue: "default" - - label: ":test_tube: Train Framework Tests" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "train_framework" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - agents: - queue: "default" - - label: ":test_tube: Eval Metrics Tests" - if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "eval" - command: "timeout 90m .buildkite/scripts/pr_test.sh" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - agents: - queue: "default" + - label: ":microscope: Transformer Tests" + key: "transformer" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,transformer,/) || + build.env("TEST_SCOPE") == "fastcheck" || + build.env("TEST_SCOPE") == null || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "transformer" || build.env("TEST_TYPE") == "transformer_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + env: + TEST_TYPE: "transformer_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + agents: + queue: "ci-runner" - # ============================================================ - # Fastcheck: Runs on every PR (~10-15 min parallel) - # Core component validation: encoders, VAEs, transformers, - # CUDA kernels, and unit tests. - # ============================================================ - - label: "Trigger Fastcheck" - if: build.env("TEST_SCOPE") == "fastcheck" || build.env("TEST_SCOPE") == null - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - plugins: - - monorepo-diff#v1.4.0: - diff: 'git fetch origin "${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" && git diff --name-only "origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}...HEAD"' - watch: - - path: - - "fastvideo/models/encoders/**" - - "fastvideo/models/loader/**" - - "fastvideo/tests/encoders/**" - - "pyproject.toml" - - "docker/Dockerfile" - config: - command: "timeout 20m .buildkite/scripts/pr_test.sh" - label: ":microscope: Encoder Tests" - env: - - TEST_TYPE=encoder - agents: - queue: "default" - - path: - - "fastvideo/models/vaes/**" - - "fastvideo/models/loader/**" - - "fastvideo/tests/vaes/**" - - "pyproject.toml" - - "docker/Dockerfile" - config: - command: "timeout 20m .buildkite/scripts/pr_test.sh" - label: ":microscope: VAE Tests" - env: - - TEST_TYPE=vae - agents: - queue: "default" - - path: - - "fastvideo/models/dits/**" - - "fastvideo/models/loader/**" - - "fastvideo/tests/transformers/**" - - "fastvideo/layers/**" - - "fastvideo/attention/**" - - "pyproject.toml" - - "docker/Dockerfile" - config: - command: "timeout 15m .buildkite/scripts/pr_test.sh" - label: ":microscope: Transformer Tests" - env: - - TEST_TYPE=transformer - agents: - queue: "default" - - path: - - "fastvideo-kernel/**" - - "pyproject.toml" - - "docker/Dockerfile" - config: - command: "timeout 15m .buildkite/scripts/pr_test.sh" - label: ":microscope: Kernel Tests" - env: - - TEST_TYPE=kernel_tests - agents: - queue: "default" - - path: - - "fastvideo/**" - - ".buildkite/**" - - ".github/**" - - "pyproject.toml" - - "docker/Dockerfile" - config: - command: "timeout 15m .buildkite/scripts/pr_test.sh" - label: ":microscope: Unit Tests" - env: - - TEST_TYPE=unit_test - agents: - queue: "default" - - path: - - "apps/dreamverse/**" - - "pyproject.toml" - config: - command: "timeout 30m .buildkite/scripts/pr_test.sh" - label: ":microscope: DreamVerse App Tests" - env: - - TEST_TYPE=dreamverse_app - agents: - queue: "default" + - label: ":microscope: Kernel Tests" + key: "kernel-tests" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,kernel-tests,/) || + build.env("TEST_SCOPE") == "fastcheck" || + build.env("TEST_SCOPE") == null || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "kernel_tests" || build.env("TEST_TYPE") == "kernel_tests_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + env: + TEST_TYPE: "kernel_tests_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + agents: + queue: "ci-runner" - # ============================================================ - # Full Suite: Runs when TEST_SCOPE=full - # Triggered by adding the 'ready' label (via ci-trigger-full-suite.yml) - # or on-demand via /test full slash command. - # Includes integration tests, SSIM regression, training pipelines, - # and performance benchmarks. - # ============================================================ - - label: "Trigger Full Suite" - if: build.env("TEST_SCOPE") == "full" - retry: - automatic: - - exit_status: 128 - limit: 3 - - exit_status: -1 - limit: 2 - plugins: - - monorepo-diff#v1.4.0: - diff: 'git fetch origin "${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" && git diff --name-only "origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}...HEAD"' - watch: - - path: - - "fastvideo/**/*.py" - - "pyproject.toml" - - "docker/Dockerfile" - config: - command: "timeout 90m .buildkite/scripts/pr_test.sh" - label: ":bar_chart: SSIM Tests" - env: - - TEST_TYPE=ssim - retry: - automatic: - - exit_status: 1 - limit: 2 - agents: - queue: "default" - - path: - - "fastvideo/tests/lora/**" - - "fastvideo/models/loader/**" - - "fastvideo/tests/transformers/**" - - "fastvideo/pipelines/**" - - "fastvideo/layers/lora/**" - - "pyproject.toml" - - "docker/Dockerfile" - config: - command: "timeout 20m .buildkite/scripts/pr_test.sh" - label: ":test_tube: LoRA Inference Tests" - env: - - TEST_TYPE=inference_lora - agents: - queue: "default" - - path: - - "scripts/lora_extraction/**" - - "fastvideo/tests/lora_extraction/**" - - "fastvideo/models/loader/**" - - "fastvideo/training/training_utils.py" - - "fastvideo/layers/lora/**" - - "pyproject.toml" - - "docker/Dockerfile" - config: - command: "timeout 90m .buildkite/scripts/pr_test.sh" - label: ":test_tube: LoRA Extraction Tests" - env: - - TEST_TYPE=lora_extraction - agents: - queue: "default" - - path: - - "fastvideo/**" - - "pyproject.toml" - - "docker/Dockerfile" - config: - command: "timeout 25m .buildkite/scripts/pr_test.sh" - label: ":test_tube: Training Tests" - env: - - TEST_TYPE=training - agents: - queue: "default" - - path: - - "fastvideo/training/*distillation_pipeline.py" - - "pyproject.toml" - - "docker/Dockerfile" - config: - command: "timeout 25m .buildkite/scripts/pr_test.sh" - label: ":test_tube: Distillation DMD Tests" - env: - - TEST_TYPE=distillation_dmd - agents: - queue: "default" - - path: - - "fastvideo/training/*self_forcing_distillation_pipeline.py" - - "fastvideo/tests/training/self-forcing/**" - - "pyproject.toml" - - "docker/Dockerfile" - config: - command: "timeout 30m .buildkite/scripts/pr_test.sh" - label: ":test_tube: Self-Forcing Tests" - env: - - TEST_TYPE=self_forcing - agents: - queue: "default" - - path: - - "fastvideo/**" - - "pyproject.toml" - - "docker/Dockerfile" - config: - command: "timeout 25m .buildkite/scripts/pr_test.sh" - label: ":test_tube: LoRA Training Tests" - env: - - TEST_TYPE=training_lora - retry: - automatic: - - exit_status: 1 - limit: 2 - agents: - queue: "default" - - path: - - "fastvideo/**" - - "fastvideo-kernel/**" - - "pyproject.toml" - - "docker/Dockerfile" - config: - command: "timeout 15m .buildkite/scripts/pr_test.sh" - label: ":test_tube: Training Tests VSA" - env: - - TEST_TYPE=training_vsa - retry: - automatic: - - exit_status: 1 - limit: 2 - agents: - queue: "default" - - path: - - "fastvideo-kernel/**" - - "fastvideo/attention/backends/vmoba.py" - - "pyproject.toml" - - "docker/Dockerfile" - config: - command: "timeout 15m .buildkite/scripts/pr_test.sh" - label: ":test_tube: Inference Tests VMoBA" - env: - - TEST_TYPE=inference_vmoba - agents: - queue: "default" - - path: - - "fastvideo/models/dits/**" - - "fastvideo/pipelines/**" - - "fastvideo/attention/**" - - "fastvideo/layers/**" - - "fastvideo/worker/**" - - "fastvideo/entrypoints/**" - - "fastvideo/performance/**" - - "fastvideo/tests/performance/**" - - ".buildkite/performance-benchmarks/**" - - "pyproject.toml" - - "docker/Dockerfile" - config: - command: "timeout 30m .buildkite/scripts/pr_test.sh" - label: ":test_tube: Performance Tests" - env: - - TEST_TYPE=performance - agents: - queue: "default" - - path: - - "fastvideo/entrypoints/openai/**" - - "fastvideo/entrypoints/cli/serve.py" - - "fastvideo/tests/entrypoints/test_openai_api_integration.py" - - "pyproject.toml" - - "docker/Dockerfile" - config: - command: "timeout 30m .buildkite/scripts/pr_test.sh" - label: ":test_tube: API Server Tests" - env: - - TEST_TYPE=api_server - agents: - queue: "default" - - path: - - "fastvideo/train/**" - - "fastvideo/tests/train/models/**" - - "fastvideo/tests/train/fixtures/**" - - "fastvideo/models/dits/**" - - "fastvideo/models/loader/**" - - "pyproject.toml" - - "docker/Dockerfile" - config: - command: "timeout 30m .buildkite/scripts/pr_test.sh" - label: ":test_tube: Train Framework Tests" - env: - - TEST_TYPE=train_framework - agents: - queue: "default" - - path: - - "fastvideo/eval/**" - - "fastvideo/tests/eval/**" - - "pyproject.toml" - - "docker/Dockerfile" - config: - command: "timeout 90m .buildkite/scripts/pr_test.sh" - label: ":test_tube: Eval Metrics Tests" - env: - - TEST_TYPE=eval - agents: - queue: "default" + - label: ":microscope: Unit Tests" + key: "unit" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,unit,/) || + build.env("TEST_SCOPE") == "fastcheck" || + build.env("TEST_SCOPE") == null || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "unit_test" || build.env("TEST_TYPE") == "unit_test_ci")) + command: "/opt/fastvideo-ci-runner/run-unit" + timeout_in_minutes: 90 + env: + TEST_TYPE: "unit_test_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + agents: + queue: "ci-runner" + + - label: ":microscope: DreamVerse App Tests" + key: "dreamverse" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,dreamverse,/) || + build.env("TEST_SCOPE") == "fastcheck" || + build.env("TEST_SCOPE") == null || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "dreamverse_app" || build.env("TEST_TYPE") == "dreamverse_app_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + env: + TEST_TYPE: "dreamverse_app_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + agents: + queue: "ci-runner" + + - label: ":test_tube: Golden-Gate Tests" + key: "golden-gate" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,golden-gate,/) || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "golden_gate" || build.env("TEST_TYPE") == "golden_gate_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + env: + TEST_TYPE: "golden_gate_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + agents: + queue: "ci-runner" + + - label: ":bar_chart: SSIM Tests" + key: "ssim" + if: | + build.env("TEST_SCOPE") == "full" || + build.env("TEST_SCOPE") == "scheduled" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,ssim,/) || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "ssim" || build.env("TEST_TYPE") == "ssim_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + concurrency: 1 + concurrency_group: "fastvideo/slinky/whole-tray" + env: + TEST_TYPE: "ssim_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + - exit_status: 1 + limit: 2 + agents: + queue: "ci-runner" + + - label: ":test_tube: LoRA Inference Tests" + key: "lora-inference" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,lora-inference,/) || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "inference_lora" || build.env("TEST_TYPE") == "inference_lora_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + env: + TEST_TYPE: "inference_lora_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + agents: + queue: "ci-runner" + + - label: ":test_tube: LoRA Extraction Tests" + key: "lora-extraction" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,lora-extraction,/) || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "lora_extraction" || build.env("TEST_TYPE") == "lora_extraction_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + env: + TEST_TYPE: "lora_extraction_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + agents: + queue: "ci-runner" + + - label: ":test_tube: Training Tests" + key: "training" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,training,/) || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "training" || build.env("TEST_TYPE") == "training_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + concurrency: 1 + concurrency_group: "fastvideo/slinky/whole-tray" + env: + TEST_TYPE: "training_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + agents: + queue: "ci-runner" + + - label: ":test_tube: Distillation DMD Tests" + key: "distillation" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,distillation,/) || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "distillation_dmd" || build.env("TEST_TYPE") == "distillation_dmd_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + env: + TEST_TYPE: "distillation_dmd_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + agents: + queue: "ci-runner" + + - label: ":test_tube: Self-Forcing Tests" + key: "self-forcing" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,self-forcing,/) || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "self_forcing" || build.env("TEST_TYPE") == "self_forcing_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + env: + TEST_TYPE: "self_forcing_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + agents: + queue: "ci-runner" + + - label: ":test_tube: LoRA Training Tests" + key: "lora-training" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,lora-training,/) || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "training_lora" || build.env("TEST_TYPE") == "training_lora_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + env: + TEST_TYPE: "training_lora_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + - exit_status: 1 + limit: 2 + agents: + queue: "ci-runner" + + - label: ":test_tube: Training Tests VSA" + key: "training-vsa" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,training-vsa,/) || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "training_vsa" || build.env("TEST_TYPE") == "training_vsa_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + env: + TEST_TYPE: "training_vsa_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + - exit_status: 1 + limit: 2 + agents: + queue: "ci-runner" + + - label: ":test_tube: Inference Tests VMoBA" + key: "inference-vmoba" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,inference-vmoba,/) || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "inference_vmoba" || build.env("TEST_TYPE") == "inference_vmoba_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + env: + TEST_TYPE: "inference_vmoba_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + agents: + queue: "ci-runner" + + - label: ":test_tube: Performance Tests" + key: "performance" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,performance,/) || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "performance" || build.env("TEST_TYPE") == "performance_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + env: + TEST_TYPE: "performance_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + agents: + queue: "ci-runner" + + - label: ":test_tube: API Server Tests" + key: "api-server" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,api-server,/) || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "api_server" || build.env("TEST_TYPE") == "api_server_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + env: + TEST_TYPE: "api_server_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + agents: + queue: "ci-runner" + + - label: ":test_tube: Train Framework Tests" + key: "train-framework" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,train-framework,/) || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "train_framework" || build.env("TEST_TYPE") == "train_framework_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + env: + TEST_TYPE: "train_framework_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + agents: + queue: "ci-runner" + + - label: ":test_tube: Eval Metrics Tests" + key: "eval" + if: | + build.env("TEST_SCOPE") == "full" || + (build.env("TEST_SCOPE") == "merge" && + build.env("MERGE_TEST_PLAN") =~ /,eval,/) || + (build.env("TEST_SCOPE") == "direct" && + (build.env("TEST_TYPE") == "eval" || build.env("TEST_TYPE") == "eval_ci")) + command: "/opt/fastvideo-ci-runner/run-ci" + timeout_in_minutes: 90 + env: + TEST_TYPE: "eval_ci" + retry: + automatic: + - exit_status: 128 + limit: 3 + - exit_status: -1 + limit: 2 + agents: + queue: "ci-runner" diff --git a/.buildkite/scripts/lanes/api_server.sh b/.buildkite/scripts/lanes/api_server.sh new file mode 100755 index 000000000..a890625f0 --- /dev/null +++ b/.buildkite/scripts/lanes/api_server.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Canonical Slurm CI selection for the OpenAI-compatible API lane. +set -euo pipefail + +exec pytest ./fastvideo/tests/entrypoints/test_openai_api_integration.py -vs diff --git a/.buildkite/scripts/lanes/distillation_dmd.sh b/.buildkite/scripts/lanes/distillation_dmd.sh new file mode 100755 index 000000000..8d8b58685 --- /dev/null +++ b/.buildkite/scripts/lanes/distillation_dmd.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Canonical Slurm CI selection for the distillation-DMD lane. +set -euo pipefail + +exec pytest ./fastvideo/tests/training/distill/test_distill_dmd.py -vs diff --git a/.buildkite/scripts/lanes/dreamverse.sh b/.buildkite/scripts/lanes/dreamverse.sh new file mode 100755 index 000000000..e1e1ef2e2 --- /dev/null +++ b/.buildkite/scripts/lanes/dreamverse.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# DreamVerse needs a GPU for import-time device resolution, but it does not +# build or exercise fastvideo-kernel. A checksummed Node archive is installed +# in the disposable Slurm container because the shared CI image is +# Python/CUDA focused. +set -euo pipefail + +node_version=v22.23.2 +case $(uname -m) in + aarch64 | arm64) + node_arch=arm64 + node_archive_sha256=013b59cfd2819703a6f4a14ab891fc46fc2a4e3f5bcd92de3fb4929b43e35b30 + ;; + x86_64 | amd64) + node_arch=x64 + node_archive_sha256=b294a556e639d64338823920e5866c21c02741742d2e1529ee1a225c1ec9252a + ;; + *) + echo "Unsupported architecture for DreamVerse Node runtime: $(uname -m)" >&2 + exit 2 + ;; +esac +node_archive="node-${node_version}-linux-${node_arch}.tar.gz" +node_runtime_root=$(mktemp -d -t fastvideo-node.XXXXXX) +node_archive_path="${node_runtime_root}/${node_archive}" +node_install_dir="${node_runtime_root}/${node_archive%.tar.gz}" +curl --proto '=https' --tlsv1.2 --retry 5 --retry-all-errors \ + --location --fail --silent --show-error \ + "https://nodejs.org/dist/${node_version}/${node_archive}" \ + --output "$node_archive_path" +printf '%s %s\n' "$node_archive_sha256" "$node_archive_path" | sha256sum --check --status +tar -xzf "$node_archive_path" -C "$node_runtime_root" +export PATH="${node_install_dir}/bin:${PATH}" +node --version +npm --version + +export PYTHONPATH="$(pwd)/apps/dreamverse${PYTHONPATH:+:$PYTHONPATH}" +pytest apps/dreamverse/dreamverse/tests -q + +cd apps/dreamverse/web +npm ci +npm run typecheck +npm test +machine_arch=$(uname -m) +if [[ $machine_arch =~ ^(aarch64|arm64)$ ]]; then + npx playwright install --with-deps chromium firefox +else + npx playwright install --with-deps chromium webkit firefox +fi + +master_port=${MASTER_PORT:-7959} +BACKEND_PORT=${BACKEND_PORT:-$((master_port + 50))} +python -m uvicorn dreamverse.mock_server:app --host 127.0.0.1 --port "$BACKEND_PORT" & +mock_server_pid=$! +cleanup() { + kill "$mock_server_pid" 2>/dev/null || true + wait "$mock_server_pid" 2>/dev/null || true +} +trap cleanup EXIT INT TERM + +for _ in {1..30}; do + curl -fsS "http://127.0.0.1:$BACKEND_PORT/healthz" && break + sleep 1 +done +curl -fsS "http://127.0.0.1:$BACKEND_PORT/healthz" + +if [[ $machine_arch =~ ^(aarch64|arm64)$ ]]; then + # Playwright WebKit traps before opening a page on Linux ARM64, and its + # bundled Chromium lacks the H.264/AAC codecs used by the fMP4 assertions. + # Firefox covers every flow, including streaming. Chromium and its mobile + # profile still cover all codec-independent UI behavior on GB200. + BACKEND_HOST=127.0.0.1 BACKEND_PORT="$BACKEND_PORT" CI=1 \ + npm run e2e -- --project=firefox + BACKEND_HOST=127.0.0.1 BACKEND_PORT="$BACKEND_PORT" CI=1 \ + npm run e2e -- \ + --project=chromium \ + --project=mobile-chromium \ + --grep-invert='streams, plays, and surfaces a downloadable clip|starts a new project and switches back to the prior session|saved projects persist across a page reload' +else + BACKEND_HOST=127.0.0.1 BACKEND_PORT="$BACKEND_PORT" CI=1 \ + npm run e2e -- \ + --project=chromium \ + --project=webkit \ + --project=firefox \ + --project=mobile-safari \ + --project=mobile-chromium +fi diff --git a/.buildkite/scripts/lanes/encoder.sh b/.buildkite/scripts/lanes/encoder.sh new file mode 100755 index 000000000..71f6d6157 --- /dev/null +++ b/.buildkite/scripts/lanes/encoder.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Canonical Slurm CI selection for the encoder lane. +set -euo pipefail + +exec pytest ./fastvideo/tests/encoders -vs diff --git a/.buildkite/scripts/lanes/eval.sh b/.buildkite/scripts/lanes/eval.sh new file mode 100755 index 000000000..8563da45f --- /dev/null +++ b/.buildkite/scripts/lanes/eval.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Canonical Slurm CI selection for the evaluation lane. +set -euo pipefail + +exec pytest ./fastvideo/tests/eval -vs diff --git a/.buildkite/scripts/lanes/golden_gate.sh b/.buildkite/scripts/lanes/golden_gate.sh new file mode 100755 index 000000000..eb233469b --- /dev/null +++ b/.buildkite/scripts/lanes/golden_gate.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Canonical Slurm CI selection for the golden-gate lane. Environment (HF_HOME +# and authentication) is the runner's responsibility. +set -euo pipefail + +golden_root=./fastvideo/tests/golden_gate +selected=${FASTVIDEO_GOLDEN_TEST_FILES-} +if [ -z "$selected" ]; then + if [ "${TEST_SCOPE:-}" = merge ]; then + echo "Missing FASTVIDEO_GOLDEN_TEST_FILES for merge scope" >&2 + exit 2 + fi + selected=all +fi +if [ "$selected" = all ]; then + exec pytest "$golden_root" -vs +fi + +[[ $selected =~ ^test_[a-z0-9_]+\.py(,test_[a-z0-9_]+\.py)*$ ]] || { + echo "Invalid FASTVIDEO_GOLDEN_TEST_FILES selection" >&2 + exit 2 +} + +IFS=, read -r -a golden_files <<< "$selected" +golden_paths=() +for golden_file in "${golden_files[@]}"; do + golden_path="$golden_root/$golden_file" + [ -f "$golden_path" ] || { + echo "Selected golden test does not exist: $golden_file" >&2 + exit 2 + } + golden_paths+=("$golden_path") +done + +exec pytest "${golden_paths[@]}" -vs diff --git a/.buildkite/scripts/lanes/inference_lora.sh b/.buildkite/scripts/lanes/inference_lora.sh new file mode 100755 index 000000000..7d9e74d30 --- /dev/null +++ b/.buildkite/scripts/lanes/inference_lora.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Canonical Slurm CI selection for the LoRA-inference lane. +set -euo pipefail + +exec pytest ./fastvideo/tests/inference/lora/test_lora_inference_similarity.py -vs diff --git a/.buildkite/scripts/lanes/inference_vmoba.sh b/.buildkite/scripts/lanes/inference_vmoba.sh new file mode 100755 index 000000000..e678d10e2 --- /dev/null +++ b/.buildkite/scripts/lanes/inference_vmoba.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Canonical Slurm CI selection for the VMoBA-inference lane. +set -euo pipefail + +exec python fastvideo/tests/inference/vmoba/test_vmoba_inference.py diff --git a/.buildkite/scripts/lanes/kernel_tests.sh b/.buildkite/scripts/lanes/kernel_tests.sh new file mode 100755 index 000000000..986108dc5 --- /dev/null +++ b/.buildkite/scripts/lanes/kernel_tests.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Canonical Slurm CI selection for the custom-kernel lane. +set -euo pipefail + +exec pytest fastvideo-kernel/tests/ -vs diff --git a/.buildkite/scripts/lanes/lora_extraction.sh b/.buildkite/scripts/lanes/lora_extraction.sh new file mode 100755 index 000000000..46d7e0ec2 --- /dev/null +++ b/.buildkite/scripts/lanes/lora_extraction.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Canonical Slurm CI selection for the LoRA-extraction lane. +set -euo pipefail + +exec pytest ./fastvideo/tests/lora_extraction/test_lora_extraction.py -vs diff --git a/.buildkite/scripts/lanes/performance.sh b/.buildkite/scripts/lanes/performance.sh new file mode 100755 index 000000000..ab2d95d6b --- /dev/null +++ b/.buildkite/scripts/lanes/performance.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Canonical Slurm performance lane. Reports are written outside the checkout +# so the trusted host driver can upload them after untrusted code exits. +set -uo pipefail + +export PERFORMANCE_TRACKING_ROOT=/tmp/perf-tracking +export PERF_REPORTS_DIR=/workspace/artifacts/performance +mkdir -p "$PERF_REPORTS_DIR" + +if [[ ${BUILDKITE_PULL_REQUEST:-false} =~ ^[1-9][0-9]*$ ]]; then + export PERF_RUN_SOURCE=pr + export PERF_UPLOAD_POLICY=pass +elif [ "${BUILDKITE_BRANCH:-}" = main ] \ + && { [ "${BUILDKITE_SOURCE:-}" = schedule ] || [ "${TEST_SCOPE:-}" = full ]; }; then + export PERF_RUN_SOURCE=scheduled_main + export PERF_UPLOAD_POLICY=always +elif [ "${TEST_SCOPE:-}" = direct ]; then + export PERF_RUN_SOURCE=unknown + export PERF_UPLOAD_POLICY=pass +else + export PERF_RUN_SOURCE=unknown + export PERF_UPLOAD_POLICY=never +fi + +nvidia-smi \ + --query-gpu=index,timestamp,clocks.sm,clocks.max.sm,power.draw,power.limit,temperature.gpu \ + --format=csv -l 10 > "$PERF_REPORTS_DIR/gpu_telemetry.csv" 2>/dev/null & +telemetry_pid=$! +cleanup() { + kill "$telemetry_pid" 2>/dev/null || true + wait "$telemetry_pid" 2>/dev/null || true +} +trap cleanup EXIT INT TERM + +pytest ./fastvideo/tests/performance -vs +pytest_rc=$? +compare_rc=0 +if [ "$pytest_rc" -eq 0 ] || [ "$PERF_UPLOAD_POLICY" = always ]; then + PERF_PYTEST_RC=$pytest_rc python ./fastvideo/tests/performance/compare_baseline.py + compare_rc=$? +fi +python ./fastvideo/tests/performance/dashboard.py || true +cp -f fastvideo/tests/performance/results/*.json "$PERF_REPORTS_DIR/" 2>/dev/null || true + +echo "--- GPU telemetry (clocks.sm vs clocks.max.sm reveals capped hosts) ---" +cat "$PERF_REPORTS_DIR/gpu_telemetry.csv" || true + +final_rc=$pytest_rc +if [ "$final_rc" -eq 0 ]; then + final_rc=$compare_rc +fi +exit "$final_rc" diff --git a/.buildkite/scripts/lanes/self_forcing.sh b/.buildkite/scripts/lanes/self_forcing.sh new file mode 100755 index 000000000..41ec66c06 --- /dev/null +++ b/.buildkite/scripts/lanes/self_forcing.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Canonical Slurm CI selection for the self-forcing lane. +set -euo pipefail + +export WANDB_MODE=offline +exec pytest ./fastvideo/tests/training/self-forcing/test_self_forcing.py -vs diff --git a/.buildkite/scripts/lanes/ssim.sh b/.buildkite/scripts/lanes/ssim.sh new file mode 100755 index 000000000..54bc0d406 --- /dev/null +++ b/.buildkite/scripts/lanes/ssim.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Canonical four-GPU SSIM lane for the Slinky Slurm worker. +set -euo pipefail + +args=() +if [ "${FASTVIDEO_SSIM_BOOTSTRAP_MODE:-0}" = 1 ]; then + args+=(--bootstrap-mode) +fi +selected=${FASTVIDEO_SSIM_TEST_FILES-} +if [ -z "$selected" ]; then + if [ "${TEST_SCOPE:-}" = merge ]; then + echo "Missing FASTVIDEO_SSIM_TEST_FILES for merge scope" >&2 + exit 2 + fi + selected=all +fi +if [ "$selected" != all ]; then + [[ $selected =~ ^test_[a-z0-9_]+\.py(,test_[a-z0-9_]+\.py)*$ ]] || { + echo "Invalid FASTVIDEO_SSIM_TEST_FILES selection" >&2 + exit 2 + } + IFS=, read -r -a ssim_files <<< "$selected" + for ssim_file in "${ssim_files[@]}"; do + args+=(--test-file "$ssim_file") + done +fi + +# MoGe's utils3d dependency builds glcontext from source on ARM64. The current +# runner image predates the baked-in X11 headers below, so keep this guarded +# bootstrap until every deployed image digest contains libx11-dev. +if [ ! -f /usr/include/X11/Xlib.h ]; then + apt-get -o Acquire::Retries=5 update + apt-get -o Acquire::Retries=5 install -y --no-install-recommends libx11-dev + rm -rf /var/lib/apt/lists/* +fi + +uv pip install git+https://github.com/microsoft/MoGe.git +uv pip install k_diffusion einops_exts alias_free_torch torchsde + +exec python fastvideo/tests/ssim/ci_runner.py "${args[@]}" diff --git a/.buildkite/scripts/lanes/train_framework.sh b/.buildkite/scripts/lanes/train_framework.sh new file mode 100755 index 000000000..a64fa8b91 --- /dev/null +++ b/.buildkite/scripts/lanes/train_framework.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Canonical Slurm CI selection for the modular training-framework lane. +set -euo pipefail + +exec pytest ./fastvideo/tests/train/models ./fastvideo/tests/train/methods -vs diff --git a/.buildkite/scripts/lanes/training.sh b/.buildkite/scripts/lanes/training.sh new file mode 100755 index 000000000..fbbf9e4ba --- /dev/null +++ b/.buildkite/scripts/lanes/training.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Canonical Slurm CI selection for the legacy vanilla-training lane. +set -euo pipefail + +export WANDB_MODE=offline +exec pytest ./fastvideo/tests/training/Vanilla -srP diff --git a/.buildkite/scripts/lanes/training_lora.sh b/.buildkite/scripts/lanes/training_lora.sh new file mode 100755 index 000000000..19b663149 --- /dev/null +++ b/.buildkite/scripts/lanes/training_lora.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Canonical Slurm CI selection for the legacy LoRA-training lane. +set -euo pipefail + +export WANDB_MODE=offline +exec pytest ./fastvideo/tests/training/lora/test_lora_training.py -srP diff --git a/.buildkite/scripts/lanes/training_vsa.sh b/.buildkite/scripts/lanes/training_vsa.sh new file mode 100755 index 000000000..bf3c93a75 --- /dev/null +++ b/.buildkite/scripts/lanes/training_vsa.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Canonical Slurm CI selection for the legacy VSA-training lane. +set -euo pipefail + +export WANDB_MODE=offline +exec pytest ./fastvideo/tests/training/VSA -srP diff --git a/.buildkite/scripts/lanes/transformer.sh b/.buildkite/scripts/lanes/transformer.sh new file mode 100755 index 000000000..9041f10df --- /dev/null +++ b/.buildkite/scripts/lanes/transformer.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Canonical Slurm CI selection for the transformer lane. +set -euo pipefail + +exec pytest ./fastvideo/tests/transformers -vs diff --git a/.buildkite/scripts/lanes/vae.sh b/.buildkite/scripts/lanes/vae.sh new file mode 100755 index 000000000..7cc60c451 --- /dev/null +++ b/.buildkite/scripts/lanes/vae.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Canonical Slurm CI selection for the VAE lane. +set -euo pipefail + +exec pytest ./fastvideo/tests/vaes -vs diff --git a/.buildkite/scripts/pr_test.sh b/.buildkite/scripts/pr_test.sh index b97b3757c..3ea9fc7c4 100755 --- a/.buildkite/scripts/pr_test.sh +++ b/.buildkite/scripts/pr_test.sh @@ -1,6 +1,19 @@ #!/bin/bash set -uo pipefail +# DORMANT ROLLBACK ONLY. Active CI is Slurm-only and pipeline.yml never calls +# this launcher. Refuse every Buildkite invocation even if a stale step or +# operator typo reaches this file; local rollback experiments require an +# explicit opt-in. +if [ -n "${BUILDKITE:-}" ]; then + echo "Legacy Modal CI is disabled; use the Slinky Slurm runner." >&2 + exit 2 +fi +if [ "${FASTVIDEO_ENABLE_LEGACY_MODAL_CI:-0}" != 1 ]; then + echo "Legacy Modal CI is dormant. Set FASTVIDEO_ENABLE_LEGACY_MODAL_CI=1 only for a manual rollback test." >&2 + exit 2 +fi + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" } diff --git a/.buildkite/scripts/unit_test.sh b/.buildkite/scripts/unit_test.sh new file mode 100644 index 000000000..1d5c4475f --- /dev/null +++ b/.buildkite/scripts/unit_test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +exec pytest \ + ./fastvideo/tests/api/ \ + ./fastvideo/tests/contract/ \ + ./fastvideo/tests/dataset/ \ + ./fastvideo/tests/workflow/ \ + ./fastvideo/tests/entrypoints/ \ + ./fastvideo/tests/train/ \ + ./fastvideo/tests/stages/ \ + ./fastvideo/tests/ops/ \ + ./fastvideo/tests/worker/ \ + ./fastvideo/tests/training/test_trackers.py \ + ./fastvideo/tests/attention/test_sdpa_metadata_mask_contract.py \ + ./fastvideo/tests/modal/test_kernel_build_cache.py \ + ./fastvideo/tests/modal/test_pr_test.py \ + ./fastvideo/tests/modal/test_ssim_test.py \ + --ignore=./fastvideo/tests/entrypoints/test_openai_api_integration.py \ + --ignore=./fastvideo/tests/train/models \ + --ignore=./fastvideo/tests/train/methods \ + -vs diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 85829d36c..76da1e7e7 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -8,10 +8,10 @@ PR TITLE: Must start with a type tag, e.g.: MERGE WORKFLOW: 1. Ensure pre-commit passes and you have at least 1 approval 2. Comment /merge (or add the "ready" label) to enter the Merge Queue - 3. Full Test Suite runs automatically on a staging branch → auto-merge on success + 3. A path-aware merge gate runs only relevant integration tests → auto-merge on success ON-DEMAND TESTING (write access required): - /test full — Full Test Suite /test ssim — SSIM regression + /test full — Explicit all-lane run /test ssim — Full SSIM regression /test training — Training pipeline /test encoder — Encoder tests /test transformer — Transformer tests /test vae — VAE tests /test kernel — CUDA kernel tests /test unit — Unit tests diff --git a/.github/scripts/gate_full_suite.sh b/.github/scripts/gate_full_suite.sh index 782178f15..4ea8effe4 100755 --- a/.github/scripts/gate_full_suite.sh +++ b/.github/scripts/gate_full_suite.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash -# Gate the expensive Buildkite full suite on the cheap GitHub checks. +# Gate the path-aware Buildkite merge plan on the cheap GitHub checks. # # Polls the workflow runs for the PR head commit and only exits 0 once the # watched cheap workflows (pre-commit, docs build) have succeeded, so the -# 'ready' label cannot burn ~20 GPU lanes on a head that a cheap check has -# already doomed. +# 'ready' label cannot burn path-selected GPU lanes on a head that a cheap +# check has already doomed. # # Semantics: # - watched run completed with a bad conclusion -> exit 1 (fail CLOSED: -# no full suite; the next push re-arms via the 'synchronize' trigger) +# no merge gate; the next push re-arms via the 'synchronize' trigger) # - watched run cancelled -> still pending: the docs # workflow's repo-global 'pages' concurrency group cancels runs superseded # by unrelated pushes, so 'cancelled' is not a verdict on this PR @@ -29,7 +29,7 @@ set -euo pipefail : "${PR_NUMBER:?PR_NUMBER (pull request number) is required}" : "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}" -# Workflow-level `name:` values that must be green before the full suite +# Workflow-level `name:` values that must be green before the merge gate # may start. "Deploy Documentation" is path-filtered on PRs, so its run may # legitimately never exist; pre-commit always runs, so it must appear. WATCHED_NAMES='["pre-commit", "Deploy Documentation"]' @@ -56,7 +56,7 @@ recheck_ready_label() { if pr_json=$(gh_api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" 2>/dev/null); then if ! jq -e '[.labels[]?.name] | index("ready")' <<<"$pr_json" >/dev/null 2>&1; then echo "::error::PR #${PR_NUMBER} no longer has the 'ready' label —" \ - "NOT triggering the Buildkite full suite. Re-add the label to re-arm." + "NOT triggering the Buildkite merge gate. Re-add the label to re-arm." exit 1 fi else @@ -84,7 +84,7 @@ while true; do | map(.name) | join(", ")' <<<"$state") if [ -n "$failed" ]; then echo "::error::Cheap check(s) failed on ${PR_SHA}: ${failed}." \ - "NOT triggering the Buildkite full suite. Push a fix (the 'ready'" \ + "NOT triggering the Buildkite merge gate. Push a fix (the 'ready'" \ "label re-arms on every push), or re-run the failed check and then" \ "re-run this workflow." exit 1 @@ -97,7 +97,7 @@ while true; do if [ "$pending" -eq 0 ]; then if [ -z "$missing" ]; then recheck_ready_label - echo "All watched cheap checks are green — full suite may proceed." + echo "All watched cheap checks are green — merge gate may proceed." exit 0 fi case "$missing" in @@ -119,14 +119,14 @@ while true; do echo "::warning::GitHub API error querying workflow runs for ${PR_SHA} (attempt ${api_fails}/3)." if [ "$api_fails" -ge 3 ]; then recheck_ready_label - echo "::warning::FAILING OPEN: cannot query GitHub check status — triggering the full suite WITHOUT the cheap-check gate." + echo "::warning::FAILING OPEN: cannot query GitHub check status — triggering the merge gate WITHOUT the cheap-check gate." exit 0 fi fi if [ "$elapsed" -ge "$MAX_WAIT_SECS" ]; then recheck_ready_label - echo "::warning::FAILING OPEN: watched checks still pending after $(( MAX_WAIT_SECS / 60 )) min${missing:+ (never appeared: ${missing})} — triggering the full suite anyway." + echo "::warning::FAILING OPEN: watched checks still pending after $(( MAX_WAIT_SECS / 60 )) min${missing:+ (never appeared: ${missing})} — triggering the merge gate anyway." exit 0 fi sleep "$POLL_SECS" diff --git a/.github/scripts/plan_merge_ci.py b/.github/scripts/plan_merge_ci.py new file mode 100644 index 000000000..074a3e652 --- /dev/null +++ b/.github/scripts/plan_merge_ci.py @@ -0,0 +1,570 @@ +#!/usr/bin/env python3 +"""Select the additive GPU integration lanes needed by a PR diff. + +Fastcheck is the universal six-lane baseline and is intentionally not repeated +here. This planner selects only the more expensive merge-gate lanes. Unknown +source/build paths fail closed to the complete integration set, while explicit +documentation and repository-metadata paths require no additional GPU work. +""" + +from __future__ import annotations + +import argparse +import fnmatch +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import TextIO + +MERGE_LANES = ( + "golden-gate", + "ssim", + "lora-inference", + "lora-extraction", + "training", + "distillation", + "self-forcing", + "lora-training", + "training-vsa", + "inference-vmoba", + "performance", + "api-server", + "train-framework", + "eval", +) + +LANE_SCRIPT_TO_KEY = { + "api_server.sh": "api-server", + "distillation_dmd.sh": "distillation", + "eval.sh": "eval", + "golden_gate.sh": "golden-gate", + "inference_lora.sh": "lora-inference", + "inference_vmoba.sh": "inference-vmoba", + "lora_extraction.sh": "lora-extraction", + "performance.sh": "performance", + "self_forcing.sh": "self-forcing", + "ssim.sh": "ssim", + "train_framework.sh": "train-framework", + "training.sh": "training", + "training_lora.sh": "lora-training", + "training_vsa.sh": "training-vsa", +} + +FASTCHECK_LANE_SCRIPTS = { + "dreamverse.sh", + "encoder.sh", + "kernel_tests.sh", + "transformer.sh", + "vae.sh", +} + +LEGACY_TRAINING_LANES = ( + "training", + "distillation", + "self-forcing", + "lora-training", + "training-vsa", +) + +ALL_TRAINING_LANES = (*LEGACY_TRAINING_LANES, "train-framework") + +SSIM_SMOKE_TESTS = ( + "test_flux_t2i_similarity.py", + "test_wan_t2v_similarity.py", +) + +SAFE_PATTERNS = ( + "*.md", + "*.rst", + ".agents/**", + ".claude/**", + ".codex/**", + ".github/ISSUE_TEMPLATE/**", + ".github/PULL_REQUEST_TEMPLATE.md", + ".github/dependabot.yml", + ".github/mergify.yml", + ".github/scripts/**", + ".github/workflows/**", + ".buildkite/scripts/pre_commit.sh", + ".git-blame-ignore-revs", + ".gitattributes", + ".gitignore", + ".pre-commit-config.yaml", + "AGENTS.md", + "CITATION.cff", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "NOTICE", + "__init__.py", + "collect_env.py", + "SECURITY.md", + "assets/**", + "comfyui/**", + "docs/**", + "examples/**", + "mkdocs.yml", + "requirements-mkdocs.in", + "requirements-mkdocs.txt", + "scripts/**", + "tests/__init__.py", + "tests/local_tests/**", +) + +ALL_IMPACT_PATTERNS = ( + ".buildkite/pipeline.yml", + "docker/**", + "pyproject.toml", + "requirements*.txt", + "setup.cfg", + "setup.py", + "uv.lock", +) + + +@dataclass(frozen=True) +class FamilyCoverage: + pattern: re.Pattern[str] + golden_tests: tuple[str, ...] + ssim_tests: tuple[str, ...] + + +FAMILY_COVERAGE = ( + FamilyCoverage( + re.compile(r"(^|[/_.-])dreamx(_world)?([/_.-]|$)"), + ("test_dreamx.py", ), + ("test_dreamx_world_similarity.py", ), + ), + FamilyCoverage( + re.compile(r"(^|[/_.-])flux[_-]?2([/_.-]|$)"), + ("test_flux2_klein.py", ), + ("test_flux2_similarity.py", ), + ), + FamilyCoverage( + re.compile(r"(^|[/_.-])flux(?![_-]?2)([/_.-]|$)"), + ("test_flux.py", ), + ("test_flux_t2i_similarity.py", ), + ), + FamilyCoverage( + re.compile(r"(^|[/_.-])(hunyuan)?gamecraft([/_.-]|$)"), + ("test_gamecraft.py", ), + ("test_gamecraft_similarity.py", ), + ), + FamilyCoverage( + re.compile(r"(^|[/_.-])gen3c([/_.-]|$)"), + ("test_gen3c.py", ), + ("test_gen3c_similarity.py", ), + ), + FamilyCoverage( + re.compile(r"(^|[/_.-])glm[_-]?image([/_.-]|$)"), + ("test_glm_image.py", ), + ("test_glm_image_similarity.py", ), + ), + FamilyCoverage( + re.compile(r"(^|[/_.-])kandinsky[_-]?5([/_.-]|$)"), + ("test_kandinsky5.py", ), + ("test_kandinsky5_similarity.py", ), + ), + FamilyCoverage( + re.compile(r"(^|[/_.-])lingbot([a-z0-9_-]*)([/_.-]|$)"), + ("test_lingbot.py", ), + ("test_lingbot_similarity.py", ), + ), + FamilyCoverage( + re.compile(r"(^|[/_.-])longcat([/_.-]|$)"), + ("test_longcat.py", ), + ("test_longcat_similarity.py", ), + ), + FamilyCoverage( + re.compile(r"(^|[/_.-])ltx[_-]?2([/_.-]|$)"), + ("test_ltx2.py", ), + ("test_ltx2_similarity.py", ), + ), + FamilyCoverage( + re.compile(r"(^|[/_.-])matrixgame[_-]?2([/_.-]|$)"), + ("test_matrixgame.py", ), + ("test_matrixgame2_similarity.py", ), + ), + FamilyCoverage( + re.compile(r"(^|[/_.-])matrixgame[_-]?3([/_.-]|$)"), + ("test_matrixgame.py", ), + ("test_matrixgame3_similarity.py", ), + ), + FamilyCoverage( + re.compile(r"(^|[/_.-])minimax[_-]?h3([/_.-]|$)"), + ("test_minimax_h3_t2v.py", ), + ("test_minimax_h3_similarity.py", ), + ), + FamilyCoverage( + re.compile(r"(^|[/_.-])sd[_-]?3([._-]?5)?([/_.-]|$)"), + ("test_sd35.py", ), + ("test_sd35_similarity.py", ), + ), + FamilyCoverage( + re.compile(r"(^|[/_.-])stable[_-]?audio([/_.-]|$)"), + ("test_stable_audio.py", ), + ("test_stable_audio_similarity.py", ), + ), + FamilyCoverage( + re.compile(r"(^|[/_.-])turbo(diffusion)?([/_.-]|$)"), + (), + ("test_turbodiffusion_similarity.py", ), + ), + FamilyCoverage( + re.compile(r"(^|[/_.-])wan(video)?([/_.-]|$)"), + ("test_wan_t2v.py", ), + ( + "test_causal_similarity.py", + "test_wan_i2v_similarity.py", + "test_wan_t2v_similarity.py", + ), + ), + FamilyCoverage( + re.compile(r"(^|[/_.-])z[_-]?image([/_.-]|$)"), + ("test_zimage.py", ), + ("test_zimage_similarity.py", ), + ), +) + + +@dataclass +class MergePlan: + lanes: set[str] = field(default_factory=set) + golden_tests: set[str] = field(default_factory=set) + ssim_tests: set[str] = field(default_factory=set) + golden_all: bool = False + ssim_all: bool = False + reasons: list[str] = field(default_factory=list) + + def add_lanes(self, *lanes: str, reason: str) -> None: + unknown = set(lanes) - set(MERGE_LANES) + if unknown: + raise ValueError(f"Unknown merge lanes: {sorted(unknown)}") + self.lanes.update(lanes) + self.reasons.append(reason) + + def add_golden(self, tests: tuple[str, ...], reason: str) -> None: + self.add_lanes("golden-gate", reason=reason) + self.golden_tests.update(tests) + + def add_ssim(self, tests: tuple[str, ...], reason: str) -> None: + self.add_lanes("ssim", reason=reason) + self.ssim_tests.update(tests) + + def require_all(self, reason: str) -> None: + self.lanes.update(MERGE_LANES) + self.golden_all = True + self.ssim_all = True + self.reasons.append(reason) + + def ordered_lanes(self) -> tuple[str, ...]: + return tuple(lane for lane in MERGE_LANES if lane in self.lanes) + + def encoded_lanes(self) -> str: + lanes = self.ordered_lanes() + return "," + ",".join(lanes or ("none", )) + "," + + def encoded_golden_tests(self) -> str: + if "golden-gate" not in self.lanes: + return "none" + if self.golden_all or not self.golden_tests: + return "all" + return ",".join(sorted(self.golden_tests)) + + def encoded_ssim_tests(self) -> str: + if "ssim" not in self.lanes: + return "none" + if self.ssim_all or not self.ssim_tests: + return "all" + return ",".join(sorted(self.ssim_tests)) + + +def _matches_any(path: str, patterns: tuple[str, ...]) -> bool: + return any(fnmatch.fnmatchcase(path, pattern) for pattern in patterns) + + +def _family_coverage(path: str) -> tuple[set[str], set[str]]: + normalized = path.lower() + golden: set[str] = set() + ssim: set[str] = set() + for family in FAMILY_COVERAGE: + if family.pattern.search(normalized): + golden.update(family.golden_tests) + ssim.update(family.ssim_tests) + return golden, ssim + + +def _select_output_coverage(plan: MergePlan, path: str) -> None: + golden, ssim = _family_coverage(path) + if golden: + plan.add_golden(tuple(sorted(golden)), reason=f"model-family golden coverage: {path}") + else: + plan.golden_all = True + plan.add_lanes("golden-gate", reason=f"shared output golden coverage: {path}") + if ssim: + plan.add_ssim(tuple(sorted(ssim)), reason=f"model-family SSIM coverage: {path}") + else: + plan.add_ssim(SSIM_SMOKE_TESTS, reason=f"shared output SSIM smoke coverage: {path}") + + +def classify_paths(paths: list[str]) -> MergePlan: + plan = MergePlan() + normalized_paths: list[str] = [] + for raw_path in paths: + path = raw_path.strip() + while path.startswith("./"): + path = path[2:] + if path: + normalized_paths.append(path) + normalized_paths = sorted(set(normalized_paths)) + if not normalized_paths: + plan.require_all("changed-file list was empty; failing closed") + return plan + + for path in normalized_paths: + if path == "__FASTVIDEO_CI_PLAN_ALL__": + plan.require_all("changed-file API failed; failing closed") + continue + + if path in {"requirements-mkdocs.in", "requirements-mkdocs.txt"}: + plan.reasons.append(f"documentation dependencies need no GPU integration: {path}") + continue + + if _matches_any(path, ALL_IMPACT_PATTERNS): + plan.require_all(f"cross-cutting build/runtime surface: {path}") + continue + + lane_script_prefix = ".buildkite/scripts/lanes/" + if path.startswith(lane_script_prefix): + script_name = Path(path).name + lane = LANE_SCRIPT_TO_KEY.get(script_name) + if lane is None: + if script_name in FASTCHECK_LANE_SCRIPTS: + plan.reasons.append(f"covered by automatic Fastcheck lane: {path}") + else: + plan.require_all(f"unknown lane script: {path}") + elif lane == "golden-gate": + plan.golden_all = True + plan.add_lanes(lane, reason=f"golden lane implementation: {path}") + elif lane == "ssim": + plan.ssim_all = True + plan.add_lanes(lane, reason=f"SSIM lane implementation: {path}") + else: + plan.add_lanes(lane, reason=f"lane implementation: {path}") + continue + + if path.startswith("fastvideo/tests/golden_gate/"): + name = Path(path).name + if name.startswith("test_") and name.endswith(".py"): + plan.add_golden((name, ), reason=f"changed golden test: {path}") + elif name in {"AGENTS.md", "README.md"}: + plan.reasons.append(f"golden documentation only: {path}") + else: + plan.golden_all = True + plan.add_lanes("golden-gate", reason=f"shared golden harness/reference: {path}") + continue + + if path.startswith("fastvideo/tests/ssim/"): + name = Path(path).name + if name.startswith("test_") and name.endswith(".py"): + plan.add_ssim((name, ), reason=f"changed SSIM test: {path}") + elif path.endswith((".py", ".json", ".pt", ".png", ".mp4")): + plan.ssim_all = True + plan.add_lanes("ssim", reason=f"shared SSIM harness/reference: {path}") + continue + + if path.startswith("fastvideo/tests/performance/") or path.startswith(".buildkite/performance-benchmarks/"): + plan.add_lanes("performance", reason=f"performance coverage: {path}") + continue + if path.startswith(("fastvideo/performance/", "fastvideo/performance_dashboard/", + "apps/performance_dashboard/")): + plan.add_lanes("performance", reason=f"performance implementation: {path}") + continue + if path.startswith("fastvideo/benchmarks/"): + if "/mlx_" in path or Path(path).name.startswith("mlx_"): + plan.reasons.append(f"covered by the path-filtered macOS MLX workflow: {path}") + else: + plan.add_lanes("performance", reason=f"benchmark implementation: {path}") + continue + if path.startswith("fastvideo/tests/eval/") or path.startswith("fastvideo/eval/"): + plan.add_lanes("eval", reason=f"evaluation coverage: {path}") + continue + if path.startswith("fastvideo/third_party/eval/"): + plan.add_lanes("eval", reason=f"vendored evaluation implementation: {path}") + continue + if path.startswith("fastvideo/tests/lora_extraction/") or path.startswith("scripts/lora_extraction/"): + plan.add_lanes("lora-extraction", reason=f"LoRA extraction coverage: {path}") + continue + if path.startswith("fastvideo/tests/inference/lora/"): + plan.add_lanes("lora-inference", reason=f"LoRA inference coverage: {path}") + continue + if path.startswith("fastvideo/tests/inference/vmoba/"): + plan.add_lanes("inference-vmoba", reason=f"VMoBA inference coverage: {path}") + continue + if path.startswith(("fastvideo/dataset/", "fastvideo/workflow/", "fastvideo/pipelines/preprocess/", + "fastvideo/pipelines/training/")): + plan.add_lanes(*ALL_TRAINING_LANES, reason=f"shared data/training input surface: {path}") + continue + if path.startswith("fastvideo/tests/train/") or path.startswith("fastvideo/train/"): + plan.add_lanes("train-framework", reason=f"modular training coverage: {path}") + continue + + if path.startswith("fastvideo/tests/training/"): + lowered = path.lower() + if "/vanilla/" in lowered: + plan.add_lanes("training", reason=f"vanilla training coverage: {path}") + elif "/distill/" in lowered: + plan.add_lanes("distillation", reason=f"distillation coverage: {path}") + elif "/self-forcing/" in lowered: + plan.add_lanes("self-forcing", reason=f"self-forcing coverage: {path}") + elif "/lora/" in lowered: + plan.add_lanes("lora-training", reason=f"LoRA training coverage: {path}") + elif "/vsa/" in lowered: + plan.add_lanes("training-vsa", reason=f"VSA training coverage: {path}") + else: + plan.add_lanes(*LEGACY_TRAINING_LANES, reason=f"shared legacy training coverage: {path}") + continue + + if path.startswith("fastvideo/training/"): + lowered = path.lower() + if "self_forcing" in lowered: + plan.add_lanes("self-forcing", reason=f"self-forcing implementation: {path}") + elif "distill" in lowered: + plan.add_lanes("distillation", reason=f"distillation implementation: {path}") + elif "lora" in lowered: + plan.add_lanes("lora-training", reason=f"LoRA training implementation: {path}") + else: + plan.add_lanes(*LEGACY_TRAINING_LANES, reason=f"shared legacy training implementation: {path}") + continue + + lowered = path.lower() + if "vmoba" in lowered and path.startswith(("fastvideo/", ".buildkite/")): + plan.add_lanes("inference-vmoba", reason=f"VMoBA implementation: {path}") + plan.add_golden(("test_wan_t2v.py", ), reason=f"VMoBA end-to-end coverage: {path}") + continue + if "lora" in lowered and path.startswith("fastvideo/"): + plan.add_lanes( + "lora-inference", + "lora-extraction", + "lora-training", + reason=f"shared LoRA implementation: {path}", + ) + _select_output_coverage(plan, path) + continue + + if path.startswith("fastvideo/entrypoints/") or path.startswith("fastvideo/api/"): + plan.add_lanes("api-server", reason=f"API/entrypoint integration: {path}") + if "openai" not in lowered and "/cli/" not in lowered: + _select_output_coverage(plan, path) + continue + if path.startswith("fastvideo/worker/"): + plan.add_lanes("api-server", reason=f"worker/API integration: {path}") + _select_output_coverage(plan, path) + continue + if path.startswith("fastvideo/distributed/"): + plan.add_lanes( + "training", + "train-framework", + reason=f"distributed runtime integration: {path}", + ) + _select_output_coverage(plan, path) + continue + if path.startswith(("fastvideo/hooks/", "fastvideo/platforms/", "fastvideo/third_party/")): + _select_output_coverage(plan, path) + continue + if path.startswith(("fastvideo/models/", "fastvideo/pipelines/", "fastvideo/configs/", + "fastvideo/layers/", "fastvideo/attention/")): + _select_output_coverage(plan, path) + continue + if path in { + "fastvideo/fastvideo_args.py", + "fastvideo/forward_context.py", + "fastvideo/image_processor.py", + "fastvideo/registry.py", + "fastvideo/utils.py", + }: + _select_output_coverage(plan, path) + continue + if path.startswith("fastvideo/mlx_runtime/"): + plan.reasons.append(f"covered by the path-filtered macOS MLX workflow: {path}") + continue + if path.startswith("fastvideo/logging_utils/") or path in { + "fastvideo/__init__.py", + "fastvideo/envs.py", + "fastvideo/logger.py", + "fastvideo/profiler.py", + "fastvideo/version.py", + }: + plan.reasons.append(f"covered by automatic Fastcheck: {path}") + continue + if path.startswith(("fastvideo-kernel/", "csrc/")): + plan.add_golden(("test_wan_t2v.py", ), reason=f"kernel integration smoke: {path}") + plan.add_ssim(("test_wan_t2v_similarity.py", ), reason=f"kernel numerical smoke: {path}") + continue + + if path.startswith("apps/dreamverse/"): + # DreamVerse is already one of the six automatic Fastcheck lanes. + plan.reasons.append(f"covered by automatic DreamVerse Fastcheck: {path}") + continue + if path.startswith("fastvideo/tests/"): + # The automatic unit/component Fastcheck lanes own the remaining + # package tests. Domain-specific expensive test roots were handled + # above. + plan.reasons.append(f"covered by automatic Fastcheck: {path}") + continue + if path in {".buildkite/scripts/unit_test.sh", ".buildkite/scripts/pr_test.sh"}: + plan.reasons.append(f"covered by automatic unit Fastcheck: {path}") + continue + if _matches_any(path, SAFE_PATTERNS): + plan.reasons.append(f"no additional GPU integration needed: {path}") + continue + + plan.require_all(f"unclassified path; failing closed: {path}") + + return plan + + +def _write_github_output(output: TextIO, plan: MergePlan) -> None: + output.write(f"merge_test_plan={plan.encoded_lanes()}\n") + output.write(f"merge_golden_tests={plan.encoded_golden_tests()}\n") + output.write(f"merge_ssim_tests={plan.encoded_ssim_tests()}\n") + output.write(f"merge_plan_label={','.join(plan.ordered_lanes()) or 'none'}\n") + + +def _write_summary(output: TextIO, plan: MergePlan) -> None: + output.write("## Change-aware merge test plan\n\n") + output.write("| Selection | Value |\n|---|---|\n") + output.write(f"| Additional Slurm lanes | `{','.join(plan.ordered_lanes()) or 'none'}` |\n") + output.write(f"| Golden tests | `{plan.encoded_golden_tests()}` |\n") + output.write(f"| SSIM tests | `{plan.encoded_ssim_tests()}` |\n\n") + output.write("Fastcheck remains the universal six-lane baseline.\n") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--paths-file", type=Path, required=True) + parser.add_argument("--github-output", type=Path) + parser.add_argument("--summary-file", type=Path) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + paths = args.paths_file.read_text(encoding="utf-8").splitlines() + plan = classify_paths(paths) + print(f"MERGE_TEST_PLAN={plan.encoded_lanes()}") + print(f"MERGE_GOLDEN_TESTS={plan.encoded_golden_tests()}") + print(f"MERGE_SSIM_TESTS={plan.encoded_ssim_tests()}") + for reason in plan.reasons: + print(f"- {reason}") + if args.github_output: + with args.github_output.open("a", encoding="utf-8") as output: + _write_github_output(output, plan) + if args.summary_file: + with args.summary_file.open("a", encoding="utf-8") as output: + _write_summary(output, plan) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/test_gate_full_suite.sh b/.github/scripts/test_gate_full_suite.sh index 884ec4a02..126ca44b4 100755 --- a/.github/scripts/test_gate_full_suite.sh +++ b/.github/scripts/test_gate_full_suite.sh @@ -53,7 +53,7 @@ PC_PENDING='{"name": "pre-commit", "id": 1, "status": "in_progress", "conclusion DOCS_OK='{"name": "Deploy Documentation", "id": 2, "status": "completed", "conclusion": "success"}' DOCS_BAD='{"name": "Deploy Documentation", "id": 2, "status": "completed", "conclusion": "failure"}' DOCS_CANCELLED='{"name": "Deploy Documentation", "id": 2, "status": "completed", "conclusion": "cancelled"}' -OTHER='{"name": "Trigger Full Suite", "id": 3, "status": "in_progress", "conclusion": null}' +OTHER='{"name": "Trigger Merge Gate", "id": 3, "status": "in_progress", "conclusion": null}' NULL_NAME='{"name": null, "id": 4, "status": "completed", "conclusion": "failure"}' PC_OK_RERUN='{"name": "pre-commit", "id": 5, "status": "completed", "conclusion": "success"}' diff --git a/.github/workflows/_template-build-image.yml b/.github/workflows/_template-build-image.yml index 3fea4822a..93bdf29aa 100644 --- a/.github/workflows/_template-build-image.yml +++ b/.github/workflows/_template-build-image.yml @@ -190,6 +190,7 @@ jobs: if: ${{ !inputs.push_by_digest }} run: | echo "✅ Python ${{ inputs.python_version }} image successfully built and pushed to ${{ steps.image.outputs.name }}:${{ inputs.tag_suffix }}-sha-${GITHUB_SHA::7}" + echo "Digest: ${{ steps.build-push.outputs.digest }}" echo "To run tests with this image, manually trigger the 'Run Tests' workflow." - name: Digest success message diff --git a/.github/workflows/ci-aggregate-status.yml b/.github/workflows/ci-aggregate-status.yml index 3204af99b..b08b31d82 100644 --- a/.github/workflows/ci-aggregate-status.yml +++ b/.github/workflows/ci-aggregate-status.yml @@ -26,29 +26,48 @@ jobs: per_page: 100, }); - const bkStatuses = data.statuses.filter( - s => s.context.startsWith('buildkite/ci/') - ); - - const FASTCHECK_PREFIX = 'buildkite/ci/microscope-'; + // Buildkite derives the GitHub context prefix from the label emoji. + // Keep hard Full Suite lanes in test-tube/bar-chart namespaces and + // Fastcheck lanes in microscope so targeted reruns cannot clear the + // wrong aggregate status. Automatic PR jobs use pr-fastcheck while + // slash-command and Full Suite jobs use ci; normalize the suffix + // and keep the newest status for each logical lane. + const FASTCHECK_PREFIXES = [ + 'buildkite/pr-fastcheck/microscope-', + 'buildkite/ci/microscope-', + ]; const FULL_SUITE_PREFIXES = [ 'buildkite/ci/test-tube-', 'buildkite/ci/bar-chart-', ]; - const fastcheck = bkStatuses.filter( - s => s.context.startsWith(FASTCHECK_PREFIX) - ); - const fullSuite = bkStatuses.filter( - s => FULL_SUITE_PREFIXES.some(p => s.context.startsWith(p)) - ); + function newestByLane(prefixes) { + const statuses = new Map(); + for (const status of data.statuses) { + const prefix = prefixes.find(p => status.context.startsWith(p)); + if (!prefix) continue; + const lane = status.context.slice(prefix.length); + const previous = statuses.get(lane); + if (!previous || Date.parse(status.updated_at) > Date.parse(previous.updated_at)) { + statuses.set(lane, status); + } + } + return statuses; + } - if ( - fastcheck.length > 0 - && fastcheck.every(s => s.state === 'success') - ) { + const fastcheck = newestByLane(FASTCHECK_PREFIXES); + const fullSuiteOnly = newestByLane(FULL_SUITE_PREFIXES); + const fastcheckPassed = + fastcheck.size === 6 + && [...fastcheck.values()].every(s => s.state === 'success'); + const fullSuitePassed = + fastcheckPassed + && fullSuiteOnly.size === 14 + && [...fullSuiteOnly.values()].every(s => s.state === 'success'); + + if (fastcheckPassed) { core.info( - `All ${fastcheck.length} fastcheck tests passed — updating fastcheck-passed` + `All ${fastcheck.size} fastcheck tests passed — updating fastcheck-passed` ); await github.rest.repos.createCommitStatus({ owner: context.repo.owner, @@ -56,17 +75,13 @@ jobs: sha, state: 'success', context: 'fastcheck-passed', - description: - `All ${fastcheck.length} fastcheck tests passed`, + description: `All ${fastcheck.size} fastcheck tests passed`, }); } - if ( - fullSuite.length > 0 - && fullSuite.every(s => s.state === 'success') - ) { + if (fullSuitePassed) { core.info( - `All ${fullSuite.length} full suite tests passed — updating full-suite-passed` + 'All 20 full suite tests passed — updating full-suite-passed' ); await github.rest.repos.createCommitStatus({ owner: context.repo.owner, @@ -74,7 +89,6 @@ jobs: sha, state: 'success', context: 'full-suite-passed', - description: - `All ${fullSuite.length} full suite tests passed`, + description: 'All 20 full suite tests passed', }); } diff --git a/.github/workflows/ci-scheduled-ssim.yml b/.github/workflows/ci-scheduled-ssim.yml new file mode 100644 index 000000000..1c8313d58 --- /dev/null +++ b/.github/workflows/ci-scheduled-ssim.yml @@ -0,0 +1,44 @@ +name: Scheduled Full SSIM + +on: + schedule: + - cron: "0 5 * * 0" + workflow_dispatch: + +permissions: + contents: read + +jobs: + trigger: + if: github.repository == 'hao-ai-lab/FastVideo' + runs-on: ubuntu-latest + steps: + - name: Trigger weekly full SSIM on Slinky Slurm + env: + BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }} + SOURCE_SHA: ${{ github.sha }} + SOURCE_BRANCH: ${{ github.event.repository.default_branch }} + BK_ORG: ${{ vars.BUILDKITE_ORG_SLUG }} + BK_PIPELINE: ${{ vars.BUILDKITE_PIPELINE_SLUG }} + run: | + set -euo pipefail + curl -sS --fail-with-body -X POST \ + "https://api.buildkite.com/v2/organizations/${BK_ORG}/pipelines/${BK_PIPELINE}/builds" \ + -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \ + -H "Content-Type: application/json" \ + --data-raw "$(jq -n \ + --arg commit "$SOURCE_SHA" \ + --arg branch "$SOURCE_BRANCH" \ + '{ + commit: $commit, + branch: $branch, + message: "Weekly full SSIM on Slinky Slurm", + ignore_pipeline_branch_filters: true, + env: { + TEST_SCOPE: "scheduled", + FULL_SUITE: "false", + TEST_TYPE: "ssim", + PR_NUMBER: "false", + PR_TITLE: "Scheduled full SSIM" + } + }')" diff --git a/.github/workflows/ci-slash-commands.yml b/.github/workflows/ci-slash-commands.yml index 06600df9f..2755a7bb6 100644 --- a/.github/workflows/ci-slash-commands.yml +++ b/.github/workflows/ci-slash-commands.yml @@ -33,7 +33,6 @@ jobs: core.setOutput('has_write', String(hasWrite)); - name: Add ready label and react - id: label if: steps.perm.outputs.has_write == 'true' uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 with: @@ -48,47 +47,6 @@ jobs: comment_id: context.payload.comment.id, content: 'rocket', }); - const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: prNumber }); - core.setOutput('pr_sha', pr.head.sha); - core.setOutput('pr_branch', pr.head.ref); - core.setOutput('pr_number', String(prNumber)); - core.setOutput('pr_title', pr.title); - - - name: Trigger Full Suite - if: steps.perm.outputs.has_write == 'true' - env: - BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }} - PR_SHA: ${{ steps.label.outputs.pr_sha }} - PR_BRANCH: ${{ steps.label.outputs.pr_branch }} - PR_NUMBER: ${{ steps.label.outputs.pr_number }} - PR_TITLE: ${{ steps.label.outputs.pr_title }} - BK_ORG: ${{ vars.BUILDKITE_ORG_SLUG }} - BK_PIPELINE: ${{ vars.BUILDKITE_PIPELINE_SLUG }} - run: | - curl -sS --fail-with-body -X POST \ - "https://api.buildkite.com/v2/organizations/${BK_ORG}/pipelines/${BK_PIPELINE}/builds" \ - -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \ - -H "Content-Type: application/json" \ - --data-raw "$(jq -n \ - --arg commit "$PR_SHA" \ - --arg branch "$PR_BRANCH" \ - --arg message "Full Suite for PR #${PR_NUMBER} (via /merge)" \ - --arg pr_title "$PR_TITLE" \ - --argjson pr_id "$PR_NUMBER" \ - '{ - commit: $commit, - branch: $branch, - message: $message, - ignore_pipeline_branch_filters: true, - pull_request_id: $pr_id, - pull_request_base_branch: "main", - env: { - TEST_SCOPE: "full", - FULL_SUITE: "true", - PR_NUMBER: ($pr_id | tostring), - PR_TITLE: $pr_title - } - }')" parse-command: if: >- @@ -129,7 +87,7 @@ jobs: set -euo pipefail TEST_NAME=$(echo "$COMMENT" | grep -oP '(?<=/test\s)\S+' | head -1 || true) - VALID="encoder vae transformer kernel unit dreamverse ssim golden-gate training lora-inference lora-training lora-extraction distillation self-forcing vsa vmoba performance api train-framework eval full fastcheck pre-commit" + VALID="encoder vae transformer kernel unit dreamverse ssim golden-gate training lora-inference lora-training lora-extraction distillation self-forcing vsa vmoba performance api train-framework eval unit-ci kernel-ci dreamverse-ci ssim-ci golden-gate-ci encoder-ci vae-ci transformer-ci lora-inference-ci lora-training-ci lora-extraction-ci training-ci distillation-ci self-forcing-ci vsa-ci vmoba-ci performance-ci api-ci train-framework-ci eval-ci full fastcheck pre-commit" if [ -z "$TEST_NAME" ] || ! echo "$VALID" | grep -qw "$TEST_NAME"; then echo "Unknown test: '$TEST_NAME'. Valid: $VALID" exit 1 @@ -137,7 +95,17 @@ jobs: declare -A MAP=( [encoder]=encoder [vae]=vae [transformer]=transformer - [kernel]=kernel_tests [unit]=unit_test [dreamverse]=dreamverse_app + [kernel]=kernel_tests [unit]=unit_test [unit-ci]=unit_test_ci + [kernel-ci]=kernel_tests_ci [dreamverse-ci]=dreamverse_app_ci + [ssim-ci]=ssim_ci [vmoba-ci]=inference_vmoba_ci + [golden-gate-ci]=golden_gate_ci [training-ci]=training_ci + [encoder-ci]=encoder_ci [vae-ci]=vae_ci [transformer-ci]=transformer_ci + [lora-inference-ci]=inference_lora_ci [lora-training-ci]=training_lora_ci + [lora-extraction-ci]=lora_extraction_ci [distillation-ci]=distillation_dmd_ci + [self-forcing-ci]=self_forcing_ci [vsa-ci]=training_vsa_ci + [performance-ci]=performance_ci [api-ci]=api_server_ci + [train-framework-ci]=train_framework_ci [eval-ci]=eval_ci + [dreamverse]=dreamverse_app [ssim]=ssim [golden-gate]=golden_gate [training]=training [lora-inference]=inference_lora [lora-training]=training_lora [lora-extraction]=lora_extraction diff --git a/.github/workflows/ci-trigger-full-suite.yml b/.github/workflows/ci-trigger-full-suite.yml index d055eb73c..e96ad9c81 100644 --- a/.github/workflows/ci-trigger-full-suite.yml +++ b/.github/workflows/ci-trigger-full-suite.yml @@ -1,4 +1,4 @@ -name: Trigger Full Suite +name: Trigger Merge Gate on: pull_request_target: @@ -10,7 +10,7 @@ permissions: actions: read concurrency: - group: full-suite-${{ github.event.pull_request.number }} + group: merge-gate-${{ github.event.pull_request.number }} cancel-in-progress: false jobs: @@ -34,29 +34,72 @@ jobs: }); const hasReady = pr.labels.some(l => l.name === 'ready'); core.setOutput('has_ready', String(hasReady)); - if (!hasReady) core.info('No ready label — skipping Full Suite trigger.'); + core.setOutput('changed_files', String(pr.changed_files)); + if (!hasReady) core.info('No ready label — skipping merge-gate trigger.'); - name: Cancel previous Buildkite builds if: steps.check.outputs.has_ready == 'true' env: BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }} PR_BRANCH: ${{ github.event.pull_request.head.ref }} + PR_NUMBER: ${{ github.event.pull_request.number }} run: | - # Find running builds for this branch with TEST_SCOPE=full and cancel them - builds=$(curl -sS -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \ - "https://api.buildkite.com/v2/organizations/${{ vars.BUILDKITE_ORG_SLUG }}/pipelines/${{ vars.BUILDKITE_PIPELINE_SLUG }}/builds?branch=${PR_BRANCH}&state=running,scheduled" \ - | jq -r '.[] | select(try (.env.TEST_SCOPE == "full") catch false) | .number') + # Match both branch and PR number: forks can reuse the same branch name. + builds=$(curl -sS --get -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \ + --data-urlencode "branch=$PR_BRANCH" \ + --data-urlencode "state=running,scheduled" \ + "https://api.buildkite.com/v2/organizations/${{ vars.BUILDKITE_ORG_SLUG }}/pipelines/${{ vars.BUILDKITE_PIPELINE_SLUG }}/builds" \ + | jq -r --arg pr_number "$PR_NUMBER" \ + '.[] | select((.env.TEST_SCOPE? == "merge") and (.env.PR_NUMBER? == $pr_number)) | .number') for build_num in $builds; do echo "Cancelling Buildkite build #$build_num" curl -sS -X PUT -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \ "https://api.buildkite.com/v2/organizations/${{ vars.BUILDKITE_ORG_SLUG }}/pipelines/${{ vars.BUILDKITE_PIPELINE_SLUG }}/builds/${build_num}/cancel" done - # Checks out the BASE branch (default for pull_request_target), so PR - # authors cannot tamper with the gate script. - - name: Checkout gate script + # Check out the immutable BASE SHA: pull_request_target must never run a + # planner or gate script from the untrusted PR head. + - name: Checkout trusted merge planner if: steps.check.outputs.has_ready == 'true' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.base.sha }} + persist-credentials: false + + - name: Collect changed paths + if: steps.check.outputs.has_ready == 'true' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EXPECTED_CHANGED_FILES: ${{ steps.check.outputs.changed_files }} + run: | + set -euo pipefail + changed_json="$RUNNER_TEMP/merge-changed-files.json" + changed_paths="$RUNNER_TEMP/merge-changed-paths.txt" + if gh api --paginate --slurp \ + "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?per_page=100" \ + > "$changed_json"; then + observed=$(jq '[.[][] | .filename] | unique | length' "$changed_json") + if [ "$observed" = "$EXPECTED_CHANGED_FILES" ]; then + jq -r '.[][] | .filename, (.previous_filename // empty)' "$changed_json" \ + | sort -u > "$changed_paths" + else + echo "::warning::Changed-file API returned $observed of $EXPECTED_CHANGED_FILES paths; selecting all merge lanes." + echo '__FASTVIDEO_CI_PLAN_ALL__' > "$changed_paths" + fi + else + echo "::warning::Changed-file API failed; selecting all merge lanes." + echo '__FASTVIDEO_CI_PLAN_ALL__' > "$changed_paths" + fi + + - name: Select minimal merge tests + id: plan + if: steps.check.outputs.has_ready == 'true' + run: | + python3 .github/scripts/plan_merge_ci.py \ + --paths-file "$RUNNER_TEMP/merge-changed-paths.txt" \ + --github-output "$GITHUB_OUTPUT" \ + --summary-file "$GITHUB_STEP_SUMMARY" - name: Wait for pre-commit and docs build if: steps.check.outputs.has_ready == 'true' @@ -66,7 +109,7 @@ jobs: PR_NUMBER: ${{ github.event.pull_request.number }} run: bash .github/scripts/gate_full_suite.sh - - name: Trigger Buildkite Full Suite + - name: Trigger Buildkite merge gate if: steps.check.outputs.has_ready == 'true' env: BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }} @@ -76,6 +119,10 @@ jobs: PR_TITLE: ${{ github.event.pull_request.title }} BK_ORG: ${{ vars.BUILDKITE_ORG_SLUG }} BK_PIPELINE: ${{ vars.BUILDKITE_PIPELINE_SLUG }} + MERGE_TEST_PLAN: ${{ steps.plan.outputs.merge_test_plan }} + MERGE_GOLDEN_TESTS: ${{ steps.plan.outputs.merge_golden_tests }} + MERGE_SSIM_TESTS: ${{ steps.plan.outputs.merge_ssim_tests }} + MERGE_PLAN_LABEL: ${{ steps.plan.outputs.merge_plan_label }} run: | curl -sS --fail-with-body -X POST \ "https://api.buildkite.com/v2/organizations/${BK_ORG}/pipelines/${BK_PIPELINE}/builds" \ @@ -84,8 +131,11 @@ jobs: --data-raw "$(jq -n \ --arg commit "$PR_SHA" \ --arg branch "$PR_BRANCH" \ - --arg message "Full Suite for PR #${PR_NUMBER}" \ + --arg message "Merge gate [${MERGE_PLAN_LABEL}] for PR #${PR_NUMBER}" \ --arg pr_title "$PR_TITLE" \ + --arg merge_test_plan "$MERGE_TEST_PLAN" \ + --arg merge_golden_tests "$MERGE_GOLDEN_TESTS" \ + --arg merge_ssim_tests "$MERGE_SSIM_TESTS" \ --argjson pr_id "$PR_NUMBER" \ '{ commit: $commit, @@ -95,8 +145,11 @@ jobs: pull_request_id: $pr_id, pull_request_base_branch: "main", env: { - TEST_SCOPE: "full", + TEST_SCOPE: "merge", FULL_SUITE: "true", + MERGE_TEST_PLAN: $merge_test_plan, + MERGE_GOLDEN_TESTS: $merge_golden_tests, + MERGE_SSIM_TESTS: $merge_ssim_tests, PR_NUMBER: ($pr_id | tostring), PR_TITLE: $pr_title } diff --git a/.github/workflows/community-welcome.yml b/.github/workflows/community-welcome.yml index 65610e132..8aac52dbe 100644 --- a/.github/workflows/community-welcome.yml +++ b/.github/workflows/community-welcome.yml @@ -38,17 +38,17 @@ jobs: **How our CI works:** - PRs run a two-tier CI system: + PRs run a three-tier CI system: 1. **Pre-commit** — formatting (yapf), linting (ruff), type checking (mypy). Runs immediately on every PR. - 2. **Fastcheck** — core GPU tests (encoders, VAEs, transformers, kernels, unit tests). Runs automatically via Buildkite on relevant file changes (~10-15 min). - 3. **Full Suite** — integration tests, training pipelines, SSIM regression. Runs only when a reviewer adds the `ready` label. + 2. **Fastcheck** — six core GPU lanes run automatically via Buildkite (~10-15 min). + 3. **Merge gate** — a reviewer adds `ready`; changed paths select only the relevant integration, training, golden, or SSIM coverage. **Before your PR is reviewed:** - [ ] `pre-commit run --all-files` passes locally - [ ] You've added or updated tests for your changes - [ ] The PR description explains what and why - If pre-commit fails, a bot comment will explain how to fix it. Fastcheck and Full Suite results appear in the Checks section below. + If pre-commit fails, a bot comment will explain how to fix it. Fastcheck and merge-gate results appear in the Checks section below. **Useful links:** - [Contributing Guide](https://hao-ai-lab.github.io/FastVideo/contributing/overview/) diff --git a/.github/workflows/infra-build-image.yml b/.github/workflows/infra-build-image.yml index d2d477958..986779168 100644 --- a/.github/workflows/infra-build-image.yml +++ b/.github/workflows/infra-build-image.yml @@ -13,6 +13,11 @@ on: required: false default: false type: boolean + build_ci_runner_image: + description: 'Build the ARM64 CUDA 13 CI runner image (sm_100)' + required: false + default: false + type: boolean # Auto-rebuild the CUDA images when a repository-controlled image input # changes on main. This includes the trusted SM89 kernel artifact's source, # metadata/key helper, ABI dependency metadata, and build orchestration. @@ -198,6 +203,28 @@ jobs: docker buildx imagetools create "${TAG_ARGS[@]}" "${IMAGE_REFS[@]}" docker buildx imagetools inspect "${TAGS[0]}" + # The CI runner is ARM64 like DGX Spark, but targets sm_100 rather than sm_121. + # Publish a single-architecture variant so the self-hosted CI runner can reuse + # the exact prebuilt kernel instead of compiling it in every job. + build-ci-runner-image: + if: ${{ (github.event_name == 'push' && github.repository == 'hao-ai-lab/FastVideo') || github.event.inputs.build_ci_runner_image == 'true' }} + uses: ./.github/workflows/_template-build-image.yml + with: + python_version: '3.12' + dockerfile_path: docker/Dockerfile + tag_suffix: py3.12-cuda13.0.0-sm100 + runner: ubuntu-24.04-arm + architecture: arm64 + build_args: | + PYTHON_VERSION=3.12 + CUDA_VERSION=13.0.0 + UV_TORCH_BACKEND=cu130 + TORCH_CUDA_ARCH_LIST=10.0 + CMAKE_BUILD_PARALLEL_LEVEL=1 + FLASH_ATTN_WHEEL_TAG=cu130torch2.12 + FLASH_ATTN_WHEEL_RELEASE_ARM64=https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.22 + secrets: inherit + # Dreamverse matrix: {backend, UI} x {12.6.3, 13.0.0}, Python 3.12. Torch backend # matches the base CUDA (cu126 / cu130). Keep these images amd64-only until the # required FA4 dependency stack is available and validated on arm64. diff --git a/docker/Dockerfile b/docker/Dockerfile index ca395292d..1fa6dff1f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -89,11 +89,29 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ zsh \ vim \ curl \ + ffmpeg \ + libgl1 \ + libglib2.0-0 \ + libx11-dev \ gcc-11 \ g++-11 \ clang-11 \ + cmake \ + pkg-config \ + build-essential \ + libssl-dev \ && rm -rf /var/lib/apt/lists/* +# Rust toolchain: some dependencies only ship sdists on aarch64 and need cargo +# to build. The dormant legacy Modal image layers the identical apt set + +# rustup on top of this image (fastvideo/tests/modal/pr_test.py); baking both +# here keeps its manual rollback path reproducible without changing the Slurm +# runner's package surface. +RUN set -o pipefail && \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal && \ + /root/.cargo/bin/cargo --version && /root/.cargo/bin/rustc --version +ENV PATH=/root/.cargo/bin:${PATH} + # Set up C++20 compilers for ThunderKittens RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 --slave /usr/bin/g++ g++ /usr/bin/g++-11 @@ -138,6 +156,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \ source /opt/venv/bin/activate && \ uv pip install --upgrade pip && \ uv pip install --excludes docker/uv-excludes ".[dev]" && \ + python -c "import cv2; print('OpenCV', cv2.__version__)" && \ PYTAG=cp$(echo "${PYTHON_VERSION}" | tr -d .) && \ case "${TARGETARCH:-amd64}" in \ amd64) \ @@ -169,26 +188,21 @@ RUN --mount=type=cache,target=/opt/uv/cache \ # flash_attn/__init__.py), so FA2/varlen/bert_padding stay from the install above; # rmtree clears the wheel's stale cute files first to avoid an install conflict. # Then verify both survive so a broken overlay fails the build instead of shipping -# an FA2-less image. x86 only: the FA4 stack (quack-kernels etc.) is unvalidated on -# arm64 / GB10 (sm_121), so there we skip the overlay; FA4 is opt-in -# (FASTVIDEO_FA4=1) and errors if set without the overlay, so leave it unset on -# arm64 and the image runs FA3/FA2 as usual. +# an FA2-less image. The pinned stack is validated on ARM64 GB200 (sm_100) as well +# as x86; FA4 remains opt-in through FASTVIDEO_FA4=1 so lanes with FA2 baselines +# keep their existing numerics. RUN --mount=type=cache,target=/opt/uv/cache \ source $HOME/.local/bin/env && \ source /opt/venv/bin/activate && \ - if [ "${TARGETARCH}" = "arm64" ]; then \ - echo "Skipping FA4 cute overlay on arm64 (FA4 stack unvalidated there; do not set FASTVIDEO_FA4)"; \ - else \ - python -c "import glob, shutil; [shutil.rmtree(d, ignore_errors=True) for d in glob.glob('/opt/venv/lib/python*/site-packages/flash_attn/cute')]" && \ - uv pip install "flash-attn-4 @ git+https://github.com/Dao-AILab/flash-attention.git@${FA4_CUTE_REF}#subdirectory=flash_attn/cute" && \ - python -c "import flash_attn; assert hasattr(flash_attn, 'flash_attn_func'), 'FA2 was clobbered by the cute overlay'; import flash_attn.cute; print('FA2 + FA4 cute OK')"; \ - fi + python -c "import glob, shutil; [shutil.rmtree(d, ignore_errors=True) for d in glob.glob('/opt/venv/lib/python*/site-packages/flash_attn/cute')]" && \ + uv pip install "flash-attn-4 @ git+https://github.com/Dao-AILab/flash-attention.git@${FA4_CUTE_REF}#subdirectory=flash_attn/cute" && \ + python -c "import flash_attn; assert hasattr(flash_attn, 'flash_attn_func'), 'FA2 was clobbered by the cute overlay'; import flash_attn.cute; print('FA2 + FA4 cute OK')" COPY . . # Build immutable FastVideo kernel wheels for the published image. The requested # architecture remains installed for normal image users; amd64 images also carry -# an SM89 artifact so the predominant L40S Modal lanes can reuse it exactly. +# an SM89 artifact for L40S users and the dormant legacy rollback path. ARG FASTVIDEO_KERNEL_PREBUILT_DIR=/opt/fastvideo-kernel-prebuilt RUN --mount=type=cache,target=/opt/uv/cache \ source $HOME/.local/bin/env && \ diff --git a/docs/contributing/ci_architecture.md b/docs/contributing/ci_architecture.md index 21e9abb45..1a630dd9f 100644 --- a/docs/contributing/ci_architecture.md +++ b/docs/contributing/ci_architecture.md @@ -6,7 +6,8 @@ lives in [Testing](testing.md). ## Overview -FastVideo splits validation across GitHub Actions, Buildkite, and Modal: +FastVideo splits validation across GitHub Actions, Buildkite, Slinky Slurm, +and Mergify: ```text PR opened or updated @@ -16,17 +17,24 @@ PR opened or updated | style, lint, type, spelling, Markdown, workflow syntax, filenames | |-- Tier 2: Fastcheck - | Buildkite orchestrates Modal GPU jobs - | path-filtered component and unit checks + | Buildkite schedules six lanes on the Slinky Slurm cluster + | encoder, VAE, transformer, kernel, unit, DreamVerse | - |-- /merge, /test full, or ready label + |-- /merge or ready label | - `-- Tier 3: Full Suite - Buildkite orchestrates Modal GPU jobs - path-filtered integration, SSIM, training, eval, and performance checks + `-- Tier 3: Change-aware merge gate + trusted base-branch planner classifies the complete PR diff + Buildkite adds only relevant integration, quality, training, + API, or performance lanes on Slinky Slurm | pass -> Mergify squash-merges when all merge conditions pass fail -> fix, push, and re-run + + |-- /test full + `-- Explicit all-20-lane diagnostic run + + `-- weekly schedule on main + `-- Complete four-GPU SSIM matrix ``` CI is not one monolithic job: @@ -34,10 +42,31 @@ CI is not one monolithic job: - GitHub Actions owns pre-commit, slash-command handling, aggregate status updates, docs deployment, image builds, package publishing, and community automations. -- Buildkite owns the GPU test pipeline and path filtering. -- Modal owns the actual GPU execution environment for test jobs. +- Buildkite owns the GPU test graph, statuses, and trusted dispatch control + plane. Its agent runs on the Slurm login plane; it does not execute test + payloads. +- Slinky Slurm is the only active CI compute backend. A host-owned dispatcher + leases GPUs from a persistent four-GPU allocation and runs each lane in an + isolated Enroot container at the immutable PR SHA. - Mergify owns merge protection, labeling, and the final squash merge. +The old files under `fastvideo/tests/modal/` are retained as dormant manual +rollback code. `.buildkite/scripts/pr_test.sh` rejects Buildkite invocations, +and no pipeline or slash-command route calls Modal. + +Three Buildkite entry pipelines share the validated graph: + +| Pipeline | Trigger | Scope | +|---|---|---| +| `pr-fastcheck` | Automatic pull-request webhook | Six Fastcheck lanes | +| `ci` | `/merge`, `ready`, schedules, and `/test` API builds | Change-aware merge gates, scheduled SSIM, explicit Full Suite, Fastcheck reruns, or one direct lane | +| `fastvideo-performance-lane` | Weekly scheduler | Direct performance lane | + +Each entry pipeline starts with the same trusted `pipeline-upload` job on the +`ci-runner` queue. The `ci` pipeline's incoming GitHub webhook is disabled; +otherwise it would duplicate the automatic `pr-fastcheck` build. API and +scheduled builds continue to work with webhook processing disabled. + ## CI Tiers ### Tier 1: Pre-commit @@ -70,34 +99,37 @@ debugging a hook implementation. | Attribute | Value | |---|---| | Triggered by | Buildkite PR builds with `TEST_SCOPE=fastcheck` or unset | -| Runner | Buildkite agent that launches Modal GPU jobs | +| Compute | Slinky Slurm (`ci-runner` queue) | | Definition | `.buildkite/pipeline.yml` | -| Entrypoint | `.buildkite/scripts/pr_test.sh` -> `fastvideo/tests/modal/pr_test.py` | +| Entrypoint | Trusted host driver -> `.buildkite/scripts/unit_test.sh` or `.buildkite/scripts/lanes/*.sh` | -Fastcheck uses Buildkite's `monorepo-diff` plugin. Jobs whose watched paths did -not change are skipped and do not block the aggregate `fastcheck-passed` -status. +Fastcheck always schedules these six lanes: encoder, VAE, transformer, custom +kernels, unit tests, and DreamVerse. Static steps replace the former +host-side path-filter plugin: the login plane never checks out or executes PR +code. -| Buildkite label | `TEST_TYPE` | Main watched paths | -|---|---|---| -| Encoder Tests | `encoder` | `fastvideo/models/encoders/**`, `fastvideo/models/loader/**`, `fastvideo/tests/encoders/**`, `pyproject.toml`, `docker/Dockerfile` | -| VAE Tests | `vae` | `fastvideo/models/vaes/**`, `fastvideo/models/loader/**`, `fastvideo/tests/vaes/**`, `pyproject.toml`, `docker/Dockerfile` | -| Transformer Tests | `transformer` | `fastvideo/models/dits/**`, `fastvideo/models/loader/**`, `fastvideo/tests/transformers/**`, `fastvideo/layers/**`, `fastvideo/attention/**`, `pyproject.toml`, `docker/Dockerfile` | -| Kernel Tests | `kernel_tests` | `fastvideo-kernel/**`, `pyproject.toml`, `docker/Dockerfile` | -| Unit Tests | `unit_test` | `fastvideo/**`, `.buildkite/**`, `.github/**`, `pyproject.toml`, `docker/Dockerfile` | -| DreamVerse App Tests | `dreamverse_app` | `apps/dreamverse/**`, `pyproject.toml` | - -### Tier 3: Full Suite +### Tier 3: Change-Aware Merge Gate | Attribute | Value | |---|---| -| Triggered by | `/merge`, adding `ready`, `/test full`, or a new push to a PR that already has `ready` | -| Runner | Buildkite agent that launches Modal GPU jobs | +| Triggered by | `/merge`, adding `ready`, or a new push to a PR that already has `ready` | +| Compute | Slinky Slurm only (`ci-runner` queue) | | Definition | `.buildkite/pipeline.yml` | -| Entrypoint | `.buildkite/scripts/pr_test.sh` -> `fastvideo/tests/modal/pr_test.py` | +| Entrypoint | `/opt/fastvideo-ci-runner/run-ci` (`run-unit` is a compatibility wrapper) | -Full Suite is also path-filtered. It validates broader behavior before Mergify -can merge a PR. +Fastcheck is the universal six-lane baseline. The merge gate does not repeat +those jobs: it classifies every changed path and adds only the relevant lanes +from the fourteen-lane integration set below. Selected jobs are hard gates; +there are no soft-fail hardware lanes. A documentation-only PR can therefore +finish its merge build after the trusted uploader, while model-family changes +typically add focused golden-gate and SSIM files and a training-only change +adds only its owning training lane. + +`.github/scripts/plan_merge_ci.py` is the canonical path policy. It runs from +the immutable base SHA under `pull_request_target`; PR code is never executed +on the GitHub runner. The changed-file list includes both sides of renames. An +API failure, truncated response, empty list, unknown build input, or unknown +source path fails closed to all fourteen integration lanes. A `ready`-labeled PR does not hit Buildkite immediately: `ci-trigger-full-suite.yml` first runs `.github/scripts/gate_full_suite.sh`, @@ -106,25 +138,100 @@ head. A red cheap check blocks the suite (fail closed; the next push re-arms it), while a GitHub outage or a >25 min wait lets it run anyway (fail open). `/test full` bypasses the gate. -| Buildkite label | `TEST_TYPE` | Main watched paths | -|---|---|---| -| SSIM Tests | `ssim` | `fastvideo/**/*.py`, `pyproject.toml`, `docker/Dockerfile` | -| LoRA Inference Tests | `inference_lora` | LoRA tests, loader, transformer tests, pipelines, LoRA layers | -| LoRA Extraction Tests | `lora_extraction` | LoRA extraction scripts/tests, loader, training utilities, LoRA layers | -| Training Tests | `training` | `fastvideo/**`, `pyproject.toml`, `docker/Dockerfile` | -| Distillation DMD Tests | `distillation_dmd` | `fastvideo/training/*distillation_pipeline.py` | -| Self-Forcing Tests | `self_forcing` | self-forcing distillation pipeline and tests | -| LoRA Training Tests | `training_lora` | `fastvideo/**`, `pyproject.toml`, `docker/Dockerfile` | -| Training Tests VSA | `training_vsa` | `fastvideo/**`, `fastvideo-kernel/**`, `pyproject.toml`, `docker/Dockerfile` | -| Inference Tests VMoBA | `inference_vmoba` | `fastvideo-kernel/**`, `fastvideo/attention/backends/vmoba.py` | -| Performance Tests | `performance` | DiTs, pipelines, attention, layers, worker, entrypoints, performance tests/configs | -| API Server Tests | `api_server` | OpenAI entrypoints, serve CLI, OpenAI API integration test | -| Train Framework Tests | `train_framework` | `fastvideo/train/**`, train model/method tests, model loader, DiTs | -| Eval Metrics Tests | `eval` | `fastvideo/eval/**`, `fastvideo/tests/eval/**`, `pyproject.toml`, `docker/Dockerfile` | +The complete static graph remains available through `/test full`; path +selection never deletes or dynamically invents a Buildkite step. + +| Lane | Public `TEST_TYPE` | GPUs | Typical merge trigger | +|---|---|---:|---| +| Encoder | `encoder` | 1 | Universal Fastcheck | +| VAE | `vae` | 1 | Universal Fastcheck | +| Transformer | `transformer` | 1 | Universal Fastcheck | +| Kernel | `kernel_tests` | 1 | Universal Fastcheck | +| Unit | `unit_test` | 1 | Universal Fastcheck | +| DreamVerse | `dreamverse_app` | 1 | Universal Fastcheck | +| Golden gate | `golden_gate` | 1 | Model, pipeline, attention, layer, or output changes | +| SSIM | `ssim` | 4 | Matching model/SSIM paths; focused files when possible | +| LoRA inference | `inference_lora` | 1 | LoRA inference/shared LoRA paths | +| LoRA extraction | `lora_extraction` | 1 | LoRA extraction/shared LoRA paths | +| Vanilla training | `training` | 4 | Legacy vanilla/shared training paths | +| DMD distillation | `distillation_dmd` | 2 | DMD/shared training paths | +| Self-forcing | `self_forcing` | 2 | Self-forcing/shared training paths | +| LoRA training | `training_lora` | 2 | LoRA/shared training paths | +| VSA training | `training_vsa` | 2 | VSA/shared training paths | +| VMoBA inference | `inference_vmoba` | 1 | VMoBA backend/config paths | +| Performance | `performance` | 2 | Performance tests or benchmark policy | +| API server | `api_server` | 1 | API, worker, or server entrypoint paths | +| Modular train framework | `train_framework` | 1 | `fastvideo/train/` and its tests | +| Eval metrics | `eval` | 1 | `fastvideo/eval/` and its tests | + +Golden-gate and SSIM selections are basenames, not arbitrary pytest arguments. +The private host checks the comma-separated allowlist before staging, and the +container checks it again before invoking pytest. Shared quality-harness +changes still run the complete owning lane. The full SSIM matrix also runs on +`main` every Sunday at 05:00 UTC through `ci-scheduled-ssim.yml`; `/test ssim` +and `/test full` remain available for deliberate complete runs. + +The four Buildkite workers may accept multiple jobs concurrently. The +agent-owned lease broker packs their requested GPU counts onto one persistent +four-GPU Slurm allocation and waits when capacity is full. A four-GPU lane +such as SSIM or vanilla training owns the whole tray; two two-GPU lanes or up +to four one-GPU lanes can overlap without sharing devices. SSIM and vanilla +training also share the `fastvideo/slinky/whole-tray` Buildkite concurrency +group. That keeps the second whole-tray lane in Buildkite instead of consuming +an agent and its command timeout while the first lane waits for four free GPUs. +Because packed Enroot containers share the node network namespace, each GPU +lease receives its own 100-port rendezvous range. Tests preserve the +runner-assigned `MASTER_PORT`, and parallel SSIM tasks use distinct offsets +inside that range. See [Performance Benchmarks](performance_benchmarks.md) for the performance lane's thresholds, rolling baseline, artifacts, and reseeding process. +## `/merge` Request Flow + +The Buildkite agent and Slurm have deliberately separate responsibilities: + +```text +/merge PR comment + -> GitHub verifies write permission and refreshes the `ready` label + -> base-branch `ci-trigger-full-suite` workflow fetches the PR file list, + computes MERGE_TEST_PLAN plus focused golden/SSIM basenames, and gates on + cheap checks + -> sends the PR SHA to pipeline `ci` with TEST_SCOPE=merge and FULL_SUITE=true + -> trusted `pipeline-upload` job on queue `ci-runner` + fetch exact-SHA .buildkite/pipeline.yml + normalize + validate the complete static 20-lane policy and conditions + upload static Buildkite steps + -> each accepted step reaches the host policy hook + validate org/repo/SHA/ref/step key/command/scope/timeout + skip checkout on the login plane + stage a mode-0600 request on Lustre + lease 1-4 GPUs and attach an `srun` step to the Slinky tray + -> Enroot worker container + clone and verify the exact PR SHA + install the lane's project extras and cached kernel + run the repository-owned lane script + write numeric exit status and approved artifacts + -> trusted host returns that status to Buildkite + -> Buildkite publishes `full-suite-passed` only when every selected lane passes +``` + +The pipeline uploader and dispatcher run on the Slurm login plane, but those +are control-plane operations only. Python tests, model loading, CUDA kernels, +Node/Playwright checks, inference, training, SSIM generation, and performance +benchmarks all execute in Slurm allocations. + +PR-controlled values never become host commands. The host policy accepts only +the pinned pipeline uploader or a known lane tuple. It rejects plugins, +artifact globs, shell injection variables, non-immutable commits, and unknown +commands before checkout. Hugging Face credentials are added only for lanes +that declare them, passed through a mode-0600 request file, and removed before +the PR payload starts. Active training lanes keep W&B offline and do not stage +a W&B credential. The ARM64 image includes the pinned FA4 CuTe overlay validated +on GB200. SSIM opts into FA4 to preserve its reference-video numerics; lanes +with FA2 baselines keep `FASTVIDEO_FA4=0`. Performance artifacts are relayed +afterward by the trusted host from an allowlisted directory and extension set. + ## Slash Commands Slash commands are handled by `.github/workflows/ci-slash-commands.yml`. @@ -132,7 +239,7 @@ Repository write permission is required. | Command | Effect | |---|---| -| `/merge` | Adds `ready` and triggers Full Suite for the PR head branch. | +| `/merge` | Adds `ready` and triggers the path-aware merge gate for the PR head. | | `/test full` | Runs the whole Full Suite with `TEST_SCOPE=full`. | | `/test fastcheck` | Runs the whole Fastcheck suite with `TEST_SCOPE=fastcheck`. | | `/test pre-commit` | Re-runs the pre-commit workflow on the PR merge ref. | @@ -149,6 +256,7 @@ Valid direct test names: | `/test unit` | `unit_test` | | `/test dreamverse` | `dreamverse_app` | | `/test ssim` | `ssim` | +| `/test golden-gate` | `golden_gate` | | `/test training` | `training` | | `/test lora-inference` | `inference_lora` | | `/test lora-training` | `training_lora` | @@ -162,12 +270,18 @@ Valid direct test names: | `/test train-framework` | `train_framework` | | `/test eval` | `eval` | +The temporary `-ci` spellings remain accepted as compatibility aliases; +they select the same Slurm lane and do not identify a second backend. + When a direct test completes successfully, Buildkite posts `direct-test-completed`. `.github/workflows/ci-aggregate-status.yml` then reads the latest Buildkite statuses for the commit and updates `fastcheck-passed` or `full-suite-passed` if all jobs in that group are green. -Skipped path-filtered jobs have no status entry and do not block the aggregate. +Buildkite label emojis define the status namespace used by that aggregation: +`:microscope:` is reserved for the six Fastcheck lanes, while Full-Suite-only +lanes use `:test_tube:` or `:bar_chart:`. Each active lane has exactly one +label and therefore one status context. ## Merge Protection @@ -177,7 +291,7 @@ Mergify enforces these conditions before it squash-merges to `main`: |---|---| | `check-success~=pre-commit` | Tier 1 passed. | | `check-success=fastcheck-passed` | All triggered Fastcheck jobs passed. | -| `check-success=full-suite-passed` | All triggered Full Suite jobs passed. | +| `check-success=full-suite-passed` | The selected merge gate or explicit Full Suite passed. | | `#approved-reviews-by>=1` | At least one approving review. | | Valid title regex | PR title starts with an accepted `[type]` tag. | | `label=ready` | The PR has entered the merge flow. | @@ -226,29 +340,40 @@ Process labels: | Label | Who sets it | Meaning | |---|---|---| -| `ready` | `/merge` or maintainer action | Triggers/keeps Full Suite active and enables auto-merge. | +| `ready` | `/merge` or maintainer action | Triggers/keeps the change-aware merge gate active and enables auto-merge. | | `needs-rebase` | Mergify | PR has merge conflicts. | | `do-not-merge` | Maintainer | Blocks merge regardless of CI status. | -## Modal Test Entrypoints +## Slurm Lane Entrypoints -All Buildkite test jobs go through `.buildkite/scripts/pr_test.sh`, which: +Every active test selection lives in `.buildkite/scripts/unit_test.sh` or a +focused `.buildkite/scripts/lanes/.sh`. The private, agent-owned lane +table binds each internal `*_ci` type to that script, its GPU count, wall-clock +limit, dependency extras, kernel-build policy, secrets, and artifacts. The +internal suffix is an implementation detail; there is only one active backend. -1. Reads Buildkite secrets for Modal, Hugging Face, and W&B when needed. -2. Selects a Modal function based on `TEST_TYPE`. -3. Passes Buildkite metadata into the Modal container. -4. Runs the selected test command from `fastvideo/tests/modal/pr_test.py` or - `fastvideo/tests/modal/ssim_test.py`. -5. Uploads performance artifacts for `TEST_TYPE=performance`. +SSIM uses `fastvideo/tests/ssim/ci_runner.py` inside a single four-GPU lease. +It discovers `REQUIRED_GPUS` and `*_MODEL_TO_PARAMS` with AST parsing, then +packs independent pytest subprocesses across the visible GPUs with fail-fast +termination. Performance writes reports to a host-mounted artifact directory; +the host uploads only `.md`, `.html`, `.json`, and `.csv` files after the +container exits. + +The Modal launchers remain in the repository for manual rollback archaeology, +but they are not CI entrypoints. `pr_test.sh` rejects Buildkite calls and needs +`FASTVIDEO_ENABLE_LEGACY_MODAL_CI=1` even for a local manual invocation. If you add a new CI test category: -1. Add the Modal function in `fastvideo/tests/modal/pr_test.py` or a focused - companion module. -2. Add the `TEST_TYPE` case in `.buildkite/scripts/pr_test.sh`. -3. Add the Buildkite direct-test step and any Fastcheck/Full Suite path filters - in `.buildkite/pipeline.yml`. -4. Add or update the `/test` mapping in `.github/workflows/ci-slash-commands.yml`. +1. Add an executable `.buildkite/scripts/lanes/.sh` containing the test + payload only. +2. Add the static Buildkite step in `.buildkite/pipeline.yml`, its source/test + ownership in `.github/scripts/plan_merge_ci.py`, and the `/test` mapping in + `.github/workflows/ci-slash-commands.yml`. +3. Extend `fastvideo/tests/contract/test_ci_test_collection.py`, + `test_merge_ci_plan.py`, and the trusted private runner's lane table plus + uploader policy in the same rollout. +4. Validate on the target GB200 hardware before making the lane a merge gate. 5. Document the lane here and link any domain-specific authoring guide. ## CD And Release Workflows @@ -282,13 +407,18 @@ the explicit `py3.12-cuda13.0.0-latest` tag. This publication policy does not change the unparameterized `docker/Dockerfile` build defaults, which remain CUDA 13 and `cu130`. -Published amd64 development images keep their configured Hopper kernel wheel -installed and also carry an immutable SM89 wheel under -`/opt/fastvideo-kernel-prebuilt`. Modal PR and SSIM jobs select the exact -source, ABI, and GPU-architecture match from that directory, so L40S jobs reuse -the trusted image artifact while kernel-changing PRs still build locally. Once -a kernel or artifact-key change reaches `main`, the image workflow republishes -the matching trusted artifact before later jobs consume the updated image tag. +Published development images carry architecture-specific kernel wheels under +`/opt/fastvideo-kernel-prebuilt`. The Slurm worker selects the exact source, +ABI, and GPU-architecture match, so normal lanes reuse the trusted artifact +while kernel-changing PRs still build locally. Once a kernel or artifact-key +change reaches `main`, the image workflow republishes the matching artifact +before later jobs consume the updated image pin. + +The same workflow publishes a single-architecture ARM64, CUDA 13, SM100 image +for the self-hosted CI runner under the +`py3.12-cuda13.0.0-sm100-{latest,sha-*}` tags. It carries the matching prebuilt +kernel wheel so runner jobs can validate and install the exact source and ABI +match instead of recompiling it in every lane. The optional Dreamverse matrix builds backend and UI images for CUDA 12.6 and CUDA 13 on `amd64`. Dreamverse remains `amd64`-only because its FA4 dependency @@ -320,15 +450,17 @@ The reusable implementation lives in | `.github/mergify.yml` | Merge protection, PR title validation, PR labels, conflict labels, auto-merge | | `.github/workflows/ci-precommit.yml` | Tier 1 pre-commit | | `.github/workflows/ci-slash-commands.yml` | `/merge` and `/test` handling | -| `.github/workflows/ci-trigger-full-suite.yml` | Full Suite trigger for `ready` PRs and new pushes to ready PRs | -| `.github/workflows/ci-aggregate-status.yml` | Aggregate Fastcheck/Full Suite commit statuses | -| `.buildkite/pipeline.yml` | Buildkite test graph and path filters | -| `.buildkite/scripts/pr_test.sh` | Buildkite-to-Modal test dispatcher | -| `fastvideo/tests/modal/pr_test.py` | Modal functions for most GPU CI lanes | -| `fastvideo/tests/modal/ssim_test.py` | Modal functions and partitioning for SSIM | +| `.github/workflows/ci-trigger-full-suite.yml` | Change-aware merge-gate trigger for `ready` PRs and new pushes | +| `.github/workflows/ci-scheduled-ssim.yml` | Weekly complete SSIM trigger on `main` | +| `.github/scripts/plan_merge_ci.py` | Trusted changed-path to integration-lane planner | +| `.github/workflows/ci-aggregate-status.yml` | Aggregate Fastcheck and explicit Full Suite direct-rerun statuses | +| `.buildkite/pipeline.yml` | Static 20-lane Slurm Buildkite graph | +| `.buildkite/scripts/unit_test.sh`, `.buildkite/scripts/lanes/*.sh` | Active Slurm lane payloads | +| `fastvideo/tests/ssim/ci_runner.py` | Four-GPU Slurm SSIM scheduler | +| `.buildkite/scripts/pr_test.sh`, `fastvideo/tests/modal/*.py` | Dormant manual Modal rollback path (disabled in Buildkite) | | `.buildkite/performance-benchmarks/tests/*.json` | Performance benchmark configs and thresholds | | `.github/workflows/infra-docs.yml` | Docs build and GitHub Pages deploy | -| `.github/workflows/infra-build-image.yml` | Automatic CUDA matrix and manual Docker image builds | +| `.github/workflows/infra-build-image.yml` | CUDA matrix, CI runner image, and manual Docker image builds | | `.github/workflows/publish-fastvideo.yml` | FastVideo PyPI publishing | | `.github/workflows/publish-kernel.yml` | FastVideo kernel PyPI publishing | | `.github/workflows/publish-comfyui.yml` | ComfyUI registry publishing | diff --git a/docs/contributing/performance_benchmarks.md b/docs/contributing/performance_benchmarks.md index 886945722..b2405eb42 100644 --- a/docs/contributing/performance_benchmarks.md +++ b/docs/contributing/performance_benchmarks.md @@ -23,8 +23,8 @@ It serves three audiences: pytest fastvideo/tests/performance/ -vs # Optional: compare against the rolling HF baseline. -# PERF_REPORTS_DIR defaults to /root/data/perf_reports for Modal/CI, so -# override it when running outside the container. +# PERF_REPORTS_DIR defaults to /root/data/perf_reports in a CI container, so +# override it for a local run. PERF_REPORTS_DIR=/tmp/fastvideo_perf_reports \ python fastvideo/tests/performance/compare_baseline.py @@ -459,16 +459,18 @@ FlashInfer, Cutlass DSL, SageAttention, Triton, and xFormers when installed. | `FASTVIDEO_FA4` | `0` | `test_inference_performance.py` | FlashAttention-4 toggle included in `software_profile_id`. | | `FASTVIDEO_PERFORMANCE_PROFILE_VERSION` | unset | `test_inference_performance.py` | Optional explicit software cohort/profile version included in `software_profile_id`. | | `IMAGE_VERSION` | unset | `test_inference_performance.py` | CI container image/profile version included in `software_profile_id` when available. | -| `FASTVIDEO_CONTAINER_IMAGE_REF` | unset | `pr_test.py`, `launch_l40s_job.py`, `test_inference_performance.py` | Resolved CI container image ref or digest recorded in `environment_metadata` for audit without changing `software_profile_id`. | +| `FASTVIDEO_CONTAINER_IMAGE_REF` | unset | Slurm runner, `test_inference_performance.py` | Pinned CI container image digest recorded in `environment_metadata` and `software_profile_id`. | | `FASTVIDEO_STAGE_LOGGING` | set by the pytest test | `test_inference_performance.py` | Enables pipeline stage timing capture for component metrics during benchmark runs. | ## CI integration -The performance step can run on demand with `/test performance` and as part of -the Full Suite (see [CI/CD Architecture](ci_architecture.md)). The Modal entry -point is `fastvideo/tests/modal/pr_test.py:run_performance_tests` and the -Buildkite artifact upload is in -`.buildkite/scripts/pr_test.sh:upload_performance_artifacts`. +The performance step can run on demand with `/test performance`, through a +merge gate when performance tests or benchmark policy changed, and as part of +an explicit `/test full` run (see [CI/CD Architecture](ci_architecture.md)). +The weekly `fastvideo-performance-lane` schedule runs the same Slurm payload. +The active entry point is `.buildkite/scripts/lanes/performance.sh`; the +trusted host dispatcher relays its allowlisted reports to Buildkite after the +isolated container exits. Each performance build runs pytest first. PR and direct runs only continue to `compare_baseline.py` when that fixed-threshold phase passes; if pytest fails, diff --git a/docs/contributing/pull_requests.md b/docs/contributing/pull_requests.md index d107b595b..02f682916 100644 --- a/docs/contributing/pull_requests.md +++ b/docs/contributing/pull_requests.md @@ -42,19 +42,20 @@ The important process labels are: | Label | Meaning | |---|---| -| `ready` | The PR is ready for Full Suite and auto-merge consideration. | +| `ready` | The PR is ready for the change-aware merge gate and auto-merge consideration. | | `needs-rebase` | The PR has merge conflicts with `main`. | | `do-not-merge` | A maintainer has blocked merge. | ## CI Summary -FastVideo has three validation tiers: +FastVideo has three routine validation tiers plus an explicit full diagnostic: | Tier | Runs when | What it does | |---|---|---| | Pre-commit | Pull requests and `/test pre-commit` | Formatting, linting, typing, spelling, Markdown, workflow syntax, filename checks | -| Fastcheck | PR Buildkite builds | Path-filtered component and unit checks on Modal GPU runners | -| Full Suite | `/merge`, `ready`, `/test full`, or new pushes to ready PRs | Path-filtered integration, SSIM, training, eval, API, and performance checks | +| Fastcheck | PR Buildkite builds | Six component, kernel, unit, and app lanes on Slinky Slurm | +| Merge gate | `/merge`, `ready`, or new pushes to ready PRs | Only path-relevant integration lanes on Slinky Slurm; Fastcheck remains the baseline | +| Full Suite | `/test full` | Explicit all-twenty-lane diagnostic run on Slinky Slurm | See [CI/CD Architecture](ci_architecture.md#ci-tiers) for exact jobs, path filters, and workflow files. @@ -66,10 +67,11 @@ filters, and workflow files. 3. Fix pre-commit failures locally with `pre-commit run --all-files`. 4. Wait for at least one approving review. 5. When the PR is approved and ready, comment `/merge`. -6. `/merge` adds `ready` and triggers the Full Suite for the PR branch. +6. `/merge` adds `ready`, waits for cheap checks, and triggers the minimal + path-relevant integration lanes for the PR branch. 7. If all required checks pass, Mergify squash-merges the PR to `main`. -8. If Full Suite fails, fix the regression, push again, and re-run `/merge` or - the failed test. +8. If the merge gate fails, fix the regression, push again, and re-run + `/merge`. Use a targeted `/test` command for diagnosis. Only contributors with repository write permission can use slash commands. If you are an external contributor, ask a maintainer to run `/merge` or add @@ -128,7 +130,7 @@ git push --force-with-lease Mergify removes `needs-rebase` after conflicts are resolved. -### Full Suite Fails +### Merge Gate Or Full Suite Fails The failing Buildkite step is the source of truth. Common causes are: diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index 7cd5c1060..78ebe0e9b 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -138,47 +138,31 @@ pytest fastvideo/tests/ssim/ -vs Use a machine whose GPU and backend match the reference folder you are testing. -## Modal Runs For SSIM +## Slurm CI Runs For SSIM -For CI-like SSIM execution, use `fastvideo/tests/modal/ssim_test.py`: +Comment `/test ssim` on a pull request to run the canonical four-GPU SSIM +lane on the Slinky Slurm cluster. `fastvideo/tests/ssim/ci_runner.py` +discovers the suite without importing test modules, packs independent pytest +processes across the four assigned GPUs, and stops the lane on the first +failure. + +The change-aware `/merge` planner may run only the SSIM test basenames owned +by the changed model family. Shared SSIM harness changes still select the +complete lane. Independently, `main` runs the full SSIM matrix every Sunday at +05:00 UTC so infrequently touched model families retain periodic coverage. + +For a focused developer run, invoke pytest directly and optionally select one +model from a parameterized test through `FASTVIDEO_SSIM_MODEL_ID`: ```bash -python -m modal run fastvideo/tests/modal/ssim_test.py::run_ssim_tests +pytest fastvideo/tests/ssim/test_wan_t2v_similarity.py -vs + +FASTVIDEO_SSIM_MODEL_ID=Wan2.1-T2V-1.3B-Diffusers \ +pytest fastvideo/tests/ssim/test_wan_t2v_similarity.py -vs ``` -Target specific files or model ids: - -```bash -python -m modal run fastvideo/tests/modal/ssim_test.py::run_ssim_tests \ - --test-files test_wan_t2v_similarity.py \ - --model-ids Wan2.1-T2V-1.3B-Diffusers -``` - -If `HF_API_KEY`, `HUGGINGFACE_HUB_TOKEN`, or `HF_TOKEN` is not set, the local -entrypoint fails fast. - -To export raw generated videos from Modal to the shared volume: - -```bash -python -m modal run fastvideo/tests/modal/ssim_test.py::run_ssim_tests \ - --sync-generated-to-volume -``` - -The raw export path is quality-tiered: - -- default params: `ssim_generated_videos/default//generated_videos` -- full-quality params: `ssim_generated_videos/full_quality//generated_videos` - -The printed `modal volume get` command downloads into -`./generated_videos_modal/`. Convert those outputs into local -references with `copy-local`: - -```bash -python fastvideo/tests/ssim/reference_videos_cli.py copy-local \ - --quality-tier full_quality \ - --generated-dir ./generated_videos_modal/full_quality/L40S_reference_videos \ - --device-folder L40S_reference_videos -``` +The files under `fastvideo/tests/modal/` are retained only as a disabled +manual rollback implementation. No active CI trigger invokes them. ### SSIM Bootstrap Mode @@ -206,28 +190,30 @@ python fastvideo/tests/ssim/reference_videos_cli.py promote-draft \ ## CI Integration -FastVideo CI tests are orchestrated by Buildkite and run on Modal GPU -instances. The main files are: +FastVideo GPU CI is orchestrated by Buildkite and runs only on isolated Slinky +Slurm workers. The main files are: | File | Purpose | |---|---| -| `.buildkite/pipeline.yml` | Buildkite test graph and path filters. | -| `.buildkite/scripts/pr_test.sh` | Dispatches `TEST_TYPE` to a Modal function. | -| `fastvideo/tests/modal/pr_test.py` | Modal functions for most test lanes. | -| `fastvideo/tests/modal/ssim_test.py` | Modal functions and partitioning for SSIM. | +| `.buildkite/pipeline.yml` | Static, validated 20-lane Slurm test graph. | +| `.github/scripts/plan_merge_ci.py` | Trusted path-to-lane and focused quality-test policy for `/merge`. | +| `.buildkite/scripts/unit_test.sh`, `.buildkite/scripts/lanes/*.sh` | Repository-owned test payloads executed inside Slurm containers. | +| `fastvideo/tests/ssim/ci_runner.py` | Four-GPU SSIM task discovery and scheduling. | +| `.buildkite/scripts/pr_test.sh`, `fastvideo/tests/modal/*.py` | Dormant manual rollback path; rejected in Buildkite. | -For exact tier membership, path filters, slash commands, and aggregate statuses, +For exact tier membership, slash commands, runner isolation, and aggregate statuses, see [CI/CD Architecture](ci_architecture.md). ### Adding A New CI Test Category If a new test does not fit an existing lane: -1. Add a Modal function in `fastvideo/tests/modal/pr_test.py` or a focused - companion module. -2. Add a matching `TEST_TYPE` case in `.buildkite/scripts/pr_test.sh`. -3. Add Buildkite direct-test and path-filter entries in `.buildkite/pipeline.yml`. -4. Add the `/test` mapping in `.github/workflows/ci-slash-commands.yml`. +1. Put the test payload in an executable `.buildkite/scripts/lanes/.sh`. +2. Add its static step to `.buildkite/pipeline.yml`, its changed-path ownership + to `.github/scripts/plan_merge_ci.py`, and extend the CI contract tests. +3. Add the `/test` mapping in `.github/workflows/ci-slash-commands.yml`. +4. Coordinate the matching GPU, timeout, dependency, secret, and artifact + policy in the private Slurm runner allowlist. 5. Document the new category in [CI/CD Architecture](ci_architecture.md) and add authoring notes here if contributors need them. diff --git a/docs/design/inference_schema_parity_inventory.yaml b/docs/design/inference_schema_parity_inventory.yaml index 31b61adbe..c5c965512 100644 --- a/docs/design/inference_schema_parity_inventory.yaml +++ b/docs/design/inference_schema_parity_inventory.yaml @@ -124,12 +124,14 @@ surfaces: vae_precision: "Precision override pending dedicated typed component precision design." vae_decode_precision: "Decode-only precision override pending dedicated typed component precision design." image_encoder_precision: "Precision override pending dedicated typed component precision design." + image_encoder_precisions: "Precision overrides pending dedicated typed component precision design." text_encoder_precisions: "Precision override pending dedicated typed component precision design." internal_only: dit_config: "Legacy internal component config object." upsampler_config: "Legacy internal component config object." vae_config: "Legacy internal component config object." image_encoder_config: "Legacy internal component config object." + image_encoder_configs: "Legacy internal component config objects." text_encoder_configs: "Legacy internal component config object." preprocess_text_funcs: "Internal text preprocessing hooks." postprocess_text_funcs: "Internal text postprocessing hooks." @@ -366,6 +368,30 @@ surfaces: sources: [fastvideo.configs.pipelines.matrixgame2.MatrixGame2I2V480PConfig] num_frames_per_block: sources: [fastvideo.configs.pipelines.matrixgame2.MatrixGame2I2V480PConfig] + duration_s: + sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig] + spectrogram_frame_rate: + sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig] + latent_downsample_rate: + sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig] + clip_frame_rate: + sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig] + sync_frame_rate: + sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig] + sync_segment_size: + sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig] + sync_segment_stride: + sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig] + sync_downsample_rate: + sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig] + clip_image_size: + sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig] + sync_image_size: + sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig] + clip_batch_size_multiplier: + sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig] + sync_batch_size_multiplier: + sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig] audio_channels: sources: - fastvideo.configs.pipelines.stable_audio.StableAudioT2AConfig @@ -380,6 +406,7 @@ surfaces: - fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig max_audio_duration_s: sources: + - fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig - fastvideo.configs.pipelines.stable_audio.StableAudioT2AConfig - fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig sample_size: @@ -388,6 +415,7 @@ surfaces: - fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig sampling_rate: sources: + - fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig - fastvideo.configs.pipelines.stable_audio.StableAudioT2AConfig - fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig audio_txt_guidance_scale: diff --git a/examples/inference/basic/README.md b/examples/inference/basic/README.md index 5b8153f90..1498dd375 100644 --- a/examples/inference/basic/README.md +++ b/examples/inference/basic/README.md @@ -32,7 +32,9 @@ python examples/inference/basic/mlx_wan_prompt_to_video.py \ --prompt "A bird's-eye view of a misty forest valley at dawn." ``` -5B uses [`mlx_wan22_generate.py`](mlx_wan22_generate.py) with +5B uses +[`mlx_wan22_generate.py`](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/mlx_wan22_generate.py) +with `FastVideo/FastMetal-5B-QAD`. `examples/inference/basic/basic_mps.py` is the older PyTorch MPS demo. @@ -49,7 +51,9 @@ python examples/inference/basic/basic_dmd_new_api.py ### FastH3 Preview -The verified [basic FastH3 example](basic_fasth3.py) runs the few-step (4-forward, DMD2-distilled) MiniMax-H3 preview, generating synchronized video and audio with its trained block-sparse VSA attention: +The verified [basic FastH3 example](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_fasth3.py) +runs the few-step (4-forward, DMD2-distilled) MiniMax-H3 preview, generating +synchronized video and audio with its trained block-sparse VSA attention: ```bash UV_TORCH_BACKEND=cu130 uv pip install -e ".[fasth3]" diff --git a/fastvideo/tests/AGENTS.md b/fastvideo/tests/AGENTS.md index ad6888b21..49dd687ad 100644 --- a/fastvideo/tests/AGENTS.md +++ b/fastvideo/tests/AGENTS.md @@ -21,7 +21,7 @@ tests/ ├── hooks/ # Runtime hook system ├── inference/ # End-to-end inference smoke ├── lora_extraction/ # LoRA merge/extract round-trip -├── modal/ # Modal CI orchestrators (ssim_test.py, nightly_test.py) +├── modal/ # Dormant manual Modal rollback helpers + their unit tests ├── nightly/ # Long-running suites (gated) ├── ops/ # Custom kernel / op tests ├── performance/ # Throughput / memory benchmarks (informational) @@ -42,7 +42,7 @@ pytest fastvideo/tests/encoders/ -v # One domain pytest fastvideo/tests/ssim/ -vs # SSIM (GPU-heavy; see ssim/AGENTS.md) pytest fastvideo/tests/ssim/ -vs --ssim-full-quality # Full-quality SSIM params pytest tests/ -v # Top-level repo tests (different scope) -modal run fastvideo/tests/modal/ssim_test.py # Orchestrated CI-style SSIM run +python fastvideo/tests/ssim/ci_runner.py # Four-GPU Slurm-style SSIM scheduler ``` `tests/local_tests/` (top-level) holds component checks that need a local @@ -59,11 +59,17 @@ working tree but are not part of the package suite. - `nightly/` and `performance/` tests should be guarded by an environment marker so the default `pytest fastvideo/tests/` stays fast. -## Modal CI +## Slurm CI -`tests/modal/ssim_test.py` is the orchestrator that auto-discovers +`tests/ssim/ci_runner.py` is the active CI orchestrator. It auto-discovers `test_*.py` files under `ssim/` and schedules one subprocess per `*_MODEL_TO_PARAMS` -key. New SSIM tests need no CI wiring — just declare `REQUIRED_GPUS = N`. +key across the four GPUs granted by the Slinky Slurm lane. The change-aware +merge planner may pass repeated `--test-file` basenames for focused model-family +coverage; direct and scheduled full runs omit the filter. New SSIM tests need no +pipeline wiring — just declare `REQUIRED_GPUS = N`. + +The modules in `tests/modal/` are retained for dormant manual rollback and +must not be referenced by active Buildkite or slash-command routes. ## Anti-Patterns @@ -71,4 +77,4 @@ key. New SSIM tests need no CI wiring — just declare `REQUIRED_GPUS = N`. `pytest.skip` when the path is missing. - Forgetting `cleanup_dist_env_and_memory()` in tests that bypass the `distributed_setup` fixture — tears the next test in the worker. -- Putting Modal-specific code outside `tests/modal/`. +- Putting Slurm host policy or credentials in repository-owned lane scripts. diff --git a/fastvideo/tests/api/test_attn_qat_infer_capability_gate.py b/fastvideo/tests/api/test_attn_qat_infer_capability_gate.py index 7094e8844..46c122059 100644 --- a/fastvideo/tests/api/test_attn_qat_infer_capability_gate.py +++ b/fastvideo/tests/api/test_attn_qat_infer_capability_gate.py @@ -4,11 +4,11 @@ the real platform resolver. ``is_attn_qat_infer_available()`` used to test only whether the kernel extension imports. CUDA 13 wheel builds can carry the sm_120/sm_121 -extension on any host (e.g. H100 sm_90, GB200 sm_100): the import -succeeds, ``CudaPlatformBase.get_attn_backend_cls`` selects the -consumer-Blackwell backend, and the first kernel call fails with an -unsupported-capability error -- instead of the FlashAttention fallback -the QAD README documents for non-sm_120 GPUs. +extension on any host (e.g. H100 sm_90, the Slurm runner's GB200 sm_100). Without the +capability gate, that successful import selects the consumer-Blackwell +backend and defers failure until the first unsupported kernel call. +Explicit ATTN_QAT_INFER requests must instead fail closed during +resolution. These tests drive the REAL resolver (``fastvideo.platforms.cuda``) and the REAL availability function; only the two physical facts are faked -- @@ -17,7 +17,7 @@ active" (``torch.cuda``). The stage-guard test (fastvideo/tests/stages/test_kandinsky5_attention_backend_guard.py) injects an already-resolved backend and by design cannot see this bug. -CPU-only: the ATTN_QAT_INFER branch and its fallback never require a +CPU-only: the ATTN_QAT_INFER branch and its error path never require a physical GPU to *resolve* (only to run). """ from __future__ import annotations @@ -35,13 +35,6 @@ from fastvideo.platforms.cuda import NonNvmlCudaPlatform from fastvideo.platforms.interface import AttentionBackendEnum ATTN_QAT_INFER_CLS = "fastvideo.attention.backends.attn_qat_infer.AttnQatInferBackend" -# What the resolver's fallthrough legitimately returns when ATTN_QAT_INFER -# is unavailable: FlashAttention, or SDPA when flash_attn isn't installed -# in the running environment (e.g. CPU-only CI). -FALLBACK_CLASSES = { - "fastvideo.attention.backends.flash_attn.FlashAttentionBackend", - "fastvideo.attention.backends.sdpa.SDPABackend", -} def _fake_gpu(monkeypatch, *, capability: tuple[int, int], extension_imports: bool, fa4_imports: bool = False) -> None: @@ -66,22 +59,24 @@ def _resolve() -> str: ) -def test_sm90_host_with_bundled_extension_falls_back(monkeypatch): +def test_sm90_host_with_bundled_extension_fails_closed(monkeypatch): """The reviewed failure: H100 + CUDA 13 wheel that bundles the sm_120 - extension. Import succeeds; selection must still fall back.""" + extension. Import succeeds; explicit selection must still fail closed.""" _fake_gpu(monkeypatch, capability=(9, 0), extension_imports=True) assert not is_attn_qat_infer_available() - assert _resolve() in FALLBACK_CLASSES + with pytest.raises(ImportError, match="ATTN_QAT_INFER selected but"): + _resolve() -def test_sm100_host_with_bundled_extension_falls_back(monkeypatch): +def test_sm100_host_with_bundled_extension_fails_closed(monkeypatch): """sm_100 with only the (unrunnable) bundled sm_12x extension and no - FP4 FA4 kernel still falls back -- the original reviewed failure class.""" + FP4 FA4 kernel still fails closed -- the original reviewed failure class.""" _fake_gpu(monkeypatch, capability=(10, 0), extension_imports=True, fa4_imports=False) assert not is_attn_qat_infer_available() - assert _resolve() in FALLBACK_CLASSES + with pytest.raises(ImportError, match="ATTN_QAT_INFER selected but"): + _resolve() @pytest.mark.parametrize("capability", [(10, 0), (10, 3)]) @@ -102,11 +97,12 @@ def test_consumer_blackwell_with_extension_selects_backend(monkeypatch, capabili assert _resolve() == ATTN_QAT_INFER_CLS -def test_consumer_blackwell_without_extension_falls_back(monkeypatch): +def test_consumer_blackwell_without_extension_fails_closed(monkeypatch): _fake_gpu(monkeypatch, capability=(12, 0), extension_imports=False) assert not is_attn_qat_infer_available() - assert _resolve() in FALLBACK_CLASSES + with pytest.raises(ImportError, match="ATTN_QAT_INFER selected but"): + _resolve() def test_no_cuda_reports_unavailable(monkeypatch): diff --git a/fastvideo/tests/contract/test_ci_test_collection.py b/fastvideo/tests/contract/test_ci_test_collection.py index 9b0152db2..4a0e0b179 100644 --- a/fastvideo/tests/contract/test_ci_test_collection.py +++ b/fastvideo/tests/contract/test_ci_test_collection.py @@ -12,17 +12,48 @@ allowlisted here with a reason. Pure text analysis — no fastvideo imports, no GPU, no torch. """ +import re from pathlib import Path +import yaml + REPO_ROOT = Path(__file__).resolve().parents[3] TESTS_ROOT = REPO_ROOT / "fastvideo" / "tests" # Files whose text constitutes "a CI lane references this directory". CI_SOURCES = [ - TESTS_ROOT / "modal" / "pr_test.py", - TESTS_ROOT / "modal" / "ssim_test.py", *sorted((REPO_ROOT / ".buildkite").rglob("*.yml")), *sorted((REPO_ROOT / ".buildkite").rglob("*.sh")), + *sorted((REPO_ROOT / ".github/workflows").glob("ci-*.yml")), +] + +# (slash name, compatibility slash name, public TEST_TYPE, runner TEST_TYPE, +# Buildkite step key, fastcheck lane, pinned command) +SLURM_LANES = [ + ("encoder", "encoder-ci", "encoder", "encoder_ci", "encoder", True, "run-ci"), + ("vae", "vae-ci", "vae", "vae_ci", "vae", True, "run-ci"), + ("transformer", "transformer-ci", "transformer", "transformer_ci", "transformer", True, "run-ci"), + ("kernel", "kernel-ci", "kernel_tests", "kernel_tests_ci", "kernel-tests", True, "run-ci"), + ("unit", "unit-ci", "unit_test", "unit_test_ci", "unit", True, "run-unit"), + ("dreamverse", "dreamverse-ci", "dreamverse_app", "dreamverse_app_ci", "dreamverse", True, "run-ci"), + ("golden-gate", "golden-gate-ci", "golden_gate", "golden_gate_ci", "golden-gate", False, "run-ci"), + ("ssim", "ssim-ci", "ssim", "ssim_ci", "ssim", False, "run-ci"), + ("lora-inference", "lora-inference-ci", "inference_lora", "inference_lora_ci", "lora-inference", False, + "run-ci"), + ("lora-extraction", "lora-extraction-ci", "lora_extraction", "lora_extraction_ci", "lora-extraction", False, + "run-ci"), + ("training", "training-ci", "training", "training_ci", "training", False, "run-ci"), + ("distillation", "distillation-ci", "distillation_dmd", "distillation_dmd_ci", "distillation", False, + "run-ci"), + ("self-forcing", "self-forcing-ci", "self_forcing", "self_forcing_ci", "self-forcing", False, "run-ci"), + ("lora-training", "lora-training-ci", "training_lora", "training_lora_ci", "lora-training", False, "run-ci"), + ("vsa", "vsa-ci", "training_vsa", "training_vsa_ci", "training-vsa", False, "run-ci"), + ("vmoba", "vmoba-ci", "inference_vmoba", "inference_vmoba_ci", "inference-vmoba", False, "run-ci"), + ("performance", "performance-ci", "performance", "performance_ci", "performance", False, "run-ci"), + ("api", "api-ci", "api_server", "api_server_ci", "api-server", False, "run-ci"), + ("train-framework", "train-framework-ci", "train_framework", "train_framework_ci", "train-framework", False, + "run-ci"), + ("eval", "eval-ci", "eval", "eval_ci", "eval", False, "run-ci"), ] # Directories that intentionally have no CI lane today. Every entry needs a @@ -56,7 +87,7 @@ def test_every_test_directory_is_collected_or_allowlisted(): dark = [name for name in _dirs_with_tests() if f"tests/{name}" not in ci_text and name not in ALLOWLIST] assert not dark, (f"Test directories not referenced by any CI lane and not " f"allowlisted: {dark}. Wire them into a lane in " - f"fastvideo/tests/modal/pr_test.py (or a Buildkite step), or add an " + f"a .buildkite lane script, or add an " f"allowlist entry with a reason in {__file__}.") @@ -68,6 +99,175 @@ def test_local_tests_stays_out_of_ci(): "reference or move the tests into a fastvideo/tests/ lane.") +def _pipeline_steps() -> list[dict[str, object]]: + pipeline = yaml.safe_load((REPO_ROOT / ".buildkite/pipeline.yml").read_text()) + return pipeline["steps"] + + +def test_all_gpu_ci_routes_use_the_trusted_slurm_dispatcher(): + slash_commands = (REPO_ROOT / ".github/workflows/ci-slash-commands.yml").read_text() + steps = _pipeline_steps() + valid_line = next(line for line in slash_commands.splitlines() if line.strip().startswith("VALID=")) + valid_names = set(valid_line.split('"', maxsplit=2)[1].split()) + assert len(steps) == len(SLURM_LANES) == 20 + by_key = {step["key"]: step for step in steps} + assert len(by_key) == len(steps) + + for slash_name, compatibility_name, public_type, runner_type, step_key, fastcheck, command in SLURM_LANES: + assert slash_name in valid_names + assert compatibility_name in valid_names + assert f"[{slash_name}]={public_type}" in slash_commands + assert f"[{compatibility_name}]={runner_type}" in slash_commands + + step = by_key[step_key] + assert step["command"] == f"/opt/fastvideo-ci-runner/{command}" + assert step["timeout_in_minutes"] == 90 + assert step["agents"] == {"queue": "ci-runner"} + assert step["env"] == {"TEST_TYPE": runner_type} + assert "soft_fail" not in step + if step_key in {"ssim", "training"}: + assert step["concurrency"] == 1 + assert step["concurrency_group"] == "fastvideo/slinky/whole-tray" + else: + assert "concurrency" not in step + assert "concurrency_group" not in step + condition = str(step["if"]) + assert 'build.env("TEST_SCOPE") == "full"' in condition + assert 'build.env("TEST_SCOPE") == "merge"' in condition + assert f'/,{step_key},/' in condition + assert 'build.env("TEST_SCOPE") == "direct"' in condition + assert f'build.env("TEST_TYPE") == "{public_type}"' in condition + assert f'build.env("TEST_TYPE") == "{runner_type}"' in condition + if fastcheck: + assert 'build.env("TEST_SCOPE") == "fastcheck"' in condition + assert 'build.env("TEST_SCOPE") == null' in condition + + +def test_merge_comment_has_one_change_aware_trigger_path(): + slash_commands = (REPO_ROOT / ".github/workflows/ci-slash-commands.yml").read_text() + merge_job = slash_commands.split(" parse-command:", maxsplit=1)[0] + ready_workflow = (REPO_ROOT / ".github/workflows/ci-trigger-full-suite.yml").read_text() + + assert "labels: ['ready']" in merge_job + assert "api.buildkite.com" not in merge_job + assert "BUILDKITE_API_TOKEN" not in merge_job + assert "api.buildkite.com" in ready_workflow + assert 'jq -r --arg pr_number "$PR_NUMBER"' in ready_workflow + assert '.env.PR_NUMBER? == $pr_number' in ready_workflow + assert 'TEST_SCOPE: "merge"' in ready_workflow + assert 'FULL_SUITE: "true"' in ready_workflow + assert "plan_merge_ci.py" in ready_workflow + assert "github.event.pull_request.base.sha" in ready_workflow + assert "MERGE_TEST_PLAN" in ready_workflow + assert "MERGE_GOLDEN_TESTS" in ready_workflow + assert "MERGE_SSIM_TESTS" in ready_workflow + assert "__FASTVIDEO_CI_PLAN_ALL__" in ready_workflow + + +def test_full_ssim_has_a_weekly_slurm_schedule(): + workflow = (REPO_ROOT / ".github/workflows/ci-scheduled-ssim.yml").read_text() + ssim_step = next(step for step in _pipeline_steps() if step["key"] == "ssim") + + assert 'cron: "0 5 * * 0"' in workflow + assert 'TEST_SCOPE: "scheduled"' in workflow + assert 'TEST_TYPE: "ssim"' in workflow + assert "modal" not in workflow.lower() + assert 'build.env("TEST_SCOPE") == "scheduled"' in str(ssim_step["if"]) + + +def test_active_pipeline_has_no_modal_or_untrusted_compute_path(): + pipeline_text = (REPO_ROOT / ".buildkite/pipeline.yml").read_text() + steps = _pipeline_steps() + + assert ".buildkite/scripts/pr_test.sh" not in pipeline_text + assert "monorepo-diff" not in pipeline_text + assert all(step.get("agents") == {"queue": "ci-runner"} for step in steps) + assert all("plugins" not in step for step in steps) + assert all(str(step.get("command", "")).startswith("/opt/fastvideo-ci-runner/") for step in steps) + + +def test_gpu_tests_preserve_a_launcher_assigned_rendezvous_port(): + hard_assignment = re.compile(r'''os\.environ\[\s*["']MASTER_PORT["']\s*\]\s*=''') + overwrites = [] + for path in TESTS_ROOT.rglob("*.py"): + if path.resolve() == Path(__file__).resolve(): + continue + if hard_assignment.search(path.read_text(errors="replace")): + overwrites.append(str(path.relative_to(REPO_ROOT))) + assert not overwrites, f"Tests overwrite the CI runner's per-lease MASTER_PORT: {overwrites}" + + +def test_dreamverse_lane_keeps_arm64_browser_coverage_explicit(): + lane = (REPO_ROOT / ".buildkite/scripts/lanes/dreamverse.sh").read_text() + + assert "deb.nodesource.com" not in lane + assert "node_version=v22.23.2" in lane + assert "mktemp -d -t fastvideo-node.XXXXXX" in lane + assert "sha256sum --check --status" in lane + assert "aarch64|arm64" in lane + assert "--project=firefox" in lane + assert "--project=chromium" in lane + assert "--project=mobile-chromium" in lane + assert "--grep-invert=" in lane + assert "--project=webkit" in lane + assert "--project=mobile-safari" in lane + + +def test_training_lanes_keep_tracking_offline_and_secret_free(): + lane_root = REPO_ROOT / ".buildkite/scripts/lanes" + for lane_name in ("training.sh", "self_forcing.sh", "training_lora.sh", "training_vsa.sh"): + lane = (lane_root / lane_name).read_text() + assert "export WANDB_MODE=offline" in lane + assert "WANDB_API_KEY" not in lane + assert "wandb login" not in lane + + +def test_slurm_lane_status_contexts_are_unique_and_aggregatable(): + labels = [str(step["label"]) for step in _pipeline_steps()] + aggregate = (REPO_ROOT / ".github/workflows/ci-aggregate-status.yml").read_text() + + assert len(labels) == len(set(labels)) + assert sum(label.startswith(":microscope:") for label in labels) == 6 + assert all(label.startswith((":microscope:", ":test_tube:", ":bar_chart:")) for label in labels) + assert "'buildkite/pr-fastcheck/microscope-'," in aggregate + assert "'buildkite/ci/microscope-'," in aggregate + assert "'buildkite/ci/test-tube-'," in aggregate + assert "'buildkite/ci/bar-chart-'," in aggregate + assert "fastcheck.size === 6" in aggregate + assert "fullSuiteOnly.size === 14" in aggregate + assert "fastcheckPassed" in aggregate + + +def test_ssim_lane_uses_the_local_four_gpu_scheduler(): + lane_script = (REPO_ROOT / ".buildkite/scripts/lanes/ssim.sh").read_text() + scheduler = (TESTS_ROOT / "ssim/ci_runner.py").read_text() + dockerfile = (REPO_ROOT / "docker/Dockerfile").read_text() + + assert "fastvideo/tests/ssim/ci_runner.py" in lane_script + assert "libx11-dev" in lane_script + assert "libx11-dev" in dockerfile + assert "Skipping FA4 cute overlay on arm64" not in dockerfile + assert "import flash_attn.cute" in dockerfile + assert "import modal" not in scheduler + assert "MAX_GPUS = 4" in scheduler + assert "REQUIRED_GPUS" in scheduler + assert "MODEL_TO_PARAMS" in scheduler + assert "--test-file" in scheduler + assert "FASTVIDEO_SSIM_TEST_FILES" in lane_script + assert 'if [ "${TEST_SCOPE:-}" = merge ]; then' in lane_script + assert "Missing FASTVIDEO_SSIM_TEST_FILES for merge scope" in lane_script + + +def test_golden_lane_accepts_only_focused_test_basenames(): + lane_script = (REPO_ROOT / ".buildkite/scripts/lanes/golden_gate.sh").read_text() + + assert "FASTVIDEO_GOLDEN_TEST_FILES" in lane_script + assert "test_[a-z0-9_]+" in lane_script + assert 'golden_root=./fastvideo/tests/golden_gate' in lane_script + assert 'if [ "${TEST_SCOPE:-}" = merge ]; then' in lane_script + assert "Missing FASTVIDEO_GOLDEN_TEST_FILES for merge scope" in lane_script + + def test_allowlist_entries_are_still_real_directories(): # A stale allowlist hides regressions; entries must track reality. missing = [name for name in ALLOWLIST if name != "modal" and not (TESTS_ROOT / name).is_dir()] diff --git a/fastvideo/tests/contract/test_merge_ci_plan.py b/fastvideo/tests/contract/test_merge_ci_plan.py new file mode 100644 index 000000000..e7f23f841 --- /dev/null +++ b/fastvideo/tests/contract/test_merge_ci_plan.py @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CPU-only coverage for the change-aware /merge test planner.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[3] +PLANNER_PATH = REPO_ROOT / ".github/scripts/plan_merge_ci.py" +SPEC = importlib.util.spec_from_file_location("plan_merge_ci", PLANNER_PATH) +assert SPEC is not None and SPEC.loader is not None +PLAN_MERGE_CI = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = PLAN_MERGE_CI +SPEC.loader.exec_module(PLAN_MERGE_CI) + + +def test_docs_only_merge_adds_no_gpu_lanes(): + plan = PLAN_MERGE_CI.classify_paths([ + "docs/contributing/testing.md", + "README.md", + "requirements-mkdocs.txt", + ]) + + assert plan.encoded_lanes() == ",none," + assert plan.encoded_golden_tests() == "none" + assert plan.encoded_ssim_tests() == "none" + + +def test_model_family_change_selects_focused_golden_and_ssim(): + plan = PLAN_MERGE_CI.classify_paths(["fastvideo/models/dits/wanvideo.py"]) + + assert plan.encoded_lanes() == ",golden-gate,ssim," + assert plan.encoded_golden_tests() == "test_wan_t2v.py" + assert plan.encoded_ssim_tests() == ( + "test_causal_similarity.py,test_wan_i2v_similarity.py,test_wan_t2v_similarity.py") + + +def test_flux2_change_does_not_pull_unrelated_flux1_quality_tests(): + plan = PLAN_MERGE_CI.classify_paths(["fastvideo/pipelines/basic/flux_2/flux_2_pipeline.py"]) + + assert plan.encoded_golden_tests() == "test_flux2_klein.py" + assert plan.encoded_ssim_tests() == "test_flux2_similarity.py" + + +def test_changed_ssim_test_selects_only_that_file(): + plan = PLAN_MERGE_CI.classify_paths(["fastvideo/tests/ssim/test_flux_t2i_similarity.py"]) + + assert plan.encoded_lanes() == ",ssim," + assert plan.encoded_ssim_tests() == "test_flux_t2i_similarity.py" + + +def test_shared_golden_harness_or_reference_requires_full_golden_lane(): + plan = PLAN_MERGE_CI.classify_paths(["fastvideo/tests/golden_gate/_harness.py"]) + + assert plan.encoded_lanes() == ",golden-gate," + assert plan.encoded_golden_tests() == "all" + + +def test_shared_ssim_harness_requires_full_ssim_lane(): + plan = PLAN_MERGE_CI.classify_paths(["fastvideo/tests/ssim/inference_similarity_utils.py"]) + + assert plan.encoded_lanes() == ",ssim," + assert plan.encoded_ssim_tests() == "all" + + +def test_training_domains_do_not_fan_out_to_unrelated_lanes(): + plan = PLAN_MERGE_CI.classify_paths([ + "fastvideo/training/wan_self_forcing_distillation_pipeline.py", + "fastvideo/tests/training/lora/test_lora_training.py", + ]) + + assert plan.encoded_lanes() == ",self-forcing,lora-training," + + +def test_shared_training_code_selects_all_legacy_training_consumers(): + plan = PLAN_MERGE_CI.classify_paths(["fastvideo/training/trackers.py"]) + + assert plan.ordered_lanes() == PLAN_MERGE_CI.LEGACY_TRAINING_LANES + + +def test_dataset_change_selects_only_training_consumers(): + plan = PLAN_MERGE_CI.classify_paths(["fastvideo/dataset/dataloader/bucket.py"]) + + assert plan.ordered_lanes() == PLAN_MERGE_CI.ALL_TRAINING_LANES + + +def test_performance_implementation_selects_only_performance_lane(): + plan = PLAN_MERGE_CI.classify_paths(["fastvideo/performance/metric_policy.py"]) + + assert plan.encoded_lanes() == ",performance," + + +def test_shared_runtime_change_gets_focused_quality_smoke_not_every_lane(): + plan = PLAN_MERGE_CI.classify_paths(["fastvideo/registry.py"]) + + assert plan.encoded_lanes() == ",golden-gate,ssim," + assert plan.encoded_golden_tests() == "all" + assert plan.encoded_ssim_tests() == "test_flux_t2i_similarity.py,test_wan_t2v_similarity.py" + + +def test_lane_script_selects_its_single_lane(): + plan = PLAN_MERGE_CI.classify_paths([".buildkite/scripts/lanes/api_server.sh"]) + + assert plan.encoded_lanes() == ",api-server," + + +def test_fastcheck_lane_script_is_not_repeated_by_merge(): + plan = PLAN_MERGE_CI.classify_paths([".buildkite/scripts/lanes/encoder.sh"]) + + assert plan.encoded_lanes() == ",none," + + +def test_cross_cutting_dependency_change_fails_out_to_every_merge_lane(): + plan = PLAN_MERGE_CI.classify_paths(["uv.lock"]) + + assert plan.ordered_lanes() == PLAN_MERGE_CI.MERGE_LANES + assert plan.encoded_golden_tests() == "all" + assert plan.encoded_ssim_tests() == "all" + + +def test_unknown_path_fails_closed_to_every_merge_lane(): + plan = PLAN_MERGE_CI.classify_paths(["new_runtime/build_rules.toml"]) + + assert plan.ordered_lanes() == PLAN_MERGE_CI.MERGE_LANES + + +def test_empty_or_failed_changed_file_query_fails_closed(): + assert PLAN_MERGE_CI.classify_paths([]).ordered_lanes() == PLAN_MERGE_CI.MERGE_LANES + assert PLAN_MERGE_CI.classify_paths([ + "__FASTVIDEO_CI_PLAN_ALL__" + ]).ordered_lanes() == PLAN_MERGE_CI.MERGE_LANES + + +def test_dreamverse_change_relies_on_its_existing_fastcheck_e2e_lane(): + plan = PLAN_MERGE_CI.classify_paths(["apps/dreamverse/web/src/App.tsx"]) + + assert plan.encoded_lanes() == ",none," + + +def test_local_only_parity_scaffold_does_not_trigger_unrelated_gpu_lanes(): + plan = PLAN_MERGE_CI.classify_paths(["tests/local_tests/flux/test_flux_dev_component_loaders.py"]) + + assert plan.encoded_lanes() == ",none," diff --git a/fastvideo/tests/contract/test_modal_fa4_policy.py b/fastvideo/tests/contract/test_modal_fa4_policy.py index 1d0fa0e12..ff64b84a9 100644 --- a/fastvideo/tests/contract/test_modal_fa4_policy.py +++ b/fastvideo/tests/contract/test_modal_fa4_policy.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -"""Guard Modal runtime policies that keep CI lanes on their intended backend. +"""Guard backend policies in the dormant Modal rollback runtime. Pure text/AST analysis: no fastvideo imports, no torch, no Modal client. """ @@ -27,13 +27,13 @@ def _function_strings(path: Path, function_name: str) -> str: def test_generic_l40s_launcher_defaults_fa4_off(): - source = LAUNCH_L40S_JOB.read_text(encoding="utf-8") - assert '"FASTVIDEO_FA4": os.environ.get("FASTVIDEO_FA4", "0")' in source + source = ast.unparse(ast.parse(LAUNCH_L40S_JOB.read_text(encoding="utf-8"))) + assert "'FASTVIDEO_FA4': os.environ.get('FASTVIDEO_FA4', '0')" in source def test_ssim_launcher_keeps_fa4_enabled_by_default(): - source = SSIM_TEST.read_text(encoding="utf-8") - assert '"FASTVIDEO_FA4": os.environ.get("FASTVIDEO_FA4", "1")' in source + source = ast.unparse(ast.parse(SSIM_TEST.read_text(encoding="utf-8"))) + assert "'FASTVIDEO_FA4': os.environ.get('FASTVIDEO_FA4', '1')" in source def test_performance_identity_env_reaches_modal_runtime(): @@ -57,18 +57,28 @@ def test_vsa_training_lane_uses_strict_h100_pair(): def test_pr_model_load_and_training_lanes_disable_fa4(): + # (pytest selection, shared lane script or None). The dormant Modal wrapper + # keeps FASTVIDEO_FA4=0 while shared selections live in the same lane + # scripts used by the Slurm runner. lanes = { - "run_transformer_tests": "pytest ./fastvideo/tests/transformers -vs", - "run_training_tests": "pytest ./fastvideo/tests/training/Vanilla -srP", - "run_training_lora_tests": "pytest ./fastvideo/tests/training/lora/test_lora_training.py -srP", - "run_training_tests_VSA": "pytest ./fastvideo/tests/training/VSA -srP", - "run_distill_dmd_tests": "pytest ./fastvideo/tests/training/distill/test_distill_dmd.py -vs", - "run_self_forcing_tests": "pytest ./fastvideo/tests/training/self-forcing/test_self_forcing.py -vs", - "run_train_framework_tests": "pytest ./fastvideo/tests/train/models ./fastvideo/tests/train/methods -vs", - "seed_grad_norm_references": "pytest ./fastvideo/tests/train/methods -vs -rs", + "run_transformer_tests": + ("pytest ./fastvideo/tests/transformers -vs", ".buildkite/scripts/lanes/transformer.sh"), + "run_training_tests": ("pytest ./fastvideo/tests/training/Vanilla -srP", None), + "run_training_lora_tests": ("pytest ./fastvideo/tests/training/lora/test_lora_training.py -srP", None), + "run_training_tests_VSA": ("pytest ./fastvideo/tests/training/VSA -srP", None), + "run_distill_dmd_tests": + ("pytest ./fastvideo/tests/training/distill/test_distill_dmd.py -vs", ".buildkite/scripts/lanes/distillation_dmd.sh"), + "run_self_forcing_tests": ("pytest ./fastvideo/tests/training/self-forcing/test_self_forcing.py -vs", None), + "run_train_framework_tests": + ("pytest ./fastvideo/tests/train/models ./fastvideo/tests/train/methods -vs", ".buildkite/scripts/lanes/train_framework.sh"), + "seed_grad_norm_references": ("pytest ./fastvideo/tests/train/methods -vs -rs", None), } - for function_name, pytest_command in lanes.items(): + for function_name, (pytest_command, lane_script) in lanes.items(): function_strings = _function_strings(PR_TEST, function_name) assert "FASTVIDEO_FA4=0" in function_strings - assert pytest_command in function_strings + if lane_script is None: + assert pytest_command in function_strings + else: + assert f"bash {lane_script}" in function_strings + assert pytest_command in (REPO_ROOT / lane_script).read_text(encoding="utf-8") diff --git a/fastvideo/tests/contract/test_ssim_ci_runner.py b/fastvideo/tests/contract/test_ssim_ci_runner.py new file mode 100644 index 000000000..4c9765f48 --- /dev/null +++ b/fastvideo/tests/contract/test_ssim_ci_runner.py @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CPU-only contracts for the Slurm SSIM task scheduler.""" + +from pathlib import Path +import sys + +import pytest + +SSIM_DIR = Path(__file__).resolve().parents[1] / "ssim" +sys.path.insert(0, str(SSIM_DIR)) + +from ci_runner import ( # noqa: E402 + discover_tasks, + extract_model_ids, + extract_required_gpus, + task_master_port, + visible_gpu_ids, +) + + +def test_discovery_splits_model_parameters_and_preserves_gpu_requirements(tmp_path: Path) -> None: + (tmp_path / "test_alpha.py").write_text( + """REQUIRED_GPUS = 2 +ALPHA_MODEL_TO_PARAMS = {"model/b": {}, "model/a": {}} +FULL_QUALITY_ALPHA_MODEL_TO_PARAMS = {"model/b": {}, "model/a": {}} +""", + encoding="utf-8", + ) + (tmp_path / "test_helpers.py").write_text("def test_helper(): pass\n", encoding="utf-8") + + tasks = discover_tasks(tmp_path) + + assert [(task.name, task.required_gpus) for task in tasks] == [ + ("test_alpha.py::model/a", 2), + ("test_alpha.py::model/b", 2), + ("test_helpers.py", 1), + ] + + +def test_ast_helpers_do_not_import_test_modules(tmp_path: Path) -> None: + path = tmp_path / "test_guard.py" + path.write_text( + """raise RuntimeError("must not import") +REQUIRED_GPUS = 3 +GUARD_MODEL_TO_PARAMS = {"model/id": {}} +""", + encoding="utf-8", + ) + + assert extract_required_gpus(path) == 3 + assert extract_model_ids(path) == ["model/id"] + + +def test_discovery_rejects_a_task_larger_than_the_slurm_tray(tmp_path: Path) -> None: + (tmp_path / "test_too_large.py").write_text("REQUIRED_GPUS = 5\n", encoding="utf-8") + + with pytest.raises(ValueError, match="supported range is 1-4"): + discover_tasks(tmp_path) + + +def test_discovery_can_select_exact_test_basenames(tmp_path: Path) -> None: + (tmp_path / "test_alpha.py").write_text("REQUIRED_GPUS = 1\n", encoding="utf-8") + (tmp_path / "test_beta.py").write_text("REQUIRED_GPUS = 2\n", encoding="utf-8") + + tasks = discover_tasks(tmp_path, ["test_beta.py"]) + + assert [(task.name, task.required_gpus) for task in tasks] == [("test_beta.py", 2)] + + +def test_discovery_rejects_missing_or_unsafe_test_file_selections(tmp_path: Path) -> None: + (tmp_path / "test_alpha.py").write_text("REQUIRED_GPUS = 1\n", encoding="utf-8") + + with pytest.raises(ValueError, match="do not exist"): + discover_tasks(tmp_path, ["test_missing.py"]) + with pytest.raises(ValueError, match="Invalid SSIM test file"): + discover_tasks(tmp_path, ["../test_alpha.py"]) + + +def test_visible_gpu_ids_preserve_the_slurm_lease(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "2,3,2") + + assert visible_gpu_ids() == ["2", "3"] + + +def test_parallel_tasks_get_offsets_in_the_runner_assigned_port_range() -> None: + assert task_master_port(0, "31200") == "31200" + assert task_master_port(17, "31200") == "31217" + + with pytest.raises(ValueError, match="100-port range"): + task_master_port(100, "31200") + + with pytest.raises(ValueError, match="outside 1-65535"): + task_master_port(17, "65520") diff --git a/fastvideo/tests/encoders/test_clip_encoder.py b/fastvideo/tests/encoders/test_clip_encoder.py index 273667db2..f97ba7ace 100644 --- a/fastvideo/tests/encoders/test_clip_encoder.py +++ b/fastvideo/tests/encoders/test_clip_encoder.py @@ -17,8 +17,8 @@ from torch.testing import assert_close logger = init_logger(__name__) -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29503" +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", "29503") BASE_MODEL_PATH = "hunyuanvideo-community/HunyuanVideo" MODEL_PATH = maybe_download_model(BASE_MODEL_PATH, local_dir=os.path.join("data", BASE_MODEL_PATH)) @@ -127,5 +127,11 @@ def test_clip_encoder(): assert pooler_output1.shape == pooler_output2.shape, ( f"Pooler output shapes don't match: {pooler_output1.shape} vs {pooler_output2.shape}") - assert_close(pooler_output1, pooler_output2, atol=1e-2, rtol=1e-3) - assert_close(last_hidden_state1, last_hidden_state2, atol=1e-2, rtol=1e-3) + # Blackwell's fp16 reduction order produces a slightly larger + # output delta between the HF and FastVideo implementations + # (observed max abs=0.01953125, rel=0.003395 on NVIDIA GB200). + # Keep the established tolerance everywhere else. + device_name = torch.cuda.get_device_name(device) if device.type == "cuda" else "" + pooler_atol, pooler_rtol = (2e-2, 4e-3) if "B200" in device_name else (1e-2, 1e-3) + assert_close(pooler_output1, pooler_output2, atol=pooler_atol, rtol=pooler_rtol) + assert_close(last_hidden_state1, last_hidden_state2, atol=pooler_atol, rtol=pooler_rtol) diff --git a/fastvideo/tests/encoders/test_hyt5_encoder.py b/fastvideo/tests/encoders/test_hyt5_encoder.py index e1d9e4f8c..9597f89b7 100644 --- a/fastvideo/tests/encoders/test_hyt5_encoder.py +++ b/fastvideo/tests/encoders/test_hyt5_encoder.py @@ -18,8 +18,8 @@ from fastvideo.configs.models.encoders import T5Config logger = init_logger(__name__) -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29503" +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", "29503") @pytest.fixture diff --git a/fastvideo/tests/encoders/test_llama_encoder.py b/fastvideo/tests/encoders/test_llama_encoder.py index 728405fee..22a08a26e 100644 --- a/fastvideo/tests/encoders/test_llama_encoder.py +++ b/fastvideo/tests/encoders/test_llama_encoder.py @@ -18,8 +18,8 @@ from torch.testing import assert_close logger = init_logger(__name__) -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29503" +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", "29503") BASE_MODEL_PATH = "hunyuanvideo-community/HunyuanVideo" MODEL_PATH = maybe_download_model(BASE_MODEL_PATH, local_dir=os.path.join("data", BASE_MODEL_PATH)) diff --git a/fastvideo/tests/encoders/test_qwen2_5_encoder.py b/fastvideo/tests/encoders/test_qwen2_5_encoder.py index 85d3de114..72b4cd0a5 100644 --- a/fastvideo/tests/encoders/test_qwen2_5_encoder.py +++ b/fastvideo/tests/encoders/test_qwen2_5_encoder.py @@ -16,8 +16,8 @@ from fastvideo.configs.models.encoders import Qwen2_5_VLConfig logger = init_logger(__name__) -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29505" +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", "29505") @pytest.fixture diff --git a/fastvideo/tests/encoders/test_siglip_encoder.py b/fastvideo/tests/encoders/test_siglip_encoder.py index b308d5ded..9bd8d9033 100644 --- a/fastvideo/tests/encoders/test_siglip_encoder.py +++ b/fastvideo/tests/encoders/test_siglip_encoder.py @@ -20,8 +20,8 @@ from fastvideo.logger import init_logger logger = init_logger(__name__) -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29505" +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", "29505") # HYWorld model path - SigLIP image encoder MODEL_ID = "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v" diff --git a/fastvideo/tests/encoders/test_t5_encoder.py b/fastvideo/tests/encoders/test_t5_encoder.py index 8d69d8cec..5183d5f48 100644 --- a/fastvideo/tests/encoders/test_t5_encoder.py +++ b/fastvideo/tests/encoders/test_t5_encoder.py @@ -19,8 +19,8 @@ from fastvideo.configs.models.encoders import T5Config, T5LargeConfig logger = init_logger(__name__) -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29503" +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", "29503") @pytest.fixture diff --git a/fastvideo/tests/golden_gate/test_kandinsky5.py b/fastvideo/tests/golden_gate/test_kandinsky5.py index 32102b8d1..43b6d2fca 100644 --- a/fastvideo/tests/golden_gate/test_kandinsky5.py +++ b/fastvideo/tests/golden_gate/test_kandinsky5.py @@ -26,8 +26,6 @@ from fastvideo.tests.golden_gate._harness import GateSpec, distributed_runtime, __all__ = ["distributed_runtime"] -os.environ.setdefault("FASTVIDEO_FA4", "0") # test_kandinsky5_similarity.py:84 - MODEL_DIM = 1792 # Kandinsky-5.0-T2V-Lite transformer/config.json TIME_DIM = 512 FF_DIM = 7168 @@ -99,5 +97,11 @@ SPEC = GateSpec( ) -def test_kandinsky5_golden_gate(distributed_runtime) -> None: +def test_kandinsky5_golden_gate(distributed_runtime, monkeypatch) -> None: + # Pin FA4 off for THIS test only (test_kandinsky5_similarity.py:84). A + # module-level os.environ.setdefault here leaked into every other + # golden-gate test's env fingerprint at pytest collection time, breaking + # full-directory runs in environments where FASTVIDEO_FA4 is unset + # (observed on the self-hosted CI runner lane). + monkeypatch.setenv("FASTVIDEO_FA4", os.environ.get("FASTVIDEO_FA4", "0")) run_gate(SPEC) diff --git a/fastvideo/tests/inference/lora/test_lora_inference_similarity.py b/fastvideo/tests/inference/lora/test_lora_inference_similarity.py index c5c7753f5..97e642bf2 100644 --- a/fastvideo/tests/inference/lora/test_lora_inference_similarity.py +++ b/fastvideo/tests/inference/lora/test_lora_inference_similarity.py @@ -18,8 +18,8 @@ from fastvideo.worker import MultiprocExecutor import torch logger = init_logger(__name__) -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29500" +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", "29500") # Base parameters for LoRA inference tests WAN_LORA_PARAMS = { diff --git a/fastvideo/tests/modal/kernel_build_cache.py b/fastvideo/tests/modal/kernel_build_cache.py index 26eb19cf5..9dd170f13 100644 --- a/fastvideo/tests/modal/kernel_build_cache.py +++ b/fastvideo/tests/modal/kernel_build_cache.py @@ -1,4 +1,4 @@ -"""Build and reuse FastVideo kernel wheels in Modal CI. +"""Build and reuse FastVideo kernel wheels in the dormant Modal rollback path. This module stays standalone because Modal launchers execute it from a freshly cloned checkout after dependency installation. Shared cache consumers are @@ -22,7 +22,7 @@ import zipfile from email.parser import Parser from pathlib import Path -CACHE_SCHEMA_VERSION = 3 +CACHE_SCHEMA_VERSION = 4 DEFAULT_PREBUILT_INFO_PATH = "/opt/fastvideo-kernel-prebuilt" KERNEL_RELATIVE_DIR = "fastvideo-kernel" DEFAULT_BUILD_INFO_OUTPUT = "/opt/fastvideo-kernel-prebuilt/default/metadata.json" @@ -166,6 +166,7 @@ def _torch_metadata() -> dict[str, object]: return { "torch_version": str(torch.__version__), "torch_cuda_version": str(torch.version.cuda), + "torch_git_version": str(getattr(torch.version, "git_version", "")), "torch_file": str(getattr(torch, "__file__", "")), "torch_config": str(torch.__config__.show()), "cxx11_abi": cxx11_abi, @@ -174,6 +175,7 @@ def _torch_metadata() -> dict[str, object]: return { "torch_version": f"", "torch_cuda_version": "", + "torch_git_version": "", "torch_file": "", "torch_config": "", "cxx11_abi": "", @@ -223,6 +225,11 @@ def _compiler_libc_metadata() -> dict[str, object]: def _build_metadata(repo_root: Path) -> dict[str, object]: explicit_arch = os.environ.get("TORCH_CUDA_ARCH_LIST", "").strip() resolved_arch = explicit_arch or _detect_arch_from_torch() + torch_metadata = _torch_metadata() + torch_cache_metadata = { + name: torch_metadata[name] + for name in ("torch_version", "torch_cuda_version", "torch_git_version", "cxx11_abi") + } cache_key_build = { "gpu_backend": os.environ.get("GPU_BACKEND", "CUDA"), "resolved_torch_cuda_arch_list": resolved_arch, @@ -242,7 +249,7 @@ def _build_metadata(repo_root: Path) -> dict[str, object]: "platform": sysconfig.get_platform(), "machine": platform.machine(), }, - "torch": _torch_metadata(), + "torch": torch_cache_metadata, "cuda": { "cuda_home": os.environ.get("CUDA_HOME", ""), "nvcc": _selected_command_metadata("CUDACXX", "nvcc"), @@ -252,6 +259,7 @@ def _build_metadata(repo_root: Path) -> dict[str, object]: } metadata = { **cache_key_metadata, + "torch": torch_metadata, "build": { **cache_key_build, "torch_cuda_arch_list": explicit_arch, diff --git a/fastvideo/tests/modal/pr_test.py b/fastvideo/tests/modal/pr_test.py index fc007dd32..15ccbb693 100644 --- a/fastvideo/tests/modal/pr_test.py +++ b/fastvideo/tests/modal/pr_test.py @@ -1,3 +1,9 @@ +"""Dormant manual Modal launcher. + +Production PR CI is Slinky/Slurm-only. This module is retained as rollback +code and is not referenced by the Buildkite pipeline or slash commands. +""" + import os import re import shutil @@ -243,7 +249,7 @@ def run_test_command(test_command: str, build_kernel: bool, install_command: str volumes={"/root/data": model_vol}) def run_encoder_tests(): run_test( - "export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/encoders -vs" + "export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && bash .buildkite/scripts/lanes/encoder.sh" ) @@ -254,7 +260,7 @@ def run_encoder_tests(): volumes={"/root/data": model_vol}) def run_vae_tests(): run_test( - "export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/vaes -vs") + "export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && bash .buildkite/scripts/lanes/vae.sh") @app.function(gpu="L40S:1", @@ -268,7 +274,7 @@ def run_golden_gate_tests(): # SSIM generation for that model cannot have regressed. Downloads only the # shards holding the gated layer, never full checkpoints. run_test( - "export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/golden_gate -vs" + "export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && bash .buildkite/scripts/lanes/golden_gate.sh" ) @@ -279,7 +285,7 @@ def run_golden_gate_tests(): volumes={"/root/data": model_vol}) def run_transformer_tests(): run_test("export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && " - "FASTVIDEO_FA4=0 pytest ./fastvideo/tests/transformers -vs") + "FASTVIDEO_FA4=0 bash .buildkite/scripts/lanes/transformer.sh") @app.function(gpu="L40S:4", @@ -313,7 +319,7 @@ def run_training_tests_VSA(): @app.function(gpu="H100:1", image=image, timeout=900, secrets=[ci_env_secret]) def run_kernel_tests(): - run_test("pytest fastvideo-kernel/tests/ -vs") + run_test("bash .buildkite/scripts/lanes/kernel_tests.sh") # @app.function(gpu="H100:1", image=image, timeout=900, secrets=[ci_env_secret]) @@ -328,17 +334,17 @@ def run_kernel_tests(): @app.function(gpu="L40S:1", image=image, timeout=900, secrets=[ci_env_secret]) def run_inference_tests_vmoba(): - run_test('python fastvideo/tests/inference/vmoba/test_vmoba_inference.py') + run_test("bash .buildkite/scripts/lanes/inference_vmoba.sh") @app.function(gpu="L40S:1", image=image, timeout=1200, secrets=[ci_env_secret]) def run_inference_lora_tests(): - run_test("pytest ./fastvideo/tests/inference/lora/test_lora_inference_similarity.py -vs") + run_test("bash .buildkite/scripts/lanes/inference_lora.sh") @app.function(gpu="L40S:2", image=image, timeout=900, secrets=[ci_env_secret]) def run_distill_dmd_tests(): - run_test("FASTVIDEO_FA4=0 pytest ./fastvideo/tests/training/distill/test_distill_dmd.py -vs") + run_test("FASTVIDEO_FA4=0 bash .buildkite/scripts/lanes/distillation_dmd.sh") @app.function(gpu="L40S:2", image=image, timeout=900, secrets=[wandb_secret, ci_env_secret]) @@ -349,15 +355,7 @@ def run_self_forcing_tests(): @app.function(gpu="L40S:1", image=image, timeout=900, secrets=[ci_env_secret]) def run_unit_test(): - run_test("pytest ./fastvideo/tests/api/ ./fastvideo/tests/contract/ ./fastvideo/tests/dataset/ " - "./fastvideo/tests/workflow/ ./fastvideo/tests/entrypoints/ ./fastvideo/tests/train/ " - "./fastvideo/tests/stages/ ./fastvideo/tests/ops/ ./fastvideo/tests/worker/ " - "./fastvideo/tests/training/test_trackers.py " - "./fastvideo/tests/attention/test_sdpa_metadata_mask_contract.py " - "./fastvideo/tests/modal/test_kernel_build_cache.py ./fastvideo/tests/modal/test_pr_test.py " - "./fastvideo/tests/modal/test_ssim_test.py " - "--ignore=./fastvideo/tests/entrypoints/test_openai_api_integration.py " - "--ignore=./fastvideo/tests/train/models --ignore=./fastvideo/tests/train/methods -vs") + run_test("bash .buildkite/scripts/unit_test.sh") # TODO: David: GPU only used to resolve import time requirement (not needed for this test). Maybe make those imports lazy? @@ -405,7 +403,7 @@ def run_dreamverse_app_tests(): volumes={"/root/data": model_vol}) def run_train_framework_tests(): run_test("export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && " - "FASTVIDEO_FA4=0 pytest ./fastvideo/tests/train/models ./fastvideo/tests/train/methods -vs") + "FASTVIDEO_FA4=0 bash .buildkite/scripts/lanes/train_framework.sh") @app.function(gpu="L40S:1", @@ -414,10 +412,10 @@ def run_train_framework_tests(): secrets=[hf_secret, ci_env_secret], volumes={"/root/data": model_vol}) def seed_grad_norm_references(): - """Record the per-method grad-norm reference for the **CI GPU (L40S only)**. + """Record the legacy L40S per-method grad-norm reference. Phase 2 / 5a-ii one-off seeding entrypoint. Pinned to ``gpu="L40S:1"`` (the - Modal CI runner), so this function only seeds the ``L40S`` key in + dormant Modal rollback runtime), so this function only seeds the ``L40S`` key in ``fastvideo/tests/train/methods/grad_norm_refs.json``. ``FASTVIDEO_GRADNORM_UPDATE=1`` makes ``check_grad_norm_regression`` record @@ -451,7 +449,7 @@ def run_eval_tests(): # pass vacuously. detectron2-backed vbench metrics remain skipped by # design (not pip-installable; see fastvideo/eval/README.md). run_test_command( - "export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/eval -vs", + "export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && bash .buildkite/scripts/lanes/eval.sh", build_kernel=True, install_command='uv pip install -e ".[test,eval-full]"') diff --git a/fastvideo/tests/modal/ssim_test.py b/fastvideo/tests/modal/ssim_test.py index 997637956..50c20e5b2 100644 --- a/fastvideo/tests/modal/ssim_test.py +++ b/fastvideo/tests/modal/ssim_test.py @@ -1,3 +1,9 @@ +"""Dormant manual Modal SSIM launcher. + +Production PR SSIM runs through ``fastvideo/tests/ssim/ci_runner.py`` on the +Slinky Slurm tray. This module remains only for an explicit manual rollback. +""" + import ast import datetime import glob diff --git a/fastvideo/tests/modal/test_kernel_build_cache.py b/fastvideo/tests/modal/test_kernel_build_cache.py index e043564e7..3345518f6 100644 --- a/fastvideo/tests/modal/test_kernel_build_cache.py +++ b/fastvideo/tests/modal/test_kernel_build_cache.py @@ -106,6 +106,7 @@ def _patch_stable_metadata(monkeypatch) -> None: lambda: { "torch_version": "2.9.0", "torch_cuda_version": "12.8", + "torch_git_version": "stable-build-commit", "torch_file": "/opt/venv/lib/python3.12/site-packages/torch/__init__.py", "torch_config": "USE_CUDA=ON", "cxx11_abi": True, @@ -155,6 +156,41 @@ def test_kernel_only_main_change_republishes_trusted_l40s_artifact() -> None: assert '--output "${l40s_wheel_dir}/metadata.json"' in dockerfile +def test_ci_runner_image_targets_arm64_sm100_with_opencv_runtime() -> None: + workflow = yaml.load( + (REPO_ROOT / ".github/workflows/infra-build-image.yml").read_text(encoding="utf-8"), + Loader=yaml.BaseLoader, + ) + job = workflow["jobs"]["build-ci-runner-image"] + + assert job["with"]["architecture"] == "arm64" + assert job["with"]["runner"] == "ubuntu-24.04-arm" + assert job["with"]["tag_suffix"] == "py3.12-cuda13.0.0-sm100" + assert "CUDA_VERSION=13.0.0" in job["with"]["build_args"] + assert "UV_TORCH_BACKEND=cu130" in job["with"]["build_args"] + assert "TORCH_CUDA_ARCH_LIST=10.0" in job["with"]["build_args"] + + dockerfile = (REPO_ROOT / "docker/Dockerfile").read_text(encoding="utf-8") + assert " ffmpeg \\\n" in dockerfile + assert " libgl1 \\\n" in dockerfile + assert " libglib2.0-0 \\\n" in dockerfile + assert 'python -c "import cv2; print(\'OpenCV\', cv2.__version__)"' in dockerfile + + +def test_docker_image_bakes_modal_apt_and_rust_layer() -> None: + """The dormant Modal rollback image layers apt packages + rustup on top of + the release image (pr_test.py). Keep those packages baked into the release + image so the archived manual path remains reproducible.""" + modal_source = (REPO_ROOT / "fastvideo/tests/modal/pr_test.py").read_text(encoding="utf-8") + assert '.apt_install(\n "cmake", "pkg-config", "build-essential", "curl", "libssl-dev", "ffmpeg")' in modal_source + + dockerfile = (REPO_ROOT / "docker/Dockerfile").read_text(encoding="utf-8") + for package in ("cmake", "pkg-config", "build-essential", "libssl-dev", "curl", "ffmpeg"): + assert f" {package} \\\n" in dockerfile, f"{package} missing from the docker/Dockerfile apt layer" + assert "sh.rustup.rs | sh -s -- -y --default-toolchain stable" in dockerfile + assert "ENV PATH=/root/.cargo/bin:${PATH}" in dockerfile + + def test_cache_key_uses_resolved_arch_not_raw_env(monkeypatch, tmp_path) -> None: _patch_stable_metadata(monkeypatch) monkeypatch.setattr(kernel_build_cache, "_detect_arch_from_torch", lambda: "9.0a") @@ -174,6 +210,25 @@ def test_cache_key_uses_resolved_arch_not_raw_env(monkeypatch, tmp_path) -> None assert detected_l40s["cache_key"] != detected_hopper["cache_key"] +def test_cache_key_ignores_runtime_only_torch_config(monkeypatch, tmp_path) -> None: + _patch_stable_metadata(monkeypatch) + build_host = kernel_build_cache._build_metadata(tmp_path) + torch_metadata = kernel_build_cache._torch_metadata() + monkeypatch.setattr( + kernel_build_cache, + "_torch_metadata", + lambda: { + **torch_metadata, + "torch_config": torch_metadata["torch_config"] + "\nCUDA Runtime 13.0\nCuDNN 92.0", + }, + ) + + gpu_host = kernel_build_cache._build_metadata(tmp_path) + + assert gpu_host["torch"]["torch_config"] != build_host["torch"]["torch_config"] + assert gpu_host["cache_key"] == build_host["cache_key"] + + @pytest.mark.parametrize("environment_name", ["CFLAGS", "CXXFLAGS", "LDFLAGS"]) def test_cache_key_changes_with_build_flags(monkeypatch, tmp_path, environment_name) -> None: _patch_stable_metadata(monkeypatch) @@ -213,7 +268,7 @@ def test_cache_key_changes_with_compiler_or_torch_abi(monkeypatch, tmp_path) -> monkeypatch.setattr(kernel_build_cache, "_torch_metadata", lambda: {**torch_metadata, "cxx11_abi": False}) torch_abi_changed = kernel_build_cache._build_metadata(tmp_path) - assert baseline["schema_version"] == 3 + assert baseline["schema_version"] == 4 assert baseline["cache_key"] != compiler_changed["cache_key"] assert baseline["cache_key"] != torch_abi_changed["cache_key"] diff --git a/fastvideo/tests/modal/test_pr_test.py b/fastvideo/tests/modal/test_pr_test.py index c55a7a695..f2360bfcc 100644 --- a/fastvideo/tests/modal/test_pr_test.py +++ b/fastvideo/tests/modal/test_pr_test.py @@ -316,17 +316,71 @@ def test_run_test_command_uses_nonshared_kernel_install_before_tests(monkeypatch assert not hasattr(module, "kernel_cache_vol") -def test_run_unit_test_collects_modal_cache_runner_tests(monkeypatch): +def test_run_unit_test_uses_shared_command(monkeypatch): module = _load_pr_test_module(monkeypatch) commands = [] monkeypatch.setattr(module, "run_test", commands.append) module.run_unit_test() - assert len(commands) == 1 + assert commands == ["bash .buildkite/scripts/unit_test.sh"] + unit_command = (Path(__file__).resolve().parents[3] / ".buildkite/scripts/unit_test.sh").read_text() for test_path in ( "./fastvideo/tests/modal/test_kernel_build_cache.py", "./fastvideo/tests/modal/test_pr_test.py", "./fastvideo/tests/modal/test_ssim_test.py", ): - assert test_path in commands[0] + assert test_path in unit_command + + +def test_wave1_lane_functions_use_shared_scripts(monkeypatch): + """Modal and the self-hosted CI runner must execute the same per-lane scripts so + their test selections cannot drift (same contract as the unit lane).""" + module = _load_pr_test_module(monkeypatch) + commands = [] + monkeypatch.setattr(module, "run_test", commands.append) + + hf_prefix = "export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && " + module.run_kernel_tests() + module.run_inference_tests_vmoba() + module.run_golden_gate_tests() + module.run_encoder_tests() + module.run_vae_tests() + module.run_transformer_tests() + module.run_inference_lora_tests() + module.run_distill_dmd_tests() + module.run_train_framework_tests() + + assert commands == [ + "bash .buildkite/scripts/lanes/kernel_tests.sh", + "bash .buildkite/scripts/lanes/inference_vmoba.sh", + hf_prefix + "bash .buildkite/scripts/lanes/golden_gate.sh", + hf_prefix + "bash .buildkite/scripts/lanes/encoder.sh", + hf_prefix + "bash .buildkite/scripts/lanes/vae.sh", + hf_prefix + "FASTVIDEO_FA4=0 bash .buildkite/scripts/lanes/transformer.sh", + "bash .buildkite/scripts/lanes/inference_lora.sh", + "FASTVIDEO_FA4=0 bash .buildkite/scripts/lanes/distillation_dmd.sh", + hf_prefix + "FASTVIDEO_FA4=0 bash .buildkite/scripts/lanes/train_framework.sh", + ] + + lanes_dir = Path(__file__).resolve().parents[3] / ".buildkite/scripts/lanes" + for script, payload in { + "kernel_tests.sh": "pytest fastvideo-kernel/tests/ -vs", + "inference_vmoba.sh": "python fastvideo/tests/inference/vmoba/test_vmoba_inference.py", + "golden_gate.sh": 'exec pytest "$golden_root" -vs', + "encoder.sh": "pytest ./fastvideo/tests/encoders -vs", + "vae.sh": "pytest ./fastvideo/tests/vaes -vs", + "transformer.sh": "pytest ./fastvideo/tests/transformers -vs", + "inference_lora.sh": "pytest ./fastvideo/tests/inference/lora/test_lora_inference_similarity.py -vs", + "distillation_dmd.sh": "pytest ./fastvideo/tests/training/distill/test_distill_dmd.py -vs", + "train_framework.sh": "pytest ./fastvideo/tests/train/models ./fastvideo/tests/train/methods -vs", + "eval.sh": "pytest ./fastvideo/tests/eval -vs", + }.items(): + assert payload in (lanes_dir / script).read_text(), script + assert "golden_root=./fastvideo/tests/golden_gate" in (lanes_dir / "golden_gate.sh").read_text() + + # run_eval_tests goes through run_test_command (custom install extras); + # pin its shared script + install command textually. + eval_source = (Path(__file__).resolve().parent / "pr_test.py").read_text() + assert "bash .buildkite/scripts/lanes/eval.sh" in eval_source + assert 'install_command=\'uv pip install -e ".[test,eval-full]"\'' in eval_source diff --git a/fastvideo/tests/ssim/AGENTS.md b/fastvideo/tests/ssim/AGENTS.md index b7f0034f3..f98be49ca 100644 --- a/fastvideo/tests/ssim/AGENTS.md +++ b/fastvideo/tests/ssim/AGENTS.md @@ -27,16 +27,20 @@ videos and compare them against device-specific reference videos. `pytest fastvideo/tests/ssim/test_wan_t2v_similarity.py -vs` - Single model split: `FASTVIDEO_SSIM_MODEL_ID= pytest fastvideo/tests/ssim/test_wan_t2v_similarity.py -vs` -- Modal orchestrator (CI-style scheduling): - `modal run fastvideo/tests/modal/ssim_test.py` +- Slurm orchestrator (CI-style four-GPU scheduling): + `python fastvideo/tests/ssim/ci_runner.py` +- Focused Slurm selection: + `python fastvideo/tests/ssim/ci_runner.py --test-file test_wan_t2v_similarity.py` ## Authoring Rules - Name files `test__similarity.py`. - Set `REQUIRED_GPUS = ` near the top of each test module. - For multi-model suites, keep configs in `*_MODEL_TO_PARAMS` dictionaries. - The Modal scheduler auto-discovers these keys and runs one subprocess per + The Slurm scheduler auto-discovers these keys and runs one subprocess per model id. - Keep runs deterministic when possible (fixed prompts/seeds/frames/backend). +- Keep SSIM filenames in `test_[a-z0-9_]+.py` form; the merge planner passes + validated basenames rather than arbitrary pytest expressions. - Persist metrics with `write_ssim_results(...)`. - Use `pytest.skip(...)` for unsupported hardware, missing assets, or insufficient GPU count. diff --git a/fastvideo/tests/ssim/README.md b/fastvideo/tests/ssim/README.md index 0e20acbdd..922615ba7 100644 --- a/fastvideo/tests/ssim/README.md +++ b/fastvideo/tests/ssim/README.md @@ -60,12 +60,17 @@ causal videos were generated on commit b318063c0a4618f1d5d99ea82ca67a06aad0d19d ## Adding a New SSIM Test -SSIM CI runs in one Modal instance (`L40S:8`). The orchestrator -(`fastvideo/tests/modal/ssim_test.py`) clones/builds/installs once, then -schedules multiple `pytest` subprocesses by GPU demand. Logs are printed in -deterministic order for failed tasks (file name, then model id), and -fail-fast is enabled: if one task fails, active tasks are terminated and the -full SSIM step fails. +SSIM CI runs in one four-GPU Slinky Slurm lease. The orchestrator +(`fastvideo/tests/ssim/ci_runner.py`) runs after the shared worker has cloned, +built, and installed the exact PR SHA, then schedules multiple `pytest` +subprocesses by GPU demand. Logs are printed in deterministic order for failed +tasks (file name, then model id), and fail-fast is enabled: if one task fails, +active tasks are terminated and the full SSIM step fails. + +Change-aware merge gates can pass repeated `--test-file ` arguments, +so a model-family PR runs only its owning SSIM files. Shared scheduler/helper +changes, `/test ssim`, `/test full`, and the weekly `main` schedule run the +complete discovered matrix. The orchestrator auto-discovers every `test_*.py` file here, so no CI config changes are needed when adding a new test. To declare how many GPUs your test @@ -107,8 +112,9 @@ when intentionally replacing reviewed references. ## Generation Details -SSIM CI currently schedules work across the GPUs configured in -`fastvideo/tests/modal/ssim_test.py`. +SSIM CI schedules work across the four GPUs leased by +`.buildkite/scripts/lanes/ssim.sh` using +`fastvideo/tests/ssim/ci_runner.py`. ## Generation Parameters diff --git a/fastvideo/tests/ssim/ci_runner.py b/fastvideo/tests/ssim/ci_runner.py new file mode 100644 index 000000000..8fc23438f --- /dev/null +++ b/fastvideo/tests/ssim/ci_runner.py @@ -0,0 +1,289 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Run the SSIM suite across the GPUs assigned to one Slurm CI lane. + +The Buildkite host grants this process an isolated four-GPU lease. This +scheduler discovers each test's ``REQUIRED_GPUS`` declaration and optional +``*_MODEL_TO_PARAMS`` dictionaries without importing the test modules, then +packs independent pytest processes onto the lease. The first failure stops +new work and terminates active siblings. +""" + +from __future__ import annotations + +import argparse +import ast +import os +import re +import signal +import subprocess +import tempfile +import time +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + +MAX_GPUS = 4 +MASTER_PORT_RANGE_SIZE = 100 +TERMINATE_TIMEOUT_SECONDS = 30 + + +@dataclass(frozen=True) +class SSIMTask: + task_id: int + test_file: str + required_gpus: int + model_id: str | None = None + + @property + def name(self) -> str: + suffix = f"::{self.model_id}" if self.model_id else "" + return f"{Path(self.test_file).name}{suffix}" + + @property + def sort_key(self) -> tuple[str, str]: + return (Path(self.test_file).name, self.model_id or "") + + +@dataclass +class RunningTask: + task: SSIMTask + process: subprocess.Popen[str] + gpu_ids: list[str] + log_path: Path + log_handle: object + + +def extract_required_gpus(path: Path) -> int: + for line in path.read_text(encoding="utf-8").splitlines(): + match = re.match(r"^REQUIRED_GPUS\s*=\s*(\d+)", line) + if match: + return int(match.group(1)) + return 1 + + +def extract_model_ids(path: Path) -> list[str]: + module = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + model_ids: list[str] = [] + for node in module.body: + target_names: list[str] = [] + value: ast.expr | None = None + if isinstance(node, ast.Assign): + target_names = [target.id for target in node.targets if isinstance(target, ast.Name)] + value = node.value + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + target_names = [node.target.id] + value = node.value + if not any(name.endswith("MODEL_TO_PARAMS") for name in target_names) or not isinstance(value, ast.Dict): + continue + for key in value.keys: + if isinstance(key, ast.Constant) and isinstance(key.value, str): + model_ids.append(key.value) + return list(dict.fromkeys(model_ids)) + + +def discover_tasks(ssim_dir: Path, test_files: Sequence[str] | None = None) -> list[SSIMTask]: + paths = sorted(ssim_dir.glob("test_*.py")) + if test_files: + requested = set(test_files) + invalid = sorted(name for name in requested if not re.fullmatch(r"test_[a-z0-9_]+\.py", name)) + if invalid: + raise ValueError(f"Invalid SSIM test file selection: {invalid}") + available = {path.name: path for path in paths} + missing = sorted(requested - set(available)) + if missing: + raise ValueError(f"Selected SSIM test files do not exist: {missing}") + paths = [path for path in paths if path.name in requested] + + tasks: list[SSIMTask] = [] + for path in paths: + required_gpus = extract_required_gpus(path) + if not 1 <= required_gpus <= MAX_GPUS: + raise ValueError(f"{path} requests {required_gpus} GPUs; supported range is 1-{MAX_GPUS}") + model_ids = extract_model_ids(path) + if not model_ids: + model_ids = [None] + for model_id in model_ids: + tasks.append( + SSIMTask( + task_id=len(tasks), + test_file=f"./fastvideo/tests/ssim/{path.name}", + required_gpus=required_gpus, + model_id=model_id, + )) + return sorted(tasks, key=lambda task: task.sort_key) + + +def visible_gpu_ids() -> list[str]: + raw = os.environ.get("CUDA_VISIBLE_DEVICES", "") + if not raw: + return [str(index) for index in range(MAX_GPUS)] + gpu_ids = list(dict.fromkeys(part.strip() for part in raw.split(",") if part.strip())) + if not gpu_ids: + raise RuntimeError("CUDA_VISIBLE_DEVICES does not contain any GPU ids") + return gpu_ids + + +def task_master_port(task_id: int, base_port: str | None = None) -> str: + """Return a task-specific port inside the runner-assigned lane range.""" + if not 0 <= task_id < MASTER_PORT_RANGE_SIZE: + raise ValueError(f"SSIM task id is outside the runner-assigned 100-port range: {task_id}") + try: + port = int(base_port or "29500") + task_id + except ValueError as error: + raise ValueError(f"Invalid MASTER_PORT: {base_port!r}") from error + if not 1 <= port <= 65535: + raise ValueError(f"SSIM task master port is outside 1-65535: {port}") + return str(port) + + +def spawn_task(task: SSIMTask, gpu_ids: list[str], log_dir: Path, extra_args: list[str]) -> RunningTask: + safe_name = re.sub(r"[^A-Za-z0-9_.-]+", "_", task.name) + log_path = log_dir / f"{task.task_id:03d}_{safe_name}.log" + log_handle = log_path.open("w", encoding="utf-8") + env = os.environ.copy() + env["CUDA_VISIBLE_DEVICES"] = ",".join(gpu_ids) + env["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:False" + env.setdefault("MASTER_ADDR", "127.0.0.1") + env["MASTER_PORT"] = task_master_port(task.task_id, env.get("MASTER_PORT")) + if task.model_id: + env["FASTVIDEO_SSIM_MODEL_ID"] = task.model_id + else: + env.pop("FASTVIDEO_SSIM_MODEL_ID", None) + process = subprocess.Popen( + ["pytest", task.test_file, "-vs", *extra_args], + env=env, + stdout=log_handle, + stderr=subprocess.STDOUT, + start_new_session=True, + text=True, + ) + return RunningTask(task, process, gpu_ids, log_path, log_handle) + + +def terminate(tasks: list[RunningTask]) -> None: + for running in tasks: + if running.process.poll() is None: + try: + os.killpg(running.process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + deadline = time.monotonic() + TERMINATE_TIMEOUT_SECONDS + while time.monotonic() < deadline and any(task.process.poll() is None for task in tasks): + time.sleep(1) + for running in tasks: + if running.process.poll() is None: + try: + os.killpg(running.process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def run(tasks: list[SSIMTask], extra_args: list[str]) -> int: + available = visible_gpu_ids() + if max((task.required_gpus for task in tasks), default=0) > len(available): + raise RuntimeError(f"SSIM tasks require more GPUs than the {len(available)} visible devices") + order = {gpu_id: index for index, gpu_id in enumerate(available)} + pending = list(tasks) + running: list[RunningTask] = [] + results: dict[int, tuple[str, int, Path | None]] = {} + log_dir = Path(tempfile.mkdtemp(prefix="fastvideo-ssim-logs-")) + failed = False + + try: + while pending or running: + while not failed: + next_index = next( + (index for index, task in enumerate(pending) if task.required_gpus <= len(available)), + None, + ) + if next_index is None: + break + task = pending.pop(next_index) + assigned = available[:task.required_gpus] + del available[:task.required_gpus] + running_task = spawn_task(task, assigned, log_dir, extra_args) + running.append(running_task) + print(f"Started {task.name} on GPUs {','.join(assigned)}", flush=True) + + completed = [task for task in running if task.process.poll() is not None] + for running_task in completed: + returncode = running_task.process.wait() + running_task.log_handle.close() + available.extend(running_task.gpu_ids) + available.sort(key=order.__getitem__) + status = "passed" if returncode == 0 else "failed" + results[running_task.task.task_id] = (status, returncode, running_task.log_path) + running.remove(running_task) + print(f"Finished {running_task.task.name} with exit code {returncode}", flush=True) + failed = failed or returncode != 0 + + if failed and running: + print("Fail-fast triggered: terminating active SSIM tasks.", flush=True) + terminate(running) + for running_task in running: + returncode = running_task.process.wait() + running_task.log_handle.close() + results[running_task.task.task_id] = ("terminated", returncode, running_task.log_path) + running.clear() + elif running and not completed: + time.sleep(1) + elif failed: + break + except BaseException: + terminate(running) + raise + + for task in pending: + results[task.task_id] = ("skipped", -1, None) + + print("\nSSIM summary:") + counts = {"passed": 0, "failed": 0, "terminated": 0, "skipped": 0} + for task in tasks: + status, returncode, log_path = results[task.task_id] + counts[status] += 1 + print(f" {status:10} {task.name} (gpus={task.required_gpus}, rc={returncode})") + if status == "failed" and log_path: + print(f"\n--- {task.name} failure log ---") + print(log_path.read_text(encoding="utf-8", errors="replace")) + print(" " + ", ".join(f"{name}={count}" for name, count in counts.items())) + return 1 if any(counts[name] for name in ("failed", "terminated", "skipped")) else 0 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--full-quality", action="store_true") + parser.add_argument("--reference-repo", default="") + parser.add_argument("--skip-reference-download", action="store_true") + parser.add_argument("--bootstrap-mode", action="store_true") + parser.add_argument( + "--test-file", + action="append", + default=[], + help="Run one SSIM test basename; repeat for a focused merge gate.", + ) + parser.add_argument("-k", dest="pytest_k", default="") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + pytest_args: list[str] = [] + if args.full_quality: + pytest_args.append("--ssim-full-quality") + if args.reference_repo: + pytest_args.extend(["--ssim-reference-repo", args.reference_repo]) + if args.skip_reference_download: + pytest_args.append("--skip-ssim-reference-download") + if args.bootstrap_mode: + pytest_args.append("--ssim-bootstrap-mode") + if args.pytest_k: + pytest_args.extend(["-k", args.pytest_k]) + tasks = discover_tasks(Path("fastvideo/tests/ssim"), args.test_file) + if not tasks: + raise RuntimeError("No SSIM tests discovered") + print(f"Discovered {len(tasks)} SSIM tasks across {len(visible_gpu_ids())} GPUs") + return run(tasks, pytest_args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/fastvideo/tests/train/methods/grad_norm_refs.json b/fastvideo/tests/train/methods/grad_norm_refs.json index bef3789ea..2b3a1092f 100644 --- a/fastvideo/tests/train/methods/grad_norm_refs.json +++ b/fastvideo/tests/train/methods/grad_norm_refs.json @@ -4,7 +4,7 @@ "L40S": 4.1553 }, "test_ltx2_3_finetune": { - "GB200": 2.2158 + "GB200": 2.6692 }, "test_ltx2_finetune": { "GB200": 0.3127 diff --git a/fastvideo/tests/train/methods/grad_norm_regression.py b/fastvideo/tests/train/methods/grad_norm_regression.py index f579108a5..b45a47563 100644 --- a/fastvideo/tests/train/methods/grad_norm_regression.py +++ b/fastvideo/tests/train/methods/grad_norm_regression.py @@ -14,15 +14,15 @@ forward/backward is reproducible within bf16 reduction noise on a given GPU. Why device-keyed: grad norms differ across GPU architectures (kernels, accumulation order), so a single golden value can't cover every runner. The -JSON currently carries refs for the two GPUs we actually run on — ``L40S`` (CI) -and ``GB200`` (our Blackwell dev box; ``B200`` maps to the same key). +JSON carries legacy ``L40S`` references and active Slinky CI ``GB200`` +references (``B200`` maps to the same Blackwell key). Seeding a reference for the current device: -- **CI / L40S** — invoke ``modal run`` against ``seed_grad_norm_references`` in - ``fastvideo/tests/modal/pr_test.py`` (pinned to ``gpu="L40S:1"``), then copy - the recorded value from the log into ``grad_norm_refs.json``. -- **Local / non-L40S GPUs** — on that workstation:: +- **CI / GB200** — run the target train-framework lane on the Slinky Slurm + runner with update mode only during an intentional reviewed reseed, then + copy the recorded value from the log into ``grad_norm_refs.json``. +- **Local / other GPUs** — on that workstation:: FASTVIDEO_GRADNORM_UPDATE=1 \\ pytest fastvideo/tests/train/methods -vs -rs diff --git a/fastvideo/tests/train/methods/test_matrixgame2_finetune.py b/fastvideo/tests/train/methods/test_matrixgame2_finetune.py index dc32afa80..9a591b983 100644 --- a/fastvideo/tests/train/methods/test_matrixgame2_finetune.py +++ b/fastvideo/tests/train/methods/test_matrixgame2_finetune.py @@ -94,7 +94,7 @@ def test_matrixgame2_finetune_single_train_step(monkeypatch: pytest.MonkeyPatch) # Matrix-Game 2.0 uses its own parquet dataloader; stub it out so # construction does not require a real ``training.data.data_path``. monkeypatch.setattr( - "fastvideo.train.utils.dataloader." + "fastvideo.train.models.matrixgame2.matrixgame2." "build_parquet_matrixgame2_train_dataloader", lambda *args, **kwargs: None, ) diff --git a/fastvideo/tests/train/methods/test_wan_causal_dfsft.py b/fastvideo/tests/train/methods/test_wan_causal_dfsft.py index 16dd606dd..7692e9b24 100644 --- a/fastvideo/tests/train/methods/test_wan_causal_dfsft.py +++ b/fastvideo/tests/train/methods/test_wan_causal_dfsft.py @@ -68,7 +68,7 @@ def test_wan_causal_dfsft_single_train_step(monkeypatch: pytest.MonkeyPatch) -> # ``single_train_step``, so the parquet train dataloader that # ``init_preprocessors`` builds is never iterated. Stub it out so # construction does not require a real ``training.data.data_path``: - # the minimal fixture deliberately omits one, and the Modal CI job + # the minimal fixture deliberately omits one, and the Slurm CI worker # mounts model weights rather than a parquet dataset. monkeypatch.setattr( "fastvideo.train.utils.dataloader." diff --git a/fastvideo/tests/train/methods/test_wan_finetune.py b/fastvideo/tests/train/methods/test_wan_finetune.py index 5867587ff..531579384 100644 --- a/fastvideo/tests/train/methods/test_wan_finetune.py +++ b/fastvideo/tests/train/methods/test_wan_finetune.py @@ -78,7 +78,7 @@ def test_wan_finetune_single_train_step(monkeypatch: pytest.MonkeyPatch) -> None # ``single_train_step``, so the parquet train dataloader that # ``init_preprocessors`` builds is never iterated. Stub it out so # construction does not require a real ``training.data.data_path``: - # the minimal fixture deliberately omits one, and the Modal CI job + # the minimal fixture deliberately omits one, and the Slurm CI worker # mounts model weights rather than a parquet dataset. monkeypatch.setattr( "fastvideo.train.utils.dataloader." diff --git a/fastvideo/tests/train/models/test_wan_override_contract.py b/fastvideo/tests/train/models/test_wan_override_contract.py new file mode 100644 index 000000000..fc391c6a5 --- /dev/null +++ b/fastvideo/tests/train/models/test_wan_override_contract.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Signature contracts for Wan subclasses that reuse ``predict_noise``.""" + +import inspect + +import pytest + +from fastvideo.train.models.hunyuan.hunyuan import HunyuanModel +from fastvideo.train.models.longcat.longcat import LongCatModel +from fastvideo.train.models.matrixgame2.matrixgame2 import MatrixGame2Model + + +@pytest.mark.parametrize("model_cls", [HunyuanModel, LongCatModel, MatrixGame2Model]) +def test_distill_kwargs_accept_wan_teacher_forcing_keywords(model_cls: type) -> None: + parameters = inspect.signature(model_cls._build_distill_input_kwargs).parameters + + assert "clean_x" in parameters + assert "aug_t" in parameters + + +@pytest.mark.parametrize("model_cls", [LongCatModel, MatrixGame2Model]) +def test_predict_noise_override_preserves_wan_teacher_forcing_keywords(model_cls: type) -> None: + parameters = inspect.signature(model_cls.predict_noise).parameters + + assert "clean_x" in parameters + assert "aug_t" in parameters diff --git a/fastvideo/tests/training/VSA/test_training_loss_VSA.py b/fastvideo/tests/training/VSA/test_training_loss_VSA.py index 7f81a6d6a..e576c3f5c 100644 --- a/fastvideo/tests/training/VSA/test_training_loss_VSA.py +++ b/fastvideo/tests/training/VSA/test_training_loss_VSA.py @@ -56,7 +56,7 @@ def run_worker(): def test_distributed_training(): """Test the distributed training setup""" - os.environ["WANDB_MODE"] = "online" + os.environ.setdefault("WANDB_MODE", "offline") data_dir = Path("data/mini_dataset_i2v_VSA") @@ -79,8 +79,8 @@ def test_distributed_training(): device_name = torch.cuda.get_device_name() print(f"INFO: device: {device_name}") - if "H100" not in device_name: - raise ValueError(f"VSA training regression requires H100 GPUs, got: {device_name}") + if "H100" not in device_name and "B200" not in device_name: + raise ValueError(f"VSA training regression supports H100 and the GB200 Slurm CI target, got: {device_name}") reference_wandb_summary = json.load(open(reference_wandb_summary_file)) wandb_summary = json.load(open(summary_file)) diff --git a/fastvideo/tests/training/Vanilla/mfu_calculation.py b/fastvideo/tests/training/Vanilla/mfu_calculation.py index 33f3d76f4..d45f8e0e8 100644 --- a/fastvideo/tests/training/Vanilla/mfu_calculation.py +++ b/fastvideo/tests/training/Vanilla/mfu_calculation.py @@ -29,10 +29,10 @@ WANDB_SUMMARY_FILE = OUTPUT_DIR / "tracker/wandb/latest-run/files/wandb-summary. NUM_NODES = "1" NUM_GPUS_PER_NODE = "2" GRAD_ACCUM = "1" -MASTER_PORT = "29504" +MASTER_PORT = os.environ.get("MASTER_PORT", "29504") -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = MASTER_PORT +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", MASTER_PORT) def run_worker(): @@ -131,7 +131,7 @@ def run_worker(): def test_distributed_training(): """Test the distributed training setup""" - os.environ["WANDB_MODE"] = "online" + os.environ.setdefault("WANDB_MODE", "offline") data_dir = Path("data/crush-smol_processed_t2v") diff --git a/fastvideo/tests/training/Vanilla/test_training_loss.py b/fastvideo/tests/training/Vanilla/test_training_loss.py index c271b69e8..59adae0a5 100644 --- a/fastvideo/tests/training/Vanilla/test_training_loss.py +++ b/fastvideo/tests/training/Vanilla/test_training_loss.py @@ -1,7 +1,7 @@ import os -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29512" +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", "29512") import sys import subprocess from pathlib import Path @@ -58,7 +58,7 @@ def run_worker(): def test_distributed_training(): """Test the distributed training setup""" - os.environ["WANDB_MODE"] = "online" + os.environ.setdefault("WANDB_MODE", "offline") data_dir = Path("data/crush-smol_processed_t2v") @@ -100,6 +100,11 @@ def test_distributed_training(): reference_wandb_summary_file = l40s_reference_wandb_summary_file elif "H200" in device_name: reference_wandb_summary_file = h200_reference_wandb_summary_file + elif "B200" in device_name: + # GB200 is the Slurm CI target. Use the closest high-memory baseline; + # correctness fields remain gated while the generous timing bounds + # intentionally absorb hardware throughput differences. + reference_wandb_summary_file = h200_reference_wandb_summary_file else: raise ValueError(f"Unknown device: {device_name}") diff --git a/fastvideo/tests/training/distill/test_distill_dmd.py b/fastvideo/tests/training/distill/test_distill_dmd.py index 806780180..8d80811e8 100644 --- a/fastvideo/tests/training/distill/test_distill_dmd.py +++ b/fastvideo/tests/training/distill/test_distill_dmd.py @@ -1,7 +1,7 @@ import os -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29512" +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", "29512") import sys import subprocess from pathlib import Path diff --git a/fastvideo/tests/training/lora/test_lora_training.py b/fastvideo/tests/training/lora/test_lora_training.py index bf8edceb0..e5664d1ad 100644 --- a/fastvideo/tests/training/lora/test_lora_training.py +++ b/fastvideo/tests/training/lora/test_lora_training.py @@ -1,7 +1,7 @@ import os -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29514" +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", "29514") import sys import subprocess from pathlib import Path @@ -18,7 +18,7 @@ NUM_GPUS_PER_NODE = "2" def test_lora_training(): """Test the LoRA training setup""" - os.environ["WANDB_MODE"] = "online" + os.environ.setdefault("WANDB_MODE", "offline") data_dir = Path("data/crush-smol_processed_t2v") @@ -55,7 +55,8 @@ def test_lora_training(): summary_file = '/workspace/tracker/wandb/latest-run/files/wandb-summary.json' device_name = torch.cuda.get_device_name() - assert "L40S" in device_name, "Test must be run on L40S" + assert "L40S" in device_name or "B200" in device_name, (f"LoRA training regression supports L40S and the " + f"GB200 Slurm CI target, got {device_name}") reference_wandb_summary_file = l40s_reference_wandb_summary_file reference_wandb_summary = json.load(open(reference_wandb_summary_file)) diff --git a/fastvideo/tests/training/self-forcing/test_self_forcing.py b/fastvideo/tests/training/self-forcing/test_self_forcing.py index db42c2473..1dc64ea93 100644 --- a/fastvideo/tests/training/self-forcing/test_self_forcing.py +++ b/fastvideo/tests/training/self-forcing/test_self_forcing.py @@ -1,7 +1,7 @@ import os -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29513" +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", "29513") import sys import subprocess from pathlib import Path diff --git a/fastvideo/tests/training/test_trackers.py b/fastvideo/tests/training/test_trackers.py index cbfc503aa..393038161 100644 --- a/fastvideo/tests/training/test_trackers.py +++ b/fastvideo/tests/training/test_trackers.py @@ -2,6 +2,7 @@ from __future__ import annotations import builtins from io import BytesIO +import json from pathlib import Path import sys from types import ModuleType @@ -88,6 +89,37 @@ def test_wandb_video_uses_shared_default_fps( assert video_calls == [{"fps": 16, "format": "mp4"}] +def test_wandb_finish_writes_local_offline_summary( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + class FakeRun: + + def __init__(self) -> None: + self.dir = str(tmp_path / "wandb" / "offline-run" / "files") + Path(self.dir).mkdir(parents=True) + self.finished = False + + def log(self, metrics: dict[str, Any], step: int) -> None: + pass + + def finish(self) -> None: + self.finished = True + + run = FakeRun() + wandb = ModuleType("wandb") + wandb.init = lambda **_: run + monkeypatch.setitem(sys.modules, "wandb", wandb) + tracker = WandbTracker("project", str(tmp_path)) + + tracker.log({"train_loss": 0.25, "grad_norm": np.float32(1.5), "video": object()}, step=4) + tracker.finish() + + summary_path = Path(run.dir) / "wandb-summary.json" + assert json.loads(summary_path.read_text()) == {"grad_norm": 1.5, "train_loss": 0.25} + assert run.finished + + def test_prepare_video_array_tiles_and_pads_batches() -> None: video = np.zeros((3, 1, 3, 1, 1), dtype=np.uint8) video[0] = 1 diff --git a/fastvideo/tests/transformers/test_cosmos.py b/fastvideo/tests/transformers/test_cosmos.py index 50930dec4..bf8455099 100644 --- a/fastvideo/tests/transformers/test_cosmos.py +++ b/fastvideo/tests/transformers/test_cosmos.py @@ -18,8 +18,8 @@ from fastvideo.pipelines.pipeline_batch_info import ForwardBatch logger = init_logger(__name__) -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29504" +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", "29504") BASE_MODEL_PATH = "nvidia/Cosmos-Predict2-2B-Video2World" diff --git a/fastvideo/tests/transformers/test_cosmos2_5.py b/fastvideo/tests/transformers/test_cosmos2_5.py index a8d863915..b1e6d2e4e 100644 --- a/fastvideo/tests/transformers/test_cosmos2_5.py +++ b/fastvideo/tests/transformers/test_cosmos2_5.py @@ -33,8 +33,8 @@ if os.path.exists(COSMOS_PREDICT2_5_PATH): else: logger.warning(f"cosmos-predict2.5 not found at: {COSMOS_PREDICT2_5_PATH}") -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29505" +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", "29505") # COSMOS 2.5 model path - update this based on the actual HuggingFace model ID # The model has subdirectories: base/pre-trained, base/post-trained, auto/multiview, robot/action-cond diff --git a/fastvideo/tests/transformers/test_hunyuangamecraft.py b/fastvideo/tests/transformers/test_hunyuangamecraft.py index e4e88f461..5660b7a3b 100644 --- a/fastvideo/tests/transformers/test_hunyuangamecraft.py +++ b/fastvideo/tests/transformers/test_hunyuangamecraft.py @@ -23,10 +23,8 @@ from fastvideo.pipelines.pipeline_batch_info import ForwardBatch logger = init_logger(__name__) -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29507" -os.environ.setdefault("TORCHDYNAMO_DISABLE", "1") -os.environ.setdefault("DISABLE_SP", "1") +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", "29507") # Path to converted weights (local path, not HuggingFace) TRANSFORMER_PATH = "official_weights/hunyuan-gamecraft/transformer" @@ -39,12 +37,19 @@ REFERENCE_LATENT = 42351.12903189659 @pytest.mark.usefixtures("distributed_setup") -def test_hunyuangamecraft_transformer(): +def test_hunyuangamecraft_transformer(monkeypatch: pytest.MonkeyPatch): """Test HunyuanGameCraft transformer regression.""" if not os.path.exists(TRANSFORMER_PATH): pytest.skip(f"Weights not found at {TRANSFORMER_PATH}") + # Keep these model-specific overrides local to this test. Setting + # TORCHDYNAMO_DISABLE during module collection disables torch.compile for + # every later transformer test in the same pytest process; FakeTensor + # compile checks then execute Triton kernels against fake CUDA pointers. + monkeypatch.setenv("TORCHDYNAMO_DISABLE", "1") + monkeypatch.setenv("DISABLE_SP", "1") + sp_rank = get_sp_parallel_rank() sp_world_size = get_sp_world_size() diff --git a/fastvideo/tests/transformers/test_hyworld.py b/fastvideo/tests/transformers/test_hyworld.py index 6e274a595..d2887ee66 100644 --- a/fastvideo/tests/transformers/test_hyworld.py +++ b/fastvideo/tests/transformers/test_hyworld.py @@ -20,8 +20,8 @@ from fastvideo.utils import maybe_download_model logger = init_logger(__name__) -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29503" +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", "29503") MODEL_PATH = maybe_download_model("FastVideo/HY-WorldPlay-Bidirectional-Diffusers") TRANSFORMER_PATH = os.path.join(MODEL_PATH, "transformer") @@ -29,6 +29,11 @@ REFERENCE_LATENTS = { AttentionBackendEnum.FLASH_ATTN: -197132.85557549074, AttentionBackendEnum.TORCH_SDPA: -211007.95158730447, } +GB200_REFERENCE_LATENTS = { + # Blackwell uses a different deterministic FlashAttention reduction path. + # Measured on the Slinky NVIDIA GB200 CI tray (build 4967). + AttentionBackendEnum.FLASH_ATTN: -201149.9431345705, +} @pytest.mark.usefixtures("distributed_setup") @@ -146,7 +151,9 @@ def test_hyworld_transformer(): latent = output.double().sum().item() - reference_latent = REFERENCE_LATENTS[attention_backend] + device_name = torch.cuda.get_device_name(device) + device_references = GB200_REFERENCE_LATENTS if "B200" in device_name else REFERENCE_LATENTS + reference_latent = device_references.get(attention_backend, REFERENCE_LATENTS[attention_backend]) diff = abs(reference_latent - latent) relative_diff = diff / abs(reference_latent) logger.info(f"Reference latent: {reference_latent}, Current latent: {latent}") diff --git a/fastvideo/tests/transformers/test_wanvideo.py b/fastvideo/tests/transformers/test_wanvideo.py index 6a6e2f509..05ddaca24 100644 --- a/fastvideo/tests/transformers/test_wanvideo.py +++ b/fastvideo/tests/transformers/test_wanvideo.py @@ -18,8 +18,8 @@ from fastvideo.pipelines.pipeline_batch_info import ForwardBatch logger = init_logger(__name__) -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29503" +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", "29503") BASE_MODEL_PATH = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" MODEL_PATH = maybe_download_model(BASE_MODEL_PATH, local_dir=os.path.join("data", BASE_MODEL_PATH)) diff --git a/fastvideo/tests/vaes/test_hunyuan15_vae.py b/fastvideo/tests/vaes/test_hunyuan15_vae.py index ab1c0ddf2..000f8c702 100644 --- a/fastvideo/tests/vaes/test_hunyuan15_vae.py +++ b/fastvideo/tests/vaes/test_hunyuan15_vae.py @@ -18,8 +18,8 @@ from fastvideo.utils import maybe_download_model logger = init_logger(__name__) -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29503" +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", "29503") BASE_MODEL_PATH = "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v" MODEL_PATH = maybe_download_model(BASE_MODEL_PATH, local_dir=os.path.join("data", BASE_MODEL_PATH)) diff --git a/fastvideo/tests/vaes/test_wan_vae.py b/fastvideo/tests/vaes/test_wan_vae.py index b60973ac1..3a0527675 100644 --- a/fastvideo/tests/vaes/test_wan_vae.py +++ b/fastvideo/tests/vaes/test_wan_vae.py @@ -16,8 +16,8 @@ from torch.testing import assert_close logger = init_logger(__name__) -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29503" +os.environ.setdefault("MASTER_ADDR", "localhost") +os.environ.setdefault("MASTER_PORT", "29503") BASE_MODEL_PATH = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" MODEL_PATH = maybe_download_model(BASE_MODEL_PATH, local_dir=os.path.join("data", BASE_MODEL_PATH)) diff --git a/fastvideo/train/models/hunyuan/hunyuan.py b/fastvideo/train/models/hunyuan/hunyuan.py index d3beeee60..2c2b17167 100644 --- a/fastvideo/train/models/hunyuan/hunyuan.py +++ b/fastvideo/train/models/hunyuan/hunyuan.py @@ -151,12 +151,16 @@ class HunyuanModel(WanModel): noise_input: torch.Tensor, timestep: torch.Tensor, text_dict: dict[str, torch.Tensor] | None, + clean_x: torch.Tensor | None = None, + aug_t: torch.Tensor | None = None, ) -> dict[str, Any]: """Build transformer forward kwargs for Hunyuan. Unlike Wan, Hunyuan does not use encoder_attention_mask or return_dict in its forward signature. """ + if clean_x is not None or aug_t is not None: + raise NotImplementedError("HunyuanModel does not support clean-history teacher forcing") if text_dict is None: raise ValueError("text_dict cannot be None for " "Hunyuan forward pass") diff --git a/fastvideo/train/models/longcat/longcat.py b/fastvideo/train/models/longcat/longcat.py index 9349f66fc..8e78d10a8 100644 --- a/fastvideo/train/models/longcat/longcat.py +++ b/fastvideo/train/models/longcat/longcat.py @@ -70,7 +70,11 @@ class LongCatModel(WanModel): noise_input: torch.Tensor, timestep: torch.Tensor, text_dict: dict[str, torch.Tensor] | None, + clean_x: torch.Tensor | None = None, + aug_t: torch.Tensor | None = None, ) -> dict[str, Any]: + if clean_x is not None or aug_t is not None: + raise NotImplementedError("LongCatModel does not support clean-history teacher forcing") if text_dict is None: raise ValueError("text_dict cannot be None for LongCat distillation") @@ -96,6 +100,8 @@ class LongCatModel(WanModel): conditional: bool, cfg_uncond: dict[str, Any] | None = None, attn_kind: Literal["dense", "vsa"] = "dense", + clean_x: torch.Tensor | None = None, + aug_t: torch.Tensor | None = None, ) -> torch.Tensor: """Adapt LongCat's sign convention to FineTuneMethod's target. @@ -128,5 +134,7 @@ class LongCatModel(WanModel): conditional=conditional, cfg_uncond=cfg_uncond, attn_kind=attn_kind, + clean_x=clean_x, + aug_t=aug_t, ) return -pred diff --git a/fastvideo/train/models/matrixgame2/matrixgame2.py b/fastvideo/train/models/matrixgame2/matrixgame2.py index 54f127f34..dea6b6daf 100644 --- a/fastvideo/train/models/matrixgame2/matrixgame2.py +++ b/fastvideo/train/models/matrixgame2/matrixgame2.py @@ -170,7 +170,11 @@ class MatrixGame2Model(WanModel): noise_input: torch.Tensor, timestep: torch.Tensor, text_dict: dict[str, Any] | None, + clean_x: torch.Tensor | None = None, + aug_t: torch.Tensor | None = None, ) -> dict[str, Any]: + if clean_x is not None or aug_t is not None: + raise NotImplementedError("Matrix-Game 2.0 does not support clean-history teacher forcing") if text_dict is None: raise ValueError("text_dict cannot be None for Matrix-Game 2.0") hidden_states = noise_input.permute(0, 2, 1, 3, 4) @@ -212,6 +216,8 @@ class MatrixGame2Model(WanModel): conditional: bool, cfg_uncond: dict[str, Any] | None = None, attn_kind: Literal["dense", "vsa"] = "dense", + clean_x: torch.Tensor | None = None, + aug_t: torch.Tensor | None = None, ) -> torch.Tensor: # Streaming long tuning re-anchors each score window on its first # image/latent, so pass window metadata through existing dict plumbing. @@ -227,6 +233,8 @@ class MatrixGame2Model(WanModel): conditional=conditional, cfg_uncond=cfg_uncond, attn_kind=attn_kind, + clean_x=clean_x, + aug_t=aug_t, ) finally: for text_dict in (batch.conditional_dict, batch.unconditional_dict): diff --git a/fastvideo/training/trackers.py b/fastvideo/training/trackers.py index aa108696a..068870f1b 100644 --- a/fastvideo/training/trackers.py +++ b/fastvideo/training/trackers.py @@ -10,6 +10,7 @@ from __future__ import annotations from collections.abc import Iterable, Iterator import contextlib import copy +import json import math import os import pathlib @@ -28,6 +29,7 @@ logger = init_logger(__name__) _DEFAULT_VIDEO_FPS = 16 _MISSING_ARTIFACT = object() +_MISSING_SUMMARY_VALUE = object() def _sanitize_wandb_config(value: Any) -> Any: @@ -62,6 +64,19 @@ def _sanitize_wandb_config(value: Any) -> Any: return repr(value) +def _local_summary_value(value: Any) -> Any: + """Return a JSON-safe scalar for the local W&B compatibility summary.""" + if value is None or isinstance(value, str | int | float | bool): + return value + if isinstance(value, Enum): + return _local_summary_value(value.value) + if isinstance(value, np.generic): + return _local_summary_value(value.item()) + if isinstance(value, torch.Tensor) and value.numel() == 1: + return _local_summary_value(value.detach().cpu().item()) + return _MISSING_SUMMARY_VALUE + + def _prepare_video_array(data: Any) -> np.ndarray: """Convert W&B-style TCHW/BTCHW video data into GIF-ready frames.""" if isinstance(data, torch.Tensor): @@ -315,6 +330,7 @@ class WandbTracker(BaseTracker): pathlib.Path(log_dir).mkdir(parents=True, exist_ok=True) self._wandb = wandb + self._local_summary: dict[str, Any] = {} self._run = wandb.init( entity=entity, project=experiment_name, @@ -328,6 +344,10 @@ class WandbTracker(BaseTracker): metrics = {**self._timed_metrics, **metrics} if metrics: self._run.log(metrics, step=step) + for key, value in metrics.items(): + summary_value = _local_summary_value(value) + if summary_value is not _MISSING_SUMMARY_VALUE: + self._local_summary[key] = summary_value self._timed_metrics = {} def log_artifacts(self, artifacts: dict[str, Any], step: int) -> None: @@ -351,7 +371,18 @@ class WandbTracker(BaseTracker): ) def finish(self) -> None: - self._run.finish() + # W&B 0.28 no longer emits the legacy wandb-summary.json file for + # offline runs. Preserve that stable local contract for regression + # tests and operators without requiring a cloud sync or API key. + try: + run_dir = getattr(self._run, "dir", None) + if run_dir: + summary_path = pathlib.Path(run_dir) / "wandb-summary.json" + temporary_path = summary_path.with_suffix(".json.tmp") + temporary_path.write_text(json.dumps(self._local_summary, sort_keys=True) + "\n", encoding="utf-8") + temporary_path.replace(summary_path) + finally: + self._run.finish() def video( self,