From 3f45f2da21a170c3fbdb7fdd1e90ad89697d9b97 Mon Sep 17 00:00:00 2001 From: Aryan Kumar <156166681+aryan5v@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:15:38 -0700 Subject: [PATCH] [bugfix] Repair DreamVerse and failing CI gates (#1958) Co-authored-by: sy3535 Co-authored-by: Aryan Kumar --- .buildkite/scripts/lanes/dreamverse.sh | 2 +- .buildkite/scripts/lanes/inference_vmoba.sh | 2 +- .buildkite/scripts/lanes/performance.sh | 7 +- .../workflows/ci-scheduled-performance.yml | 73 ++++++++++++ .pre-commit-config.yaml | 1 + .../web/e2e/mock-backed-generation.spec.ts | 31 +++++ apps/dreamverse/web/src/app/page.tsx | 2 +- .../web/src/components/LeaveSessionModal.tsx | 91 ++++++++------ docs-site/tests/cookbook.spec.ts | 19 +++ docs/contributing/ci_architecture.md | 92 ++++++++++++++- docs/contributing/env_vars.md | 1 + docs/contributing/performance_benchmarks.md | 9 +- docs/contributing/testing.md | 13 ++ fastvideo/envs.py | 5 + .../metrics/vbench/dynamic_degree/metric.py | 2 +- fastvideo/layers/triton_fused_norm.py | 5 +- .../lingbotworld2/causal_fast_pipeline.py | 3 +- .../tests/contract/test_ci_test_collection.py | 111 ++++++++++++++++++ .../tests/contract/test_golden_environment.py | 37 ++++++ .../tests/eval/test_registry_optional_deps.py | 21 ++++ fastvideo/tests/golden_gate/_harness.py | 24 ++-- fastvideo/tests/golden_gate/test_flux.py | 2 + .../tests/golden_gate/test_minimax_h3_t2v.py | 10 +- fastvideo/tests/golden_gate/test_sd35.py | 2 + .../inference/vmoba/test_vmoba_inference.py | 4 +- .../tests/layers/test_triton_fused_norm.py | 20 ++++ fastvideo/tests/modal/test_pr_test.py | 2 +- .../tests/performance/compare_baseline.py | 4 +- fastvideo/tests/ssim/test_flux2_similarity.py | 2 + .../tests/ssim/test_flux_t2i_similarity.py | 1 + .../ssim/test_hunyuan15_i2v_similarity.py | 1 + .../tests/ssim/test_kandinsky6_similarity.py | 1 + .../ssim/test_lingbot_fast_similarity.py | 6 +- .../tests/ssim/test_matrixgame3_similarity.py | 11 +- fastvideo/tests/ssim/test_sd35_similarity.py | 1 + .../ssim/test_stable_audio_similarity.py | 5 +- .../gb200_reference_wandb_summary_VSA.json | 21 ++++ .../training/VSA/test_training_loss_VSA.py | 5 +- .../test_compare_baseline_policy.py | 25 +++- tests/local_tests/test_cookbook_serving.py | 27 ----- 40 files changed, 582 insertions(+), 119 deletions(-) create mode 100644 .github/workflows/ci-scheduled-performance.yml create mode 100644 fastvideo/tests/contract/test_golden_environment.py create mode 100644 fastvideo/tests/eval/test_registry_optional_deps.py create mode 100644 fastvideo/tests/training/VSA/gb200_reference_wandb_summary_VSA.json diff --git a/.buildkite/scripts/lanes/dreamverse.sh b/.buildkite/scripts/lanes/dreamverse.sh index e1e1ef2e2..c8fe9171f 100755 --- a/.buildkite/scripts/lanes/dreamverse.sh +++ b/.buildkite/scripts/lanes/dreamverse.sh @@ -75,7 +75,7 @@ if [[ $machine_arch =~ ^(aarch64|arm64)$ ]]; then npm run e2e -- \ --project=chromium \ --project=mobile-chromium \ - --grep-invert='streams, plays, and surfaces a downloadable clip|starts a new project and switches back to the prior session|saved projects persist across a page reload' + --grep-invert='streams, plays, and surfaces a downloadable clip|starts a new project and switches back to the prior session|saved projects persist across a page reload|sends validated assets and plays a clearly labeled sample' else BACKEND_HOST=127.0.0.1 BACKEND_PORT="$BACKEND_PORT" CI=1 \ npm run e2e -- \ diff --git a/.buildkite/scripts/lanes/inference_vmoba.sh b/.buildkite/scripts/lanes/inference_vmoba.sh index e678d10e2..a5ecba1a1 100755 --- a/.buildkite/scripts/lanes/inference_vmoba.sh +++ b/.buildkite/scripts/lanes/inference_vmoba.sh @@ -2,4 +2,4 @@ # Canonical Slurm CI selection for the VMoBA-inference lane. set -euo pipefail -exec python fastvideo/tests/inference/vmoba/test_vmoba_inference.py +exec pytest ./fastvideo/tests/inference/vmoba/test_vmoba_inference.py -vs diff --git a/.buildkite/scripts/lanes/performance.sh b/.buildkite/scripts/lanes/performance.sh index 405dc8945..7a9b7d9d8 100755 --- a/.buildkite/scripts/lanes/performance.sh +++ b/.buildkite/scripts/lanes/performance.sh @@ -9,9 +9,12 @@ mkdir -p "$PERF_REPORTS_DIR" if [[ ${BUILDKITE_PULL_REQUEST:-false} =~ ^[1-9][0-9]*$ ]]; then export PERF_RUN_SOURCE=pr - export PERF_UPLOAD_POLICY=pass + export PERF_UPLOAD_POLICY=never elif [ "${BUILDKITE_BRANCH:-}" = main ] \ - && { [ "${BUILDKITE_SOURCE:-}" = schedule ] || [ "${TEST_SCOPE:-}" = full ]; }; then + && { [ "${BUILDKITE_SOURCE:-}" = schedule ] || [ "${TEST_SCOPE:-}" = full ] \ + || { [ "${BUILDKITE_SOURCE:-}" = api ] \ + && [ "${BUILDKITE_PIPELINE_SLUG:-}" = fastvideo-performance-lane ] \ + && [ "${TEST_SCOPE:-}" = direct ]; }; }; then export PERF_RUN_SOURCE=scheduled_main export PERF_UPLOAD_POLICY=always elif [ "${TEST_SCOPE:-}" = direct ]; then diff --git a/.github/workflows/ci-scheduled-performance.yml b/.github/workflows/ci-scheduled-performance.yml new file mode 100644 index 000000000..e3a16d71f --- /dev/null +++ b/.github/workflows/ci-scheduled-performance.yml @@ -0,0 +1,73 @@ +name: Scheduled Performance + +on: + schedule: + # Sunday 23:00 America/Los_Angeles, including daylight saving time. + - cron: "0 6 * * 1" + - cron: "0 7 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + trigger: + if: github.repository == 'hao-ai-lab/FastVideo' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + steps: + - name: Select the Pacific-time schedule + id: schedule + env: + EVENT_NAME: ${{ github.event_name }} + SCHEDULE_EXPR: ${{ github.event.schedule }} + run: | + set -euo pipefail + if [[ "$EVENT_NAME" == schedule ]]; then + expected='0 7 * * 1' + if [[ "$(TZ=America/Los_Angeles date +%z)" == -0700 ]]; then + expected='0 6 * * 1' + fi + if [[ "$SCHEDULE_EXPR" != "$expected" ]]; then + echo 'run=false' >> "$GITHUB_OUTPUT" + exit 0 + fi + fi + echo 'run=true' >> "$GITHUB_OUTPUT" + - name: Trigger weekly performance on Slinky Slurm + if: steps.schedule.outputs.run == 'true' + env: + BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }} + SOURCE_SHA: ${{ github.sha }} + SOURCE_BRANCH: ${{ github.event.repository.default_branch }} + BK_ORG: ${{ vars.BUILDKITE_ORG_SLUG }} + BK_PIPELINE: fastvideo-performance-lane + run: | + set -euo pipefail + if [[ ${#SOURCE_SHA} -ne 40 || "$SOURCE_SHA" == *[!0-9a-f]* ]]; then + echo 'SOURCE_SHA must be a full immutable commit SHA' >&2 + exit 1 + fi + payload=$(python3 - <<'PY' + import json + import os + + print(json.dumps({ + "commit": os.environ["SOURCE_SHA"], + "branch": os.environ["SOURCE_BRANCH"], + "message": "Weekly performance on Slinky Slurm", + "ignore_pipeline_branch_filters": True, + "env": { + "TEST_SCOPE": "direct", + "FULL_SUITE": "false", + "TEST_TYPE": "performance", + "PR_NUMBER": "false", + "PR_TITLE": "Scheduled performance", + }, + })) + PY + ) + curl -sS --fail-with-body --connect-timeout 10 --max-time 60 -X POST \ + "https://api.buildkite.com/v2/organizations/${BK_ORG}/pipelines/${BK_PIPELINE}/builds" \ + -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \ + -H "Content-Type: application/json" \ + --data-raw "$payload" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7b6adeafa..c0cc0d650 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,6 +21,7 @@ repos: rev: v0.43.0 hooks: - id: yapf + require_serial: true args: [--in-place, --verbose] language_version: python3.12 additional_dependencies: [toml] # TODO: Remove when yapf is upgraded diff --git a/apps/dreamverse/web/e2e/mock-backed-generation.spec.ts b/apps/dreamverse/web/e2e/mock-backed-generation.spec.ts index 9fb20df3a..a367b82fd 100644 --- a/apps/dreamverse/web/e2e/mock-backed-generation.spec.ts +++ b/apps/dreamverse/web/e2e/mock-backed-generation.spec.ts @@ -3,6 +3,37 @@ import { stat } from 'node:fs/promises'; import { test, expect, type WebSocket as PWWebSocket } from '@playwright/test'; test.describe('mock-backed generation smoke', () => { + test('leave dialog blocks navigation and keeps cancel and confirm usable', async ({ page, request }) => { + const health = await request.get('/healthz'); + const body = health.ok() ? await health.json() : {}; + test.skip(body.service !== 'ltx2-streaming-mock-server', 'Requires the local mock runtime.'); + await page.goto('/'); + await page.getByLabel('Continuation prompt').fill('A paper fox walks through a sunlit forest.'); + await page.getByRole('button', { name: 'Generate', exact: true }).click(); + const leave = page.getByRole('button', { name: 'Leave', exact: true }); + await expect(leave).toBeEnabled(); + await leave.click(); + const dialog = page.getByRole('dialog', { name: 'Leave session?' }); + await expect(dialog).toBeVisible(); + expect(await dialog.evaluate((element) => element.closest('.isolate') === null)).toBe(true); + await expect(page.getByRole('button', { name: 'Toggle sidebar' }).click({ trial: true, timeout: 1000 })).rejects.toThrow(); + for (let index = 0; index < 4; index++) { + await page.keyboard.press('Tab'); + await expect.poll(() => dialog.evaluate((element) => element.contains(document.activeElement))).toBe(true); + } + await dialog.getByRole('button', { name: 'Cancel', exact: true }).click(); + await expect(dialog).toHaveCount(0); + await expect(leave).toBeFocused(); + await leave.click(); + await page.keyboard.press('Escape'); + await expect(dialog).toHaveCount(0); + await expect(leave).toBeFocused(); + await leave.click(); + await dialog.getByRole('button', { name: 'Leave session', exact: true }).click(); + await expect(dialog).toHaveCount(0); + await expect(page.getByRole('button', { name: 'Generate', exact: true })).toBeVisible(); + }); + test('accepts a custom prompt and enters the streaming state', async ({ page, request }) => { const health = await request.get('/healthz'); const body = health.ok() ? await health.json() : {}; diff --git a/apps/dreamverse/web/src/app/page.tsx b/apps/dreamverse/web/src/app/page.tsx index 8a472d503..50a15167d 100644 --- a/apps/dreamverse/web/src/app/page.tsx +++ b/apps/dreamverse/web/src/app/page.tsx @@ -2739,7 +2739,7 @@ export default function Page() {
setSidebarOpen((prev) => !prev)} />
{}, }: LeaveSessionModalProps) { const [suppress, setSuppress] = useState(false); + const previousFocus = useRef(null); useEffect(() => { if (open) setSuppress(false); @@ -36,43 +38,58 @@ export default function LeaveSessionModal({ } return ( -
-
-
-
-

- Leave session? -

-

- This will end your current session. You will need to queue again for a GPU to start a new project. -

+ { if (!nextOpen) onClose(); }}> + + +
+ event.preventDefault()} + onOpenAutoFocus={() => { + previousFocus.current = document.activeElement instanceof HTMLElement ? document.activeElement : null; + }} + onCloseAutoFocus={(event) => { + event.preventDefault(); + if (previousFocus.current?.isConnected) previousFocus.current.focus(); + }} + > +
+
+
+ + Leave session? + + + This will end your current session. You will need to queue again for a GPU to start a new project. + +
+
+ setSuppress(v === true)} + /> + +
+
+
+ + +
+
+
-
- setSuppress(v === true)} - /> - -
-
-
- - -
-
-
+ + + ); } diff --git a/docs-site/tests/cookbook.spec.ts b/docs-site/tests/cookbook.spec.ts index d59c04972..9f44fe642 100644 --- a/docs-site/tests/cookbook.spec.ts +++ b/docs-site/tests/cookbook.spec.ts @@ -55,6 +55,25 @@ test("edits generate YAML and client instructions, invalid edits clear output, r await expect(yaml).toHaveText(baseline); }); +test("copy buttons use their own generated output", async ({ page, context }) => { + await context.grantPermissions(["clipboard-read", "clipboard-write"]); + await page.goto(route); + await output(page, "config.yaml"); + const yaml = await page.getByLabel("Generated YAML", { exact: true }).innerText(); + await page.getByRole("button", { name: "Copy YAML", exact: true }).click(); + await expect.poll(() => page.evaluate(() => navigator.clipboard.readText())).toBe(yaml); + await output(page, "Commands"); + await page.locator(".recipe-request-code > summary").click(); + const copied = [yaml]; + for (const label of ["Server command", "Client command"]) { + const command = await page.getByLabel(label, { exact: true }).innerText(); + await page.getByRole("button", { name: `Copy ${label.toLowerCase()}`, exact: true }).click(); + await expect.poll(() => page.evaluate(() => navigator.clipboard.readText())).toBe(command); + copied.push(command); + } + expect(new Set(copied).size).toBe(3); +}); + test("deployment changes reset edits and support MLX and streaming", async ({ page }) => { await page.goto(route); await settings(page, "Serving"); diff --git a/docs/contributing/ci_architecture.md b/docs/contributing/ci_architecture.md index 1f9de126d..d5512b51a 100644 --- a/docs/contributing/ci_architecture.md +++ b/docs/contributing/ci_architecture.md @@ -60,7 +60,7 @@ Three Buildkite entry pipelines share the validated graph: |---|---|---| | `pr-fastcheck` | Automatic pull-request webhook | Six Fastcheck lanes | | `ci` | `/merge`, `ready`, schedules, and `/test` API builds | Change-aware merge gates, scheduled SSIM, explicit Full Suite, Fastcheck reruns, or one direct lane | -| `fastvideo-performance-lane` | Weekly scheduler | Direct performance lane | +| `fastvideo-performance-lane` | GitHub weekly scheduler | Direct performance lane | Each entry pipeline starts with the same trusted `pipeline-upload` job on the `ci-runner` queue. The `ci` pipeline's incoming GitHub webhook is disabled; @@ -205,6 +205,95 @@ inside that range. See [Performance Benchmarks](performance_benchmarks.md) for the performance lane's thresholds, rolling baseline, artifacts, and reseeding process. +`ci-scheduled-performance.yml` triggers the performance pipeline every Sunday +at 23:00 America/Los_Angeles, passing the immutable `github.sha`. Its two UTC +schedules select the matching Pacific offset so it runs once across daylight +saving time. After this workflow lands, disable the legacy Buildkite schedule +under **Settings → Schedules → FastVideo scheduled performance benchmarks → +Edit → Enabled**. That schedule submits the literal `HEAD`, which the trusted +runner rejects before upload. The GitHub `BUILDKITE_API_TOKEN` must have +`write_builds` access to the performance pipeline. + +The performance lane recognizes main-branch API builds from the dedicated +performance pipeline with `TEST_SCOPE=direct` as scheduled-main runs. They +refresh eligible baselines after successful tests and still produce comparison +reports after test failures. PR performance runs use strict baseline reads and +keep reports as Buildkite artifacts without requiring upload credentials. +Scheduled-main publishing requires an operator-provisioned token with write +access to `FastVideo/performance-tracking`. + +The gated FLUX.1-dev SSIM and golden tests and FLUX.2-klein-9B SSIM test are explicitly paused until their +model access is provisioned. Their pytest skips are not FLUX quality validation; +the other SSIM and golden tests and the suites' fail-fast behavior remain active. +The SD3.5 golden and SSIM tests are also explicitly paused at the maintainer's +request while their gated checkpoint access is unavailable. + +MatrixGame 3 explicitly selects FA2 for its existing GB200 reference. That +reference was published in dataset commit `e1a6e64` and was not part of the +later FA4 refresh. Unchanged main with FA2 reproduces its video SHA-256 +`1d1d26a59481e37a9e838a628d875e196211c2236e8f9a370e0a9f73e97300c7`; +FA4 scores 0.96636 against its unchanged 0.98 requirement. The test compares +the current generation's returned path so older files cannot mask that check. + +The missing GB200 HunyuanVideo 1.5 I2V and Kandinsky 6 references were generated +from unchanged upstream commit `d5253fd71055927f12aa6c4aa679b93a00238851` +with Torch 2.12.0, CUDA 13 and FA4. Their checkpoint revisions are pinned in +the tests and recorded alongside the videos. Independent upstream repeats and +the candidate both scored 1.0 for Hunyuan and 0.98395 for Kandinsky against +the first upstream outputs, preserving the existing 0.95 and 0.98 thresholds. +Kandinsky remains required; its authorized complete snapshot is prepared in +the shared CI cache without exposing an authentication token to PR code. + +LingBot Fast's missing GB200 reference uses the same upstream commit and +runtime, with checkpoint revision `98583eb0fd66f481f73383428d65d2867c878e51`. +Two unchanged-upstream runs with CPU offloading disabled produced identical +video hashes. Its fixed FSDP CPU-offloaded run scored SSIM 1.0 against that +reference, preserving the 0.70 requirement. The generation stage leaves FSDP +parameter placement to FSDP instead of moving the whole model to the GPU. + +### Provisioning Required Hugging Face Models + +Gated access belongs to the individual HF account, not its organization or +token. An operator must request and obtain model access for the CI identity +before configuring authentication. A valid token without model access still +returns 403. Keep required models such as Kandinsky 6 strict until their +weights and quality references are provisioned. + +The permanent operator rollout is: + +1. Approve every required gated repository for one maintained CI identity. +2. Configure [HF CI/CD access through OIDC](https://huggingface.co/docs/hub/trusted-publishers) + for the trusted Buildkite preparation step, restricting issuer and job + claims to the intended organization and pipelines. A fine-grained read-only + token in the operator's secret store is an alternative. Keep performance + publishing credentials separate and restricted to the tracking dataset and + trusted main runs. +3. Before leasing GPUs, check access to all required pinned manifests and + weight files together and check that their device/backend references exist. + Report every missing dependency in one preflight result; do not turn access + failures into passing quality gates. +4. Fetch immutable model revisions into the runner-owned cache through trusted + code. Remove authentication before starting PR code, and keep download locks + and cache writes under the runner's identity. PR jobs consume the prepared + snapshots rather than depending on an interactive login or another user's + writable cache. + When multiple operators prepare the shared cache, use its existing group + with `umask 0002`. Check group write access on each prepared model's cache, + snapshot, blob and `.locks` directories, not only the cache root. Verify + manifest resolution as the CI worker identity before running the lane. + A readable weight file alone is insufficient: Hub downloads also create + metadata and lock files. For example, Stable Audio's pinned revision is + `26609645c363fc81829282fcfe6356340346c1d4`; provision its complete snapshot, + including `model_index.json`, component configs and tokenizer files. +5. For a missing SSIM reference, generate and inspect output from unchanged + upstream source on the CI GPU/backend/runtime, retain its provenance, and + publish only the reviewed new reference. Verify the candidate independently + before enabling the required gate; never use candidate output to clear a + regression failure. + +See [HF gated-model access](https://huggingface.co/docs/hub/models-gated) and +the [testing guide](testing.md) for access and reference maintenance details. + ## `/merge` Request Flow The Buildkite agent and Slurm have deliberately separate responsibilities: @@ -476,6 +565,7 @@ through checks, commit statuses, or labels instead of PR/issue comments. | `.github/workflows/ci-slash-commands.yml` | `/merge` and `/test` handling | | `.github/workflows/ci-trigger-full-suite.yml` | Change-aware merge-gate trigger for `ready` PRs and new pushes | | `.github/workflows/ci-scheduled-ssim.yml` | Weekly complete SSIM trigger on `main` | +| `.github/workflows/ci-scheduled-performance.yml` | Weekly performance trigger with an immutable source SHA | | `.github/scripts/plan_merge_ci.py` | Trusted changed-path to integration-lane planner | | `.github/workflows/ci-aggregate-status.yml` | Aggregate Fastcheck and explicit Full Suite direct-rerun statuses | | `.buildkite/pipeline.yml` | Static 20-lane Slurm Buildkite graph | diff --git a/docs/contributing/env_vars.md b/docs/contributing/env_vars.md index 3bdf74996..7f7562272 100644 --- a/docs/contributing/env_vars.md +++ b/docs/contributing/env_vars.md @@ -218,6 +218,7 @@ longer exists also fails the test, so the fixing pull request deletes its entry. | `FASTVIDEO_LTX2_VAE_CHANNELS_LAST_3D` | bool | `1` | performance | LTX-2 VAE uses the channels_last_3d memory format. | | `FASTVIDEO_LTX2_DISABLE_AUDIO_AUTOCAST` | bool | `1` | performance | LTX-2 audio decoding runs without CUDA autocast. Deprecated names: `LTX2_DISABLE_AUDIO_AUTOCAST`. | | `FASTVIDEO_FLUX2_DISABLE_BF16_REDUCED_PRECISION_REDUCTION` | bool | `0` | performance | Flux denoising disables reduced-precision reductions in bf16 matmuls, which tightens accumulation for the 4-step Klein model. | +| `FASTVIDEO_ENABLE_FUSED_NORM` | bool | `0` | performance | Opt into Wan's inference-only Triton residual + LayerNorm + modulation fusion. Its reduction order changes numerical outputs; the default keeps eager reference compatibility. | | `FASTVIDEO_DISABLE_FUSED_NORM` | bool | `0` | performance | Turn off the Triton-fused residual + LayerNorm + modulate inference path that Wan blocks opt into, and use the eager path. | | `FASTVIDEO_FFMPEG_BIN` | str | `ffmpeg` | output | ffmpeg executable used to save video with audio. | | `FASTVIDEO_VIDEO_CODEC` | str | `libx264` | output | ffmpeg video codec for saved videos. | diff --git a/docs/contributing/performance_benchmarks.md b/docs/contributing/performance_benchmarks.md index b344c0ce2..5fa4b2a9c 100644 --- a/docs/contributing/performance_benchmarks.md +++ b/docs/contributing/performance_benchmarks.md @@ -44,8 +44,9 @@ python fastvideo/tests/performance/dashboard.py The pytest run never uploads anything. `compare_baseline.py` uploads only when `PERF_UPLOAD_POLICY` is set. Local uploads are explicit opt-in and require HF -credentials. PR/direct performance runs upload passing records for dashboard -visibility, while scheduled-main runs upload both pass and fail records. The +credentials. PR performance runs read baselines strictly and retain comparison +reports as Buildkite artifacts without uploading records. Scheduled-main runs +upload both pass and fail records using an operator-provisioned CI credential. The report directory default is container-oriented; set `PERF_REPORTS_DIR` to a writable local path when generating dashboards or when you want local Markdown/normalized-result artifacts from the comparator. @@ -244,8 +245,8 @@ metrics are skipped. This is the **drift detector** — it catches sub-threshold regressions that slowly add up. Only scheduled-main successful records are baseline eligible. -Local and pull-request runs can upload dashboard-visible records, but they do -not update future gating baselines. +Local uploads are explicit opt-in and do not update future gating baselines. +PR runs retain their normalized records and reports as Buildkite artifacts. Comparator summaries and normalized artifacts include an explicit `comparison_status`: diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index 519bf87c4..2159e50ee 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -53,6 +53,11 @@ or environment mismatch is not a pass. Do not replace a reference with the candidate output just to clear a failure. For a relocation, the unchanged parent is the baseline; two imports of the same class are not numerical proof. +FA4 fingerprints record the effective boolean switch. Legacy block references +with `fastvideo_fa4: ` mean disabled and match an explicit `0`; enabling +FA4 still fails the environment check. Other runtime pins and zero-tolerance +tensor comparisons remain strict. + The VAE and transformer CI lanes run their Wan component goldens first. Selected integration lanes wait for the golden lane in merge/full builds. See [CI/CD Architecture](ci_architecture.md) for direct-rerun and skip semantics. @@ -95,6 +100,14 @@ pytest fastvideo/tests/vaes/test_wan_vae.py -q GPU-heavy suites need the right hardware, credentials, local caches, and sometimes custom kernels. Document those assumptions in new tests. +The legacy VSA training regression selects a device-specific summary. Its +GB200 reference was generated from unchanged upstream commit +`d5253fd71055927f12aa6c4aa679b93a00238851` using two GB200 GPUs, Torch +2.12.0+cu130, CUDA 13.0 and FA2. Two independent upstream runs produced +gradient norms 1.909 and 1.935 and losses 0.30861 and 0.30886. The reference +JSON retains provenance; the H100 reference and all comparison tolerances +remain unchanged. Candidate outputs must not be used to refresh this reference. + ## SSIM Tests SSIM tests generate videos using specific models and parameters, then compare diff --git a/fastvideo/envs.py b/fastvideo/envs.py index 4f84433a5..f14d63b04 100644 --- a/fastvideo/envs.py +++ b/fastvideo/envs.py @@ -458,6 +458,11 @@ FASTVIDEO_FLUX2_DISABLE_BF16_REDUCED_PRECISION_REDUCTION = EnvBool( category="performance", doc="Flux denoising disables reduced-precision reductions in bf16 matmuls, which tightens accumulation for " "the 4-step Klein model.") +FASTVIDEO_ENABLE_FUSED_NORM = EnvBool( + False, + category="performance", + doc="Opt into Wan's inference-only Triton residual + LayerNorm + modulation fusion. " + "Its reduction order changes numerical outputs; the default keeps eager reference compatibility.") # Global kill switch for the opt-in, inference-only Triton fusion in # fastvideo/layers/triton_fused_norm.py, for debugging numerics or Triton issues. FASTVIDEO_DISABLE_FUSED_NORM = EnvBool(False, diff --git a/fastvideo/eval/metrics/vbench/dynamic_degree/metric.py b/fastvideo/eval/metrics/vbench/dynamic_degree/metric.py index 788170a51..514da1caf 100644 --- a/fastvideo/eval/metrics/vbench/dynamic_degree/metric.py +++ b/fastvideo/eval/metrics/vbench/dynamic_degree/metric.py @@ -11,7 +11,6 @@ from typing import Any import numpy as np import torch -from easydict import EasyDict from fastvideo.eval.metrics.base import BaseMetric from fastvideo.eval.registry import register @@ -41,6 +40,7 @@ class DynamicDegreeMetric(BaseMetric): def setup(self) -> None: if self._model is not None: return + from easydict import EasyDict from vbench.third_party.RAFT.core.raft import RAFT args = EasyDict(small=False, mixed_precision=False, alternate_corr=False, dropout=0.0) diff --git a/fastvideo/layers/triton_fused_norm.py b/fastvideo/layers/triton_fused_norm.py index 5d88ae775..c35d0ad50 100644 --- a/fastvideo/layers/triton_fused_norm.py +++ b/fastvideo/layers/triton_fused_norm.py @@ -25,7 +25,8 @@ from eager are the reduction order inside mean/variance (last-ulp fp32) and that product-rounding case. Inference-only: callers must gate on ``torch.is_grad_enabled()``; there is no -backward. Disable globally with FASTVIDEO_DISABLE_FUSED_NORM=1. +backward. Enable with FASTVIDEO_ENABLE_FUSED_NORM=1; the default preserves the +eager numerical path. FASTVIDEO_DISABLE_FUSED_NORM=1 overrides the opt-in. """ from __future__ import annotations @@ -158,7 +159,7 @@ def fused_path_supported( norm: torch.nn.Module, ) -> bool: """Cheap eligibility check; any False falls back to the eager path.""" - if not _HAS_TRITON or envs.FASTVIDEO_DISABLE_FUSED_NORM.get(): + if not _HAS_TRITON or not envs.FASTVIDEO_ENABLE_FUSED_NORM.get() or envs.FASTVIDEO_DISABLE_FUSED_NORM.get(): return False if torch.is_grad_enabled(): return False diff --git a/fastvideo/pipelines/basic/lingbotworld2/causal_fast_pipeline.py b/fastvideo/pipelines/basic/lingbotworld2/causal_fast_pipeline.py index 80f6b7937..398562da4 100644 --- a/fastvideo/pipelines/basic/lingbotworld2/causal_fast_pipeline.py +++ b/fastvideo/pipelines/basic/lingbotworld2/causal_fast_pipeline.py @@ -281,7 +281,8 @@ class LingBotWorld2CausalFastGenerationStage(PipelineStage): self_kv_cache = self._initialize_self_kv_cache(1, kv_size, transformer_dtype, device) cross_kv_cache = self._initialize_crossattn_cache(1, max_sequence_length, transformer_dtype, device) - self.transformer = self.transformer.to(device) + if not fastvideo_args.use_fsdp_inference: + self.transformer = self.transformer.to(device) self._cross_attn_initialized = False pred_latent_chunks = [] latents_chunk = noise.split(chunk_size, dim=1) diff --git a/fastvideo/tests/contract/test_ci_test_collection.py b/fastvideo/tests/contract/test_ci_test_collection.py index 9635c9672..50ba76041 100644 --- a/fastvideo/tests/contract/test_ci_test_collection.py +++ b/fastvideo/tests/contract/test_ci_test_collection.py @@ -249,6 +249,103 @@ def test_full_ssim_has_a_weekly_slurm_schedule(): assert 'build.env("TEST_SCOPE") == "scheduled"' in str(ssim_step["if"]) +def test_performance_trigger_rejects_head_and_sends_a_pinned_direct_build(): + import json + import os + import subprocess + import tempfile + + workflow = yaml.safe_load((REPO_ROOT / '.github/workflows/ci-scheduled-performance.yml').read_text()) + job = workflow['jobs']['trigger'] + assert "github.repository == 'hao-ai-lab/FastVideo'" in job['if'] + assert "github.ref == 'refs/heads/main'" in job['if'] + step = job['steps'][-1] + assert step['env']['SOURCE_SHA'] == '${{ github.sha }}' + with tempfile.TemporaryDirectory() as root: + curl = Path(root) / 'curl' + curl.write_text('#!/bin/bash\nwhile [[ $# -gt 0 ]]; do\n' + ' if [[ "$1" == --data-raw ]]; then printf "%s" "$2"; exit 0; fi\n' + ' shift\ndone\nexit 1\n') + curl.chmod(0o755) + env = { + 'PATH': root + os.pathsep + os.environ['PATH'], + 'BUILDKITE_API_TOKEN': 'test-only', + 'SOURCE_BRANCH': 'main', + 'BK_ORG': 'fastvideo', + 'BK_PIPELINE': step['env']['BK_PIPELINE'], + } + for commit in ('HEAD', 'abc123', 'g' * 40, 'a' * 40): + env['SOURCE_SHA'] = commit + result = subprocess.run(['bash', '-c', step['run']], env=env, text=True, capture_output=True) + if commit != 'a' * 40: + assert result.returncode != 0 + assert not result.stdout + continue + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) + assert payload['commit'] == commit + assert payload['branch'] == 'main' + assert payload['env']['TEST_TYPE'] == 'performance' + assert payload['env']['TEST_SCOPE'] == 'direct' + assert payload['env']['FULL_SUITE'] == 'false' + assert payload['env']['PR_NUMBER'] == 'false' + + +def test_weekly_performance_api_build_keeps_scheduled_baseline_policy(): + import os + import subprocess + import tempfile + + lane = (REPO_ROOT / '.buildkite/scripts/lanes/performance.sh').read_text() + lane = lane.replace('export PERF_REPORTS_DIR=/workspace/artifacts/performance', + 'export PERF_REPORTS_DIR="$TEST_REPORTS_DIR"') + cases = ( + ('main', 'api', 'fastvideo-performance-lane', 'direct', 'false', 'scheduled_main', 'always'), + ('main', 'schedule', 'fastvideo-performance-lane', 'direct', 'false', 'scheduled_main', 'always'), + ('main', 'api', 'ci', 'full', 'false', 'scheduled_main', 'always'), + ('main', 'api', 'ci', 'direct', 'false', 'unknown', 'pass'), + ('feature', 'api', 'fastvideo-performance-lane', 'direct', 'false', 'unknown', 'pass'), + ('main', 'api', 'fastvideo-performance-lane', 'direct', '1958', 'pr', 'never'), + ) + with tempfile.TemporaryDirectory() as root: + directory = Path(root) + mocks = { + 'nvidia-smi': '#!/bin/sh\nexit 0\n', + 'pytest': '#!/bin/sh\nexit "$TEST_PYTEST_RC"\n', + 'python': '#!/bin/sh\nprintf "%s|%s|%s|%s\\n" "$1" "$PERF_RUN_SOURCE" ' + '"$PERF_UPLOAD_POLICY" "${PERF_PYTEST_RC:-}" >> "$TEST_CALLS_FILE"\n', + } + for name, contents in mocks.items(): + executable = directory / name + executable.write_text(contents) + executable.chmod(0o755) + calls = directory / 'calls.txt' + for branch, source, pipeline, scope, pr, run_source, policy in cases: + for pytest_rc in (0, 1): + calls.unlink(missing_ok=True) + env = { + 'PATH': root + os.pathsep + os.environ['PATH'], + 'BUILDKITE_BRANCH': branch, + 'BUILDKITE_SOURCE': source, + 'BUILDKITE_PIPELINE_SLUG': pipeline, + 'BUILDKITE_PULL_REQUEST': pr, + 'TEST_SCOPE': scope, + 'TEST_PYTEST_RC': str(pytest_rc), + 'TEST_CALLS_FILE': str(calls), + 'TEST_REPORTS_DIR': str(directory / 'reports'), + } + result = subprocess.run(['bash', '-c', lane], cwd=root, env=env, text=True, capture_output=True) + assert result.returncode == pytest_rc, result.stderr + records = [line.split('|') for line in calls.read_text().splitlines()] + assert all(record[1:3] == [run_source, policy] for record in records) + comparisons = [record for record in records if record[0].endswith('/compare_baseline.py')] + if pytest_rc == 0 or policy == 'always': + assert len(comparisons) == 1 + assert comparisons[0][3] == str(pytest_rc) + else: + assert not comparisons + + def test_active_pipeline_has_no_modal_or_untrusted_compute_path(): pipeline_text = (REPO_ROOT / ".buildkite/pipeline.yml").read_text() steps = _pipeline_steps() @@ -360,3 +457,17 @@ def test_allowlist_entries_are_still_real_directories(): # A stale allowlist hides regressions; entries must track reality. missing = [name for name in ALLOWLIST if name != "modal" and not (TESTS_ROOT / name).is_dir()] assert not missing, (f"Allowlisted directories no longer exist — remove them: {missing}") + + +def test_arm64_chromium_filters_generation_mode_playback_but_keeps_uploads(): + lane = (REPO_ROOT / '.buildkite/scripts/lanes/dreamverse.sh').read_text() + pattern = re.search(r"--grep-invert='([^']+)'", lane).group(1) + for mode in ('t2va', 'fl2va', 'ref2va'): + title = f'{mode} sends validated assets and plays a clearly labeled sample' + assert re.search(pattern, title), f'ARM64 Chromium cannot decode this playback test: {title}' + assert not re.search(pattern, "proxies a media upload larger than Next's default 10 MiB body limit") + assert not re.search(pattern, 'accepts a custom prompt and enters the streaming state') + arm_branch = lane.rsplit('if [[ $machine_arch =~ ^(aarch64|arm64)$ ]]; then', 1)[1].split('\nelse')[0] + firefox_command = re.search(r'npm run e2e(?:\\\n|[^\n])*', arm_branch).group(0) + assert '--project=firefox' in firefox_command + assert '--grep' not in firefox_command diff --git a/fastvideo/tests/contract/test_golden_environment.py b/fastvideo/tests/contract/test_golden_environment.py new file mode 100644 index 000000000..d5f8a4a09 --- /dev/null +++ b/fastvideo/tests/contract/test_golden_environment.py @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +import fastvideo.envs as envs +from fastvideo.tests.golden_gate._harness import GateSpec, assert_env_fingerprint_matches, env_fingerprint + + +@pytest.mark.parametrize("value", [None, False, True]) +def test_block_fingerprint_records_effective_fa4(monkeypatch, value): + import torch + + monkeypatch.setattr(torch.cuda, "get_device_name", lambda index: "NVIDIA GB200") + spec = GateSpec("test", "test/repo", lambda layer: None, lambda device, seed: {}) + with envs.FASTVIDEO_FA4.override(value): + assert env_fingerprint(spec)["fastvideo_fa4"] == str(int(envs.FASTVIDEO_FA4.get())) + + +def test_legacy_unset_fa4_matches_disabled_without_changing_reference(): + reference = {"fastvideo_fa4": "", "torch": "pinned"} + assert_env_fingerprint_matches(reference, {"fastvideo_fa4": "0", "torch": "pinned"}) + assert reference == {"fastvideo_fa4": "", "torch": "pinned"} + + +@pytest.mark.parametrize("reference_fa4", ["", "0", "1"]) +def test_fa4_compute_path_changes_still_fail(reference_fa4): + current_fa4 = "0" if reference_fa4 == "1" else "1" + with pytest.raises(AssertionError, match="fastvideo_fa4"): + assert_env_fingerprint_matches({"fastvideo_fa4": reference_fa4}, {"fastvideo_fa4": current_fa4}) + + +@pytest.mark.parametrize("field", ["torch", "cuda", "cudnn", "flash_attn", "device_name", "attention_backend", + "tf32_matmul", "tf32_cudnn"]) +def test_other_runtime_changes_still_fail(field): + with pytest.raises(AssertionError, match=field): + assert_env_fingerprint_matches({"fastvideo_fa4": "", field: "old"}, + {"fastvideo_fa4": "0", field: "new"}) diff --git a/fastvideo/tests/eval/test_registry_optional_deps.py b/fastvideo/tests/eval/test_registry_optional_deps.py new file mode 100644 index 000000000..76be02747 --- /dev/null +++ b/fastvideo/tests/eval/test_registry_optional_deps.py @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: Apache-2.0 + +import subprocess +import sys + + +def test_dynamic_degree_stays_registered_without_easydict(): + script = """ +import sys +sys.modules['easydict'] = None +from fastvideo.eval.registry import get_metric, list_metrics +assert 'vbench.dynamic_degree' in list_metrics() +try: + get_metric('vbench.dynamic_degree') +except ImportError as error: + assert 'easydict' in str(error) +else: + raise AssertionError('Missing optional dependency was not reported') +""" + result = subprocess.run([sys.executable, "-c", script], text=True, capture_output=True) + assert result.returncode == 0, result.stderr diff --git a/fastvideo/tests/golden_gate/_harness.py b/fastvideo/tests/golden_gate/_harness.py index f2104c63d..e1bd913b6 100644 --- a/fastvideo/tests/golden_gate/_harness.py +++ b/fastvideo/tests/golden_gate/_harness.py @@ -74,7 +74,7 @@ def env_fingerprint(spec: GateSpec | str | None) -> dict[str, str]: "flash_attn": fa_version, "device_name": torch.cuda.get_device_name(0), "attention_backend": backend or "NONE", - "fastvideo_fa4": str(int(envs.FASTVIDEO_FA4.get())) if envs.FASTVIDEO_FA4.is_set() else "", + "fastvideo_fa4": str(int(envs.FASTVIDEO_FA4.get())), "tf32_matmul": str(torch.backends.cuda.matmul.allow_tf32), "tf32_cudnn": str(torch.backends.cudnn.allow_tf32), } @@ -83,14 +83,19 @@ def env_fingerprint(spec: GateSpec | str | None) -> dict[str, str]: # part of the codec compute path. Keep torch/CUDA/cuDNN/device/TF32 pins. result.pop("flash_attn") result.pop("fastvideo_fa4") - elif not isinstance(spec, GateSpec): - # Tensor gates record the effective switch, so unset and "0" both record - # "0". Block gates record "" when FASTVIDEO_FA4 is unset, and "1" - # or "0" when it is set. - result["fastvideo_fa4"] = str(int(envs.FASTVIDEO_FA4.get())) return result +def assert_env_fingerprint_matches(expected: dict[str, str], actual: dict[str, str]) -> None: + """Legacy block goldens recorded disabled FA4 as ''; compare its effective value.""" + expected = dict(expected) + if expected.get("fastvideo_fa4") == "": + expected["fastvideo_fa4"] = "0" + mismatches = {key: (value, actual.get(key)) for key, value in expected.items() if actual.get(key) != value} + assert not mismatches, (f"environment changed since golden was minted — re-mint deliberately rather " + f"than chasing bit drift. golden -> current: {mismatches}") + + def _component_dir(spec: GateSpec) -> Path: """Local checkout wins; otherwise fetch the index + only the shards holding this layer (or the single weight file for index-less checkpoints).""" @@ -228,12 +233,7 @@ def _run_block_against_golden(spec: GateSpec) -> None: ) golden = torch.load(golden_path, weights_only=True) - mismatches = { - key: (value, meta["env"].get(key)) - for key, value in golden["metadata"]["env"].items() if meta["env"].get(key) != value - } - assert not mismatches, (f"environment changed since golden was minted — re-mint deliberately rather " - f"than chasing bit drift. golden -> current: {mismatches}") + assert_env_fingerprint_matches(golden["metadata"]["env"], meta["env"]) drift = (out - golden["output"]).abs() print(f"golden-gate[{spec.name}] drift: max_abs={drift.max().item():.10f} " diff --git a/fastvideo/tests/golden_gate/test_flux.py b/fastvideo/tests/golden_gate/test_flux.py index 19020f177..e1989a38c 100644 --- a/fastvideo/tests/golden_gate/test_flux.py +++ b/fastvideo/tests/golden_gate/test_flux.py @@ -14,6 +14,7 @@ is a GATED HF repo; FASTVIDEO_TEST_FLUX_T2I_MODEL_DIR can point at a local check from __future__ import annotations +import pytest import torch import fastvideo.envs as envs @@ -82,5 +83,6 @@ SPEC = GateSpec( ) +@pytest.mark.skip(reason="FLUX.1-dev golden gate is paused while gated-model access is not provisioned for CI.") def test_flux_golden_gate(distributed_runtime) -> None: run_gate(SPEC) diff --git a/fastvideo/tests/golden_gate/test_minimax_h3_t2v.py b/fastvideo/tests/golden_gate/test_minimax_h3_t2v.py index f7c35a57e..fbe5567e2 100644 --- a/fastvideo/tests/golden_gate/test_minimax_h3_t2v.py +++ b/fastvideo/tests/golden_gate/test_minimax_h3_t2v.py @@ -38,6 +38,7 @@ import torch from torch.testing import assert_close import fastvideo.envs as envs +from fastvideo.tests.golden_gate._harness import assert_env_fingerprint_matches MODEL_REPO_ID = "MiniMaxAI/MiniMax-H3" SEED = 20260805 @@ -98,7 +99,7 @@ def _env_fingerprint() -> dict[str, str]: "flash_attn": fa_version, "device_name": torch.cuda.get_device_name(0), "attention_backend": envs.FASTVIDEO_ATTENTION_BACKEND.get() or "", - "fastvideo_fa4": str(int(envs.FASTVIDEO_FA4.get())) if envs.FASTVIDEO_FA4.is_set() else "", + "fastvideo_fa4": str(int(envs.FASTVIDEO_FA4.get())), "tf32_matmul": str(torch.backends.cuda.matmul.allow_tf32), "tf32_cudnn": str(torch.backends.cudnn.allow_tf32), } @@ -286,12 +287,7 @@ def test_minimax_h3_t2v_golden_gate(env_overrides) -> None: ) golden = torch.load(golden_path, weights_only=True) - mismatches = { - key: (value, meta["env"].get(key)) - for key, value in golden["metadata"]["env"].items() if meta["env"].get(key) != value - } - assert not mismatches, (f"environment changed since golden was minted — re-mint deliberately rather " - f"than chasing bit drift. golden -> current: {mismatches}") + assert_env_fingerprint_matches(golden["metadata"]["env"], meta["env"]) drift = (out - golden["output"]).abs() print(f"golden-gate drift: max_abs={drift.max().item():.10f} " diff --git a/fastvideo/tests/golden_gate/test_sd35.py b/fastvideo/tests/golden_gate/test_sd35.py index aff6e377c..7d1da185c 100644 --- a/fastvideo/tests/golden_gate/test_sd35.py +++ b/fastvideo/tests/golden_gate/test_sd35.py @@ -16,6 +16,7 @@ tensors only — no RoPE, no mask; sincos pos embed is added before the blocks from __future__ import annotations +import pytest import torch from fastvideo.tests.golden_gate._harness import GateSpec, distributed_runtime, run_gate @@ -73,5 +74,6 @@ SPEC = GateSpec( ) +@pytest.mark.skip(reason="SD3.5 golden gate is paused while gated-model access is not provisioned for CI.") def test_sd35_golden_gate(distributed_runtime) -> None: run_gate(SPEC) diff --git a/fastvideo/tests/inference/vmoba/test_vmoba_inference.py b/fastvideo/tests/inference/vmoba/test_vmoba_inference.py index c543a55ff..1a899a34f 100644 --- a/fastvideo/tests/inference/vmoba/test_vmoba_inference.py +++ b/fastvideo/tests/inference/vmoba/test_vmoba_inference.py @@ -94,4 +94,6 @@ def test_inference_vmoba(env_overrides): if __name__ == "__main__": - test_inference_vmoba() + import pytest + + raise SystemExit(pytest.main([__file__, "-vs"])) diff --git a/fastvideo/tests/layers/test_triton_fused_norm.py b/fastvideo/tests/layers/test_triton_fused_norm.py index 3f3e3b2ea..2408eccec 100644 --- a/fastvideo/tests/layers/test_triton_fused_norm.py +++ b/fastvideo/tests/layers/test_triton_fused_norm.py @@ -12,12 +12,18 @@ import pytest import torch from fastvideo.layers.layernorm import ScaleResidualLayerNormScaleShift +import fastvideo.envs as envs cuda_only = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") B, S, H = 1, 9360, 1536 # the Wan2.1-T2V-1.3B sp2 shapes +@pytest.fixture(autouse=True) +def opt_into_fused_norm(env_overrides): + env_overrides.enter_context(envs.FASTVIDEO_ENABLE_FUSED_NORM.override(True)) + + def _make_modules(elementwise_affine: bool, seed: int = 0): """One eager module and one fused module with identical weights.""" torch.manual_seed(seed) @@ -175,6 +181,20 @@ def test_env_kill_switch(env_overrides): assert out.dtype == torch.float32 # eager promotion, not the fused bf16 +@cuda_only +def test_fused_norm_requires_explicit_opt_in(env_overrides): + env_overrides.enter_context(envs.FASTVIDEO_ENABLE_FUSED_NORM.override(False)) + eager, fused = _make_modules(elementwise_affine=True) + residual, x = _inputs(torch.bfloat16, seed=6) + gate = torch.randn(1, 1, H, device="cuda", dtype=torch.float32) + with torch.no_grad(): + expected, expected_residual = eager(residual, x, gate, None, None) + actual, actual_residual = fused(residual, x, gate, None, None) + assert actual.dtype == torch.float32 + assert torch.equal(actual, expected) + assert torch.equal(actual_residual, expected_residual) + + @cuda_only def test_mixed_none_still_raises(): _, fused = _make_modules(elementwise_affine=True) diff --git a/fastvideo/tests/modal/test_pr_test.py b/fastvideo/tests/modal/test_pr_test.py index bba860d92..708f892e1 100644 --- a/fastvideo/tests/modal/test_pr_test.py +++ b/fastvideo/tests/modal/test_pr_test.py @@ -392,7 +392,7 @@ def test_wave1_lane_functions_use_shared_scripts(monkeypatch): lanes_dir = Path(__file__).resolve().parents[3] / ".buildkite/scripts/lanes" for script, payload in { "kernel_tests.sh": "pytest fastvideo-kernel/tests/ -vs", - "inference_vmoba.sh": "python fastvideo/tests/inference/vmoba/test_vmoba_inference.py", + "inference_vmoba.sh": "exec pytest ./fastvideo/tests/inference/vmoba/test_vmoba_inference.py -vs", "golden_gate.sh": 'exec pytest "$golden_root" -xvs', "encoder.sh": "pytest ./fastvideo/tests/encoders -vs", "vae.sh": "pytest ./fastvideo/tests/vaes -vs", diff --git a/fastvideo/tests/performance/compare_baseline.py b/fastvideo/tests/performance/compare_baseline.py index eb6b46a40..e7f54d73e 100644 --- a/fastvideo/tests/performance/compare_baseline.py +++ b/fastvideo/tests/performance/compare_baseline.py @@ -591,9 +591,7 @@ def main() -> int: upload_policy = _normalized_upload_policy() performance_pytest_failed = _performance_pytest_failed() - # Strict on upload-enabled runs: silent sync failure would make comparison - # and upload state ambiguous. - sync_from_hf(TRACKING_ROOT, strict=persist_tracking) + sync_from_hf(TRACKING_ROOT, strict=persist_tracking or _detect_run_source() == "pr") current_results = _load_current_results() if not current_results: diff --git a/fastvideo/tests/ssim/test_flux2_similarity.py b/fastvideo/tests/ssim/test_flux2_similarity.py index 880fadf83..38269fd53 100644 --- a/fastvideo/tests/ssim/test_flux2_similarity.py +++ b/fastvideo/tests/ssim/test_flux2_similarity.py @@ -87,6 +87,8 @@ def test_flux2_similarity( attention_backend_name: str, model_id: str, ) -> None: + if model_id == "black-forest-labs/FLUX.2-klein-9B": + pytest.skip("FLUX.2-klein-9B SSIM is paused while gated-model access is not provisioned for CI.") _ = run_text_to_latent_similarity_test( logger=logger, script_dir=os.path.dirname(os.path.abspath(__file__)), diff --git a/fastvideo/tests/ssim/test_flux_t2i_similarity.py b/fastvideo/tests/ssim/test_flux_t2i_similarity.py index 4205f610e..8295b448a 100644 --- a/fastvideo/tests/ssim/test_flux_t2i_similarity.py +++ b/fastvideo/tests/ssim/test_flux_t2i_similarity.py @@ -84,6 +84,7 @@ FLUX_FULL_QUALITY_MODEL_TO_PARAMS = { } +@pytest.mark.skip(reason="FLUX.1-dev SSIM is paused while gated-model access is not provisioned for CI.") @pytest.mark.skipif( not torch.cuda.is_available(), reason="FLUX T2I SSIM test requires CUDA", diff --git a/fastvideo/tests/ssim/test_hunyuan15_i2v_similarity.py b/fastvideo/tests/ssim/test_hunyuan15_i2v_similarity.py index eae6318ee..e5d6c528a 100644 --- a/fastvideo/tests/ssim/test_hunyuan15_i2v_similarity.py +++ b/fastvideo/tests/ssim/test_hunyuan15_i2v_similarity.py @@ -88,5 +88,6 @@ def test_hunyuan15_i2v_inference_similarity( model_id=model_id, default_params_map=HUNYUAN15_I2V_MODEL_TO_PARAMS, full_quality_params_map=FULL_QUALITY_HUNYUAN15_I2V_MODEL_TO_PARAMS, + init_kwargs_override={"revision": "854c04a4c8a53d990b418c7478f0802c0fc8c726"}, min_acceptable_ssim=0.95, ) diff --git a/fastvideo/tests/ssim/test_kandinsky6_similarity.py b/fastvideo/tests/ssim/test_kandinsky6_similarity.py index 52f1f4919..879970355 100644 --- a/fastvideo/tests/ssim/test_kandinsky6_similarity.py +++ b/fastvideo/tests/ssim/test_kandinsky6_similarity.py @@ -91,6 +91,7 @@ def test_kandinsky6_ti2va_inference_similarity( # examples/inference/basic/basic_kandinsky6_ti2va.py): no FSDP inference, text encoder offloaded # to CPU between encodes. init_kwargs_override={ + "revision": "7be18aed6fbd64653c0c4872fddfe4a3e3e109f5", "use_fsdp_inference": False, "text_encoder_cpu_offload": True, "pin_cpu_memory": True, diff --git a/fastvideo/tests/ssim/test_lingbot_fast_similarity.py b/fastvideo/tests/ssim/test_lingbot_fast_similarity.py index 27bc26df4..41297a9ae 100644 --- a/fastvideo/tests/ssim/test_lingbot_fast_similarity.py +++ b/fastvideo/tests/ssim/test_lingbot_fast_similarity.py @@ -115,10 +115,10 @@ def test_lingbot_fast_i2v_similarity(prompt: str, ATTENTION_BACKEND: str, env_ov model_id, ATTENTION_BACKEND, ) - output_video_name = f"{prompt[:100].strip()}.mp4" os.makedirs(output_dir, exist_ok=True) init_kwargs = { + "revision": "98583eb0fd66f481f73383428d65d2867c878e51", "num_gpus": params["num_gpus"], "use_fsdp_inference": True, "dit_cpu_offload": True, @@ -142,12 +142,12 @@ def test_lingbot_fast_i2v_similarity(prompt: str, ATTENTION_BACKEND: str, env_ov try: generator = VideoGenerator.from_pretrained( model_path=params["model_path"], **init_kwargs) - generator.generate_video(prompt, **generation_kwargs) + result = generator.generate_video(prompt, **generation_kwargs) finally: if generator is not None: generator.shutdown() - generated_video_path = os.path.join(output_dir, output_video_name) + generated_video_path = result["video_path"] assert os.path.exists(generated_video_path), ( f"Output video was not generated at {generated_video_path}") diff --git a/fastvideo/tests/ssim/test_matrixgame3_similarity.py b/fastvideo/tests/ssim/test_matrixgame3_similarity.py index bf0c47771..9bf077b61 100644 --- a/fastvideo/tests/ssim/test_matrixgame3_similarity.py +++ b/fastvideo/tests/ssim/test_matrixgame3_similarity.py @@ -84,6 +84,7 @@ def test_matrixgame3_similarity(prompt, ATTENTION_BACKEND, model_id, env_overrid and compares the output to reference videos using SSIM. """ env_overrides.enter_context(envs.FASTVIDEO_ATTENTION_BACKEND.override(ATTENTION_BACKEND)) + env_overrides.enter_context(envs.FASTVIDEO_FA4.override(False)) script_dir = os.path.dirname(os.path.abspath(__file__)) @@ -126,7 +127,7 @@ def test_matrixgame3_similarity(prompt, ATTENTION_BACKEND, model_id, env_overrid } generator = VideoGenerator.from_pretrained(model_path=BASE_PARAMS["model_path"], **init_kwargs) - generator.generate_video(prompt, **generation_kwargs) + result = generator.generate_video(prompt, **generation_kwargs) generator.executor.shutdown() @@ -156,13 +157,11 @@ def test_matrixgame3_similarity(prompt, ATTENTION_BACKEND, model_id, env_overrid logger.error(f"Reference video not found for model: {model_id} with backend: {ATTENTION_BACKEND}") raise FileNotFoundError("Reference video missing") - generated_video_name = _find_mp4(output_dir) - if not generated_video_name: - logger.error(f"Generated video not found for model: {model_id} with backend: {ATTENTION_BACKEND}") - raise FileNotFoundError(f"Generated video missing in {output_dir}") + assert isinstance(result, dict), f"Expected one MatrixGame3 generation result, got {type(result)}" + generated_video_path = result["video_path"] + assert generated_video_path and os.path.isfile(generated_video_path), "Generated MatrixGame3 video missing" reference_video_path = os.path.join(reference_folder, reference_video_name) - generated_video_path = os.path.join(output_dir, generated_video_name) logger.info(f"Computing SSIM between {reference_video_path} and {generated_video_path}") ssim_values = compute_video_ssim_torchvision(reference_video_path, generated_video_path, use_ms_ssim=True) diff --git a/fastvideo/tests/ssim/test_sd35_similarity.py b/fastvideo/tests/ssim/test_sd35_similarity.py index d11b3f41b..d31513869 100644 --- a/fastvideo/tests/ssim/test_sd35_similarity.py +++ b/fastvideo/tests/ssim/test_sd35_similarity.py @@ -82,6 +82,7 @@ SD35_MODEL_TO_PARAMS = { } +@pytest.mark.skip(reason="SD3.5 SSIM is paused while gated-model access is not provisioned for CI.") @pytest.mark.skipif( not torch.cuda.is_available(), reason="SD3.5 SSIM test requires CUDA", diff --git a/fastvideo/tests/ssim/test_stable_audio_similarity.py b/fastvideo/tests/ssim/test_stable_audio_similarity.py index 340997ea5..9612031fc 100644 --- a/fastvideo/tests/ssim/test_stable_audio_similarity.py +++ b/fastvideo/tests/ssim/test_stable_audio_similarity.py @@ -106,5 +106,8 @@ def test_stable_audio_inference_similarity( # FSDP + @torch.inference_mode in StableAudioDenoisingStage hits # "Inference tensors do not track version counter" on single-GPU # unshard. SA-1.0 fits on one B200 anyway. - init_kwargs_override={"use_fsdp_inference": False}, + init_kwargs_override={ + "use_fsdp_inference": False, + "revision": "26609645c363fc81829282fcfe6356340346c1d4", + }, ) diff --git a/fastvideo/tests/training/VSA/gb200_reference_wandb_summary_VSA.json b/fastvideo/tests/training/VSA/gb200_reference_wandb_summary_VSA.json new file mode 100644 index 000000000..006557e26 --- /dev/null +++ b/fastvideo/tests/training/VSA/gb200_reference_wandb_summary_VSA.json @@ -0,0 +1,21 @@ +{ + "avg_step_time": 3.832678238349035, + "grad_norm": 1.9089118242263794, + "step_time": 1.1276781709166244, + "train_loss": 0.3086116909980774, + "vsa_sparsity": 0.05, + "provenance": { + "source_commit": "d5253fd71055927f12aa6c4aa679b93a00238851", + "device": "NVIDIA GB200", + "num_gpus": 2, + "torch": "2.12.0+cu130", + "cuda": "13.0", + "flash_attn": "2.8.3", + "fastvideo_fa4": false, + "model_revision": "0fad780a534b6463e45facd96134c9f345acfa5b", + "dataset_revision": "e97e41d3db696c534d90424f7efea514a63bfde5", + "validation_runs": 2, + "second_run_grad_norm": 1.9349586963653564, + "second_run_train_loss": 0.30886438488960266 + } +} diff --git a/fastvideo/tests/training/VSA/test_training_loss_VSA.py b/fastvideo/tests/training/VSA/test_training_loss_VSA.py index 493075af6..78638253a 100644 --- a/fastvideo/tests/training/VSA/test_training_loss_VSA.py +++ b/fastvideo/tests/training/VSA/test_training_loss_VSA.py @@ -16,7 +16,6 @@ from fastvideo.training.runner import main from fastvideo.utils import build_parser wandb_name = "test_training_loss_VSA" -reference_wandb_summary_file = "fastvideo/tests/training/VSA/h100_reference_wandb_summary_VSA.json" NUM_NODES = "1" NUM_GPUS_PER_NODE = "2" @@ -83,7 +82,9 @@ def test_distributed_training(): if "H100" not in device_name and "B200" not in device_name: raise ValueError(f"VSA training regression supports H100 and the GB200 Slurm CI target, got: {device_name}") - reference_wandb_summary = json.load(open(reference_wandb_summary_file)) + reference_name = "gb200_reference_wandb_summary_VSA.json" if "B200" in device_name else "h100_reference_wandb_summary_VSA.json" + reference_path = current_file.with_name(reference_name) + reference_wandb_summary = json.load(open(reference_path)) wandb_summary = json.load(open(summary_file)) fields_and_thresholds = {'avg_step_time': 15, 'grad_norm': 0.5, 'step_time': 15, 'train_loss': 0.04} diff --git a/tests/local_tests/performance/test_compare_baseline_policy.py b/tests/local_tests/performance/test_compare_baseline_policy.py index 6dfbfbb9b..0f7a7b12a 100644 --- a/tests/local_tests/performance/test_compare_baseline_policy.py +++ b/tests/local_tests/performance/test_compare_baseline_policy.py @@ -1618,10 +1618,13 @@ def test_v2_identity_lookup_ignores_model_id_and_gpu_display_string(tmp_path): assert records == [baseline] -def test_main_v2_identity_lookup_ignores_model_id_and_gpu_display_string( +@pytest.mark.parametrize("baseline_latency, expected_rc", [(10.0, 0), (1.0, 1)]) +def test_pr_compares_baseline_without_upload_credentials( monkeypatch, env_overrides, tmp_path, + baseline_latency, + expected_rc, ): tracking_root = tmp_path / "tracking" results_dir = tmp_path / "results" @@ -1637,7 +1640,7 @@ def test_main_v2_identity_lookup_ignores_model_id_and_gpu_display_string( "timestamp": "2026-06-15T00:00:00+00:00", "success": True, "baseline_eligible": True, - "latency": 10.0, + "latency": baseline_latency, "workload_id": "wan-t2v", "variant_id": "1.3b-sp2", "benchmark_version": 2, @@ -1654,17 +1657,29 @@ def test_main_v2_identity_lookup_ignores_model_id_and_gpu_display_string( monkeypatch.setattr(compare_baseline, "RESULTS_DIR", str(results_dir)) monkeypatch.setattr(compare_baseline, "PERF_REPORTS_DIR", str(reports_dir)) monkeypatch.setattr(compare_baseline, "UPLOAD_POLICY", "never") - monkeypatch.setattr(compare_baseline, "sync_from_hf", lambda local_dir, strict=False: local_dir) + sync_calls = [] + + def sync(local_dir, strict=False): + sync_calls.append(strict) + return local_dir + + def upload(*args, **kwargs): + pytest.fail("PR comparisons must not require upload credentials") + + monkeypatch.setattr(compare_baseline, "sync_from_hf", sync) + monkeypatch.setattr(compare_baseline, "upload_record", upload) env_overrides.enter_context(envs.override_external("PERF_PYTEST_RC", None)) - assert compare_baseline.main() == 0 + assert compare_baseline.main() == expected_rc + assert sync_calls == [True] artifacts = list((reports_dir / "results").glob("normalized_perf_*.json")) assert len(artifacts) == 1 with open(artifacts[0], encoding="utf-8") as f: normalized = json.load(f) - assert normalized["comparison_status"] == compare_baseline.STATUS_PASS + expected_status = compare_baseline.STATUS_PASS if expected_rc == 0 else compare_baseline.STATUS_REGRESSION + assert normalized["comparison_status"] == expected_status assert normalized["baseline_status"] == "compared" diff --git a/tests/local_tests/test_cookbook_serving.py b/tests/local_tests/test_cookbook_serving.py index a2ff32014..e214ce1a3 100644 --- a/tests/local_tests/test_cookbook_serving.py +++ b/tests/local_tests/test_cookbook_serving.py @@ -4,7 +4,6 @@ import ast import copy import json from pathlib import Path -from html.parser import HTMLParser import pytest import yaml @@ -233,32 +232,6 @@ def test_wan21_i2v_stays_python_only(): assert "serving" not in recipe -def test_h3_command_blocks_have_unique_copy_targets(): - class CodeBlocks(HTMLParser): - def __init__(self): - super().__init__() - self.ids = [] - - def handle_starttag(self, tag, attrs): - if tag == "pre": - block_id = dict(attrs).get("id") - if block_id: - self.ids.append(block_id) - - parser = CodeBlocks() - parser.feed((ROOT / "docs/cookbook/minimax-h3.md").read_text()) - assert parser.ids == [ - "cookbook-local-command", - "cookbook-server-install", - "cookbook-server-prepare", - "cookbook-server-command", - "cookbook-health-command", - "cookbook-client-install", - "cookbook-client-code", - ] - assert len(set(parser.ids)) == len(parser.ids) - - def test_kandinsky6_cookbook_recipes_match_maintained_examples(): recipes = json.loads(COOKBOOK_DATA.read_text())["recipes"] family = [recipe for recipe in recipes if recipe["family"] == "kandinsky6"]