nodes/hao-ai-lab-FastVideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[bugfix] Repair DreamVerse and failing CI gates (#1958)

Browse Source 
Co-authored-by: sy3535 <sy3535@maxwell.poly.edu>
Co-authored-by: Aryan Kumar <aryan5v@users.noreply.github.com>
This commit is contained in:
Aryan Kumar
2026-10-16 17:15:38 -07:00
committed by GitHub
co-authored by sy3535 Aryan Kumar
parent 6212e7262e
commit 3f45f2da21
40 changed files with 582 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.buildkite/scripts/lanes/dreamverse.sh
+1 -1
View File
@@ -75,7 +75,7 @@ if [[ $machine_arch =~ ^(aarch64|arm64)$ ]]; then
npm run e2e -- \
--project=chromium \
--project=mobile-chromium \
--grep-invert='streams, plays, and surfaces a downloadable clip|starts a new project and switches back to the prior session|saved projects persist across a page reload'
--grep-invert='streams, plays, and surfaces a downloadable clip|starts a new project and switches back to the prior session|saved projects persist across a page reload|sends validated assets and plays a clearly labeled sample'
else
BACKEND_HOST=127.0.0.1 BACKEND_PORT="$BACKEND_PORT" CI=1 \
npm run e2e -- \
.buildkite/scripts/lanes/inference_vmoba.sh
+1 -1
View File
@@ -2,4 +2,4 @@
# Canonical Slurm CI selection for the VMoBA-inference lane.
set -euo pipefail
exec python fastvideo/tests/inference/vmoba/test_vmoba_inference.py
exec pytest ./fastvideo/tests/inference/vmoba/test_vmoba_inference.py -vs
.buildkite/scripts/lanes/performance.sh
+5 -2
View File
@@ -9,9 +9,12 @@ mkdir -p "$PERF_REPORTS_DIR"
if [[ ${BUILDKITE_PULL_REQUEST:-false} =~ ^[1-9][0-9]*$ ]]; then
export PERF_RUN_SOURCE=pr
export PERF_UPLOAD_POLICY=pass
export PERF_UPLOAD_POLICY=never
elif [ "${BUILDKITE_BRANCH:-}" = main ] \
&& { [ "${BUILDKITE_SOURCE:-}" = schedule ] || [ "${TEST_SCOPE:-}" = full ]; }; then
&& { [ "${BUILDKITE_SOURCE:-}" = schedule ] || [ "${TEST_SCOPE:-}" = full ] \
|| { [ "${BUILDKITE_SOURCE:-}" = api ] \
&& [ "${BUILDKITE_PIPELINE_SLUG:-}" = fastvideo-performance-lane ] \
&& [ "${TEST_SCOPE:-}" = direct ]; }; }; then
export PERF_RUN_SOURCE=scheduled_main
export PERF_UPLOAD_POLICY=always
elif [ "${TEST_SCOPE:-}" = direct ]; then
.github/workflows/ci-scheduled-performance.yml
+73
View File
@@ -0,0 +1,73 @@
name: Scheduled Performance
on:
schedule:
# Sunday 23:00 America/Los_Angeles, including daylight saving time.
- cron: "0 6 * * 1"
- cron: "0 7 * * 1"
workflow_dispatch:
permissions:
contents: read
jobs:
trigger:
if: github.repository == 'hao-ai-lab/FastVideo' && github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
steps:
- name: Select the Pacific-time schedule
id: schedule
env:
EVENT_NAME: ${{ github.event_name }}
SCHEDULE_EXPR: ${{ github.event.schedule }}
run: |
set -euo pipefail
if [[ "$EVENT_NAME" == schedule ]]; then
expected='0 7 * * 1'
if [[ "$(TZ=America/Los_Angeles date +%z)" == -0700 ]]; then
expected='0 6 * * 1'
fi
if [[ "$SCHEDULE_EXPR" != "$expected" ]]; then
echo 'run=false' >> "$GITHUB_OUTPUT"
exit 0
fi
fi
echo 'run=true' >> "$GITHUB_OUTPUT"
- name: Trigger weekly performance on Slinky Slurm
if: steps.schedule.outputs.run == 'true'
env:
BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
SOURCE_SHA: ${{ github.sha }}
SOURCE_BRANCH: ${{ github.event.repository.default_branch }}
BK_ORG: ${{ vars.BUILDKITE_ORG_SLUG }}
BK_PIPELINE: fastvideo-performance-lane
run: |
set -euo pipefail
if [[ ${#SOURCE_SHA} -ne 40 || "$SOURCE_SHA" == *[!0-9a-f]* ]]; then
echo 'SOURCE_SHA must be a full immutable commit SHA' >&2
exit 1
fi
payload=$(python3 - <<'PY'
import json
import os
print(json.dumps({
"commit": os.environ["SOURCE_SHA"],
"branch": os.environ["SOURCE_BRANCH"],
"message": "Weekly performance on Slinky Slurm",
"ignore_pipeline_branch_filters": True,
"env": {
"TEST_SCOPE": "direct",
"FULL_SUITE": "false",
"TEST_TYPE": "performance",
"PR_NUMBER": "false",
"PR_TITLE": "Scheduled performance",
},
}))
PY
)
curl -sS --fail-with-body --connect-timeout 10 --max-time 60 -X POST \
"https://api.buildkite.com/v2/organizations/${BK_ORG}/pipelines/${BK_PIPELINE}/builds" \
-H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
-H "Content-Type: application/json" \
--data-raw "$payload"
.pre-commit-config.yaml
+1
View File
@@ -21,6 +21,7 @@ repos:
rev: v0.43.0
hooks:
- id: yapf
require_serial: true
args: [--in-place, --verbose]
language_version: python3.12
additional_dependencies: [toml] # TODO: Remove when yapf is upgraded
apps/dreamverse/web/e2e/mock-backed-generation.spec.ts
+31
View File
@@ -3,6 +3,37 @@ import { stat } from 'node:fs/promises';
import { test, expect, type WebSocket as PWWebSocket } from '@playwright/test';
test.describe('mock-backed generation smoke', () => {
test('leave dialog blocks navigation and keeps cancel and confirm usable', async ({ page, request }) => {
const health = await request.get('/healthz');
const body = health.ok() ? await health.json() : {};
test.skip(body.service !== 'ltx2-streaming-mock-server', 'Requires the local mock runtime.');
await page.goto('/');
await page.getByLabel('Continuation prompt').fill('A paper fox walks through a sunlit forest.');
await page.getByRole('button', { name: 'Generate', exact: true }).click();
const leave = page.getByRole('button', { name: 'Leave', exact: true });
await expect(leave).toBeEnabled();
await leave.click();
const dialog = page.getByRole('dialog', { name: 'Leave session?' });
await expect(dialog).toBeVisible();
expect(await dialog.evaluate((element) => element.closest('.isolate') === null)).toBe(true);
await expect(page.getByRole('button', { name: 'Toggle sidebar' }).click({ trial: true, timeout: 1000 })).rejects.toThrow();
for (let index = 0; index < 4; index++) {
await page.keyboard.press('Tab');
await expect.poll(() => dialog.evaluate((element) => element.contains(document.activeElement))).toBe(true);
}
await dialog.getByRole('button', { name: 'Cancel', exact: true }).click();
await expect(dialog).toHaveCount(0);
await expect(leave).toBeFocused();
await leave.click();
await page.keyboard.press('Escape');
await expect(dialog).toHaveCount(0);
await expect(leave).toBeFocused();
await leave.click();
await dialog.getByRole('button', { name: 'Leave session', exact: true }).click();
await expect(dialog).toHaveCount(0);
await expect(page.getByRole('button', { name: 'Generate', exact: true })).toBeVisible();
});
test('accepts a custom prompt and enters the streaming state', async ({ page, request }) => {
const health = await request.get('/healthz');
const body = health.ok() ? await health.json() : {};
apps/dreamverse/web/src/app/page.tsx
+1 -1
View File
@@ -2739,7 +2739,7 @@ export default function Page() {
<Header timeLeft={headerTimeLeft} formatTime={formatTime} onToggleSidebar={() => setSidebarOpen((prev) => !prev)} />
<div className={cn(
"relative flex flex-1 min-h-0 flex-col px-4 pb-2 sm:px-6 sm:pb-12",
"relative isolate flex flex-1 min-h-0 flex-col px-4 pb-2 sm:px-6 sm:pb-12",
!isViewingMode && !showActiveProject && generationMode !== "t2va"
? "justify-start overflow-y-auto pt-4"
: "justify-center",
apps/dreamverse/web/src/components/LeaveSessionModal.tsx
+54 -37
View File
@@ -1,6 +1,7 @@
"use client";
import { useState, useEffect } from "react";
import { useState, useEffect, useRef } from "react";
import { Dialog as DialogPrimitive } from "radix-ui";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import { Label } from "@/components/ui/label";
@@ -19,6 +20,7 @@ export default function LeaveSessionModal({
onConfirmLeave = () => {},
}: LeaveSessionModalProps) {
const [suppress, setSuppress] = useState(false);
const previousFocus = useRef<HTMLElement | null>(null);
useEffect(() => {
if (open) setSuppress(false);
@@ -36,43 +38,58 @@ export default function LeaveSessionModal({
}
return (
<div className="fixed inset-0 z-[70] flex items-center justify-center bg-black/45 px-4 backdrop-blur-[3px]">
<div
role="dialog"
aria-modal="true"
aria-labelledby="leave-session-title"
className="w-full max-w-md rounded-3xl border border-border/70 bg-card/95 p-6 shadow-2xl"
>
<div className="space-y-3">
<div className="space-y-1">
<h2 id="leave-session-title" className="text-lg font-semibold text-foreground">
Leave session?
</h2>
<p className="text-sm text-muted-foreground">
This will end your current session. You will need to queue again for a GPU to start a new project.
</p>
<DialogPrimitive.Root open={open} onOpenChange={(nextOpen) => { if (!nextOpen) onClose(); }}>
<DialogPrimitive.Portal>
<DialogPrimitive.Overlay asChild>
<div className="fixed inset-0 z-[70] flex items-center justify-center bg-black/45 px-4 backdrop-blur-[3px]">
<DialogPrimitive.Content
asChild
onInteractOutside={(event) => event.preventDefault()}
onOpenAutoFocus={() => {
previousFocus.current = document.activeElement instanceof HTMLElement ? document.activeElement : null;
}}
onCloseAutoFocus={(event) => {
event.preventDefault();
if (previousFocus.current?.isConnected) previousFocus.current.focus();
}}
>
<div
className="w-full max-w-md rounded-3xl border border-border/70 bg-card/95 p-6 shadow-2xl"
>
<div className="space-y-3">
<div className="space-y-1">
<DialogPrimitive.Title className="text-lg font-semibold text-foreground">
Leave session?
</DialogPrimitive.Title>
<DialogPrimitive.Description className="text-sm text-muted-foreground">
This will end your current session. You will need to queue again for a GPU to start a new project.
</DialogPrimitive.Description>
</div>
<div className="flex items-center gap-2">
<Checkbox
id="suppress-leave-warning"
checked={suppress}
onCheckedChange={(v) => setSuppress(v === true)}
/>
<Label htmlFor="suppress-leave-warning" className="text-sm text-muted-foreground cursor-pointer">
Do not warn again
</Label>
</div>
</div>
<div className="mt-5 flex flex-col-reverse gap-2 sm:flex-row sm:justify-end">
<Button variant="outline" onClick={onClose}>
Cancel
</Button>
<Button variant="destructive" onClick={handleConfirm}>
Leave session
</Button>
</div>
</div>
</DialogPrimitive.Content>
</div>
<div className="flex items-center gap-2">
<Checkbox
id="suppress-leave-warning"
checked={suppress}
onCheckedChange={(v) => setSuppress(v === true)}
/>
<Label htmlFor="suppress-leave-warning" className="text-sm text-muted-foreground cursor-pointer">
Do not warn again
</Label>
</div>
</div>
<div className="mt-5 flex flex-col-reverse gap-2 sm:flex-row sm:justify-end">
<Button variant="outline" onClick={onClose}>
Cancel
</Button>
<Button variant="destructive" onClick={handleConfirm}>
Leave session
</Button>
</div>
</div>
</div>
</DialogPrimitive.Overlay>
</DialogPrimitive.Portal>
</DialogPrimitive.Root>
);
}
docs-site/tests/cookbook.spec.ts
+19
View File
@@ -55,6 +55,25 @@ test("edits generate YAML and client instructions, invalid edits clear output, r
await expect(yaml).toHaveText(baseline);
});
test("copy buttons use their own generated output", async ({ page, context }) => {
await context.grantPermissions(["clipboard-read", "clipboard-write"]);
await page.goto(route);
await output(page, "config.yaml");
const yaml = await page.getByLabel("Generated YAML", { exact: true }).innerText();
await page.getByRole("button", { name: "Copy YAML", exact: true }).click();
await expect.poll(() => page.evaluate(() => navigator.clipboard.readText())).toBe(yaml);
await output(page, "Commands");
await page.locator(".recipe-request-code > summary").click();
const copied = [yaml];
for (const label of ["Server command", "Client command"]) {
const command = await page.getByLabel(label, { exact: true }).innerText();
await page.getByRole("button", { name: `Copy ${label.toLowerCase()}`, exact: true }).click();
await expect.poll(() => page.evaluate(() => navigator.clipboard.readText())).toBe(command);
copied.push(command);
}
expect(new Set(copied).size).toBe(3);
});
test("deployment changes reset edits and support MLX and streaming", async ({ page }) => {
await page.goto(route);
await settings(page, "Serving");
docs/contributing/ci_architecture.md
+91 -1
View File
@@ -60,7 +60,7 @@ Three Buildkite entry pipelines share the validated graph:
|---|---|---|
| `pr-fastcheck` | Automatic pull-request webhook | Six Fastcheck lanes |
| `ci` | `/merge`, `ready`, schedules, and `/test` API builds | Change-aware merge gates, scheduled SSIM, explicit Full Suite, Fastcheck reruns, or one direct lane |
| `fastvideo-performance-lane` | Weekly scheduler | Direct performance lane |
| `fastvideo-performance-lane` | GitHub weekly scheduler | Direct performance lane |
Each entry pipeline starts with the same trusted `pipeline-upload` job on the
`ci-runner` queue. The `ci` pipeline's incoming GitHub webhook is disabled;
@@ -205,6 +205,95 @@ inside that range.
See [Performance Benchmarks](performance_benchmarks.md) for the performance
lane's thresholds, rolling baseline, artifacts, and reseeding process.
`ci-scheduled-performance.yml` triggers the performance pipeline every Sunday
at 23:00 America/Los_Angeles, passing the immutable `github.sha`. Its two UTC
schedules select the matching Pacific offset so it runs once across daylight
saving time. After this workflow lands, disable the legacy Buildkite schedule
under **Settings → Schedules → FastVideo scheduled performance benchmarks →
Edit → Enabled**. That schedule submits the literal `HEAD`, which the trusted
runner rejects before upload. The GitHub `BUILDKITE_API_TOKEN` must have
`write_builds` access to the performance pipeline.
The performance lane recognizes main-branch API builds from the dedicated
performance pipeline with `TEST_SCOPE=direct` as scheduled-main runs. They
refresh eligible baselines after successful tests and still produce comparison
reports after test failures. PR performance runs use strict baseline reads and
keep reports as Buildkite artifacts without requiring upload credentials.
Scheduled-main publishing requires an operator-provisioned token with write
access to `FastVideo/performance-tracking`.
The gated FLUX.1-dev SSIM and golden tests and FLUX.2-klein-9B SSIM test are explicitly paused until their
model access is provisioned. Their pytest skips are not FLUX quality validation;
the other SSIM and golden tests and the suites' fail-fast behavior remain active.
The SD3.5 golden and SSIM tests are also explicitly paused at the maintainer's
request while their gated checkpoint access is unavailable.
MatrixGame 3 explicitly selects FA2 for its existing GB200 reference. That
reference was published in dataset commit `e1a6e64` and was not part of the
later FA4 refresh. Unchanged main with FA2 reproduces its video SHA-256
`1d1d26a59481e37a9e838a628d875e196211c2236e8f9a370e0a9f73e97300c7`;
FA4 scores 0.96636 against its unchanged 0.98 requirement. The test compares
the current generation's returned path so older files cannot mask that check.
The missing GB200 HunyuanVideo 1.5 I2V and Kandinsky 6 references were generated
from unchanged upstream commit `d5253fd71055927f12aa6c4aa679b93a00238851`
with Torch 2.12.0, CUDA 13 and FA4. Their checkpoint revisions are pinned in
the tests and recorded alongside the videos. Independent upstream repeats and
the candidate both scored 1.0 for Hunyuan and 0.98395 for Kandinsky against
the first upstream outputs, preserving the existing 0.95 and 0.98 thresholds.
Kandinsky remains required; its authorized complete snapshot is prepared in
the shared CI cache without exposing an authentication token to PR code.
LingBot Fast's missing GB200 reference uses the same upstream commit and
runtime, with checkpoint revision `98583eb0fd66f481f73383428d65d2867c878e51`.
Two unchanged-upstream runs with CPU offloading disabled produced identical
video hashes. Its fixed FSDP CPU-offloaded run scored SSIM 1.0 against that
reference, preserving the 0.70 requirement. The generation stage leaves FSDP
parameter placement to FSDP instead of moving the whole model to the GPU.
### Provisioning Required Hugging Face Models
Gated access belongs to the individual HF account, not its organization or
token. An operator must request and obtain model access for the CI identity
before configuring authentication. A valid token without model access still
returns 403. Keep required models such as Kandinsky 6 strict until their
weights and quality references are provisioned.
The permanent operator rollout is:
1. Approve every required gated repository for one maintained CI identity.
2. Configure [HF CI/CD access through OIDC](https://huggingface.co/docs/hub/trusted-publishers)
for the trusted Buildkite preparation step, restricting issuer and job
claims to the intended organization and pipelines. A fine-grained read-only
token in the operator's secret store is an alternative. Keep performance
publishing credentials separate and restricted to the tracking dataset and
trusted main runs.
3. Before leasing GPUs, check access to all required pinned manifests and
weight files together and check that their device/backend references exist.
Report every missing dependency in one preflight result; do not turn access
failures into passing quality gates.
4. Fetch immutable model revisions into the runner-owned cache through trusted
code. Remove authentication before starting PR code, and keep download locks
and cache writes under the runner's identity. PR jobs consume the prepared
snapshots rather than depending on an interactive login or another user's
writable cache.
When multiple operators prepare the shared cache, use its existing group
with `umask 0002`. Check group write access on each prepared model's cache,
snapshot, blob and `.locks` directories, not only the cache root. Verify
manifest resolution as the CI worker identity before running the lane.
A readable weight file alone is insufficient: Hub downloads also create
metadata and lock files. For example, Stable Audio's pinned revision is
`26609645c363fc81829282fcfe6356340346c1d4`; provision its complete snapshot,
including `model_index.json`, component configs and tokenizer files.
5. For a missing SSIM reference, generate and inspect output from unchanged
upstream source on the CI GPU/backend/runtime, retain its provenance, and
publish only the reviewed new reference. Verify the candidate independently
before enabling the required gate; never use candidate output to clear a
regression failure.
See [HF gated-model access](https://huggingface.co/docs/hub/models-gated) and
the [testing guide](testing.md) for access and reference maintenance details.
## `/merge` Request Flow
The Buildkite agent and Slurm have deliberately separate responsibilities:
@@ -476,6 +565,7 @@ through checks, commit statuses, or labels instead of PR/issue comments.
| `.github/workflows/ci-slash-commands.yml` | `/merge` and `/test` handling |
| `.github/workflows/ci-trigger-full-suite.yml` | Change-aware merge-gate trigger for `ready` PRs and new pushes |
| `.github/workflows/ci-scheduled-ssim.yml` | Weekly complete SSIM trigger on `main` |
| `.github/workflows/ci-scheduled-performance.yml` | Weekly performance trigger with an immutable source SHA |
| `.github/scripts/plan_merge_ci.py` | Trusted changed-path to integration-lane planner |
| `.github/workflows/ci-aggregate-status.yml` | Aggregate Fastcheck and explicit Full Suite direct-rerun statuses |
| `.buildkite/pipeline.yml` | Static 20-lane Slurm Buildkite graph |
docs/contributing/env_vars.md
+1
View File
@@ -218,6 +218,7 @@ longer exists also fails the test, so the fixing pull request deletes its entry.
| `FASTVIDEO_LTX2_VAE_CHANNELS_LAST_3D` | bool | `1` | performance | LTX-2 VAE uses the channels_last_3d memory format. |
| `FASTVIDEO_LTX2_DISABLE_AUDIO_AUTOCAST` | bool | `1` | performance | LTX-2 audio decoding runs without CUDA autocast. Deprecated names: `LTX2_DISABLE_AUDIO_AUTOCAST`. |
| `FASTVIDEO_FLUX2_DISABLE_BF16_REDUCED_PRECISION_REDUCTION` | bool | `0` | performance | Flux denoising disables reduced-precision reductions in bf16 matmuls, which tightens accumulation for the 4-step Klein model. |
| `FASTVIDEO_ENABLE_FUSED_NORM` | bool | `0` | performance | Opt into Wan's inference-only Triton residual + LayerNorm + modulation fusion. Its reduction order changes numerical outputs; the default keeps eager reference compatibility. |
| `FASTVIDEO_DISABLE_FUSED_NORM` | bool | `0` | performance | Turn off the Triton-fused residual + LayerNorm + modulate inference path that Wan blocks opt into, and use the eager path. |
| `FASTVIDEO_FFMPEG_BIN` | str | `ffmpeg` | output | ffmpeg executable used to save video with audio. |
| `FASTVIDEO_VIDEO_CODEC` | str | `libx264` | output | ffmpeg video codec for saved videos. |
docs/contributing/performance_benchmarks.md
+5 -4
View File
@@ -44,8 +44,9 @@ python fastvideo/tests/performance/dashboard.py
The pytest run never uploads anything. `compare_baseline.py` uploads only when
`PERF_UPLOAD_POLICY` is set. Local uploads are explicit opt-in and require HF
credentials. PR/direct performance runs upload passing records for dashboard
visibility, while scheduled-main runs upload both pass and fail records. The
credentials. PR performance runs read baselines strictly and retain comparison
reports as Buildkite artifacts without uploading records. Scheduled-main runs
upload both pass and fail records using an operator-provisioned CI credential. The
report directory default is container-oriented; set `PERF_REPORTS_DIR` to a
writable local path when generating dashboards or when you want local
Markdown/normalized-result artifacts from the comparator.
@@ -244,8 +245,8 @@ metrics are skipped.
This is the **drift detector** — it catches sub-threshold regressions that
slowly add up. Only scheduled-main successful records are baseline eligible.
Local and pull-request runs can upload dashboard-visible records, but they do
not update future gating baselines.
Local uploads are explicit opt-in and do not update future gating baselines.
PR runs retain their normalized records and reports as Buildkite artifacts.
Comparator summaries and normalized artifacts include an explicit
`comparison_status`:
docs/contributing/testing.md
+13
View File
@@ -53,6 +53,11 @@ or environment mismatch is not a pass. Do not replace a reference with the
candidate output just to clear a failure. For a relocation, the unchanged
parent is the baseline; two imports of the same class are not numerical proof.
FA4 fingerprints record the effective boolean switch. Legacy block references
with `fastvideo_fa4: <unset>` mean disabled and match an explicit `0`; enabling
FA4 still fails the environment check. Other runtime pins and zero-tolerance
tensor comparisons remain strict.
The VAE and transformer CI lanes run their Wan component goldens first.
Selected integration lanes wait for the golden lane in merge/full builds.
See [CI/CD Architecture](ci_architecture.md) for direct-rerun and skip semantics.
@@ -95,6 +100,14 @@ pytest fastvideo/tests/vaes/test_wan_vae.py -q
GPU-heavy suites need the right hardware, credentials, local caches, and
sometimes custom kernels. Document those assumptions in new tests.
The legacy VSA training regression selects a device-specific summary. Its
GB200 reference was generated from unchanged upstream commit
`d5253fd71055927f12aa6c4aa679b93a00238851` using two GB200 GPUs, Torch
2.12.0+cu130, CUDA 13.0 and FA2. Two independent upstream runs produced
gradient norms 1.909 and 1.935 and losses 0.30861 and 0.30886. The reference
JSON retains provenance; the H100 reference and all comparison tolerances
remain unchanged. Candidate outputs must not be used to refresh this reference.
## SSIM Tests
SSIM tests generate videos using specific models and parameters, then compare
fastvideo/envs.py
+5
View File
@@ -458,6 +458,11 @@ FASTVIDEO_FLUX2_DISABLE_BF16_REDUCED_PRECISION_REDUCTION = EnvBool(
category="performance",
doc="Flux denoising disables reduced-precision reductions in bf16 matmuls, which tightens accumulation for "
"the 4-step Klein model.")
FASTVIDEO_ENABLE_FUSED_NORM = EnvBool(
False,
category="performance",
doc="Opt into Wan's inference-only Triton residual + LayerNorm + modulation fusion. "
"Its reduction order changes numerical outputs; the default keeps eager reference compatibility.")
# Global kill switch for the opt-in, inference-only Triton fusion in
# fastvideo/layers/triton_fused_norm.py, for debugging numerics or Triton issues.
FASTVIDEO_DISABLE_FUSED_NORM = EnvBool(False,
fastvideo/eval/metrics/vbench/dynamic_degree/metric.py
+1 -1
View File
@@ -11,7 +11,6 @@ from typing import Any
import numpy as np
import torch
from easydict import EasyDict
from fastvideo.eval.metrics.base import BaseMetric
from fastvideo.eval.registry import register
@@ -41,6 +40,7 @@ class DynamicDegreeMetric(BaseMetric):
def setup(self) -> None:
if self._model is not None:
return
from easydict import EasyDict
from vbench.third_party.RAFT.core.raft import RAFT
args = EasyDict(small=False, mixed_precision=False, alternate_corr=False, dropout=0.0)
fastvideo/layers/triton_fused_norm.py
+3 -2
View File
@@ -25,7 +25,8 @@ from eager are the reduction order inside mean/variance (last-ulp fp32) and
that product-rounding case.
Inference-only: callers must gate on ``torch.is_grad_enabled()``; there is no
backward. Disable globally with FASTVIDEO_DISABLE_FUSED_NORM=1.
backward. Enable with FASTVIDEO_ENABLE_FUSED_NORM=1; the default preserves the
eager numerical path. FASTVIDEO_DISABLE_FUSED_NORM=1 overrides the opt-in.
"""
from __future__ import annotations
@@ -158,7 +159,7 @@ def fused_path_supported(
norm: torch.nn.Module,
) -> bool:
"""Cheap eligibility check; any False falls back to the eager path."""
if not _HAS_TRITON or envs.FASTVIDEO_DISABLE_FUSED_NORM.get():
if not _HAS_TRITON or not envs.FASTVIDEO_ENABLE_FUSED_NORM.get() or envs.FASTVIDEO_DISABLE_FUSED_NORM.get():
return False
if torch.is_grad_enabled():
return False
fastvideo/pipelines/basic/lingbotworld2/causal_fast_pipeline.py
+2 -1
View File
@@ -281,7 +281,8 @@ class LingBotWorld2CausalFastGenerationStage(PipelineStage):
self_kv_cache = self._initialize_self_kv_cache(1, kv_size, transformer_dtype, device)
cross_kv_cache = self._initialize_crossattn_cache(1, max_sequence_length, transformer_dtype, device)
self.transformer = self.transformer.to(device)
if not fastvideo_args.use_fsdp_inference:
self.transformer = self.transformer.to(device)
self._cross_attn_initialized = False
pred_latent_chunks = []
latents_chunk = noise.split(chunk_size, dim=1)
fastvideo/tests/contract/test_ci_test_collection.py
+111
View File
@@ -249,6 +249,103 @@ def test_full_ssim_has_a_weekly_slurm_schedule():
assert 'build.env("TEST_SCOPE") == "scheduled"' in str(ssim_step["if"])
def test_performance_trigger_rejects_head_and_sends_a_pinned_direct_build():
import json
import os
import subprocess
import tempfile
workflow = yaml.safe_load((REPO_ROOT / '.github/workflows/ci-scheduled-performance.yml').read_text())
job = workflow['jobs']['trigger']
assert "github.repository == 'hao-ai-lab/FastVideo'" in job['if']
assert "github.ref == 'refs/heads/main'" in job['if']
step = job['steps'][-1]
assert step['env']['SOURCE_SHA'] == '${{ github.sha }}'
with tempfile.TemporaryDirectory() as root:
curl = Path(root) / 'curl'
curl.write_text('#!/bin/bash\nwhile [[ $# -gt 0 ]]; do\n'
' if [[ "$1" == --data-raw ]]; then printf "%s" "$2"; exit 0; fi\n'
' shift\ndone\nexit 1\n')
curl.chmod(0o755)
env = {
'PATH': root + os.pathsep + os.environ['PATH'],
'BUILDKITE_API_TOKEN': 'test-only',
'SOURCE_BRANCH': 'main',
'BK_ORG': 'fastvideo',
'BK_PIPELINE': step['env']['BK_PIPELINE'],
}
for commit in ('HEAD', 'abc123', 'g' * 40, 'a' * 40):
env['SOURCE_SHA'] = commit
result = subprocess.run(['bash', '-c', step['run']], env=env, text=True, capture_output=True)
if commit != 'a' * 40:
assert result.returncode != 0
assert not result.stdout
continue
assert result.returncode == 0, result.stderr
payload = json.loads(result.stdout)
assert payload['commit'] == commit
assert payload['branch'] == 'main'
assert payload['env']['TEST_TYPE'] == 'performance'
assert payload['env']['TEST_SCOPE'] == 'direct'
assert payload['env']['FULL_SUITE'] == 'false'
assert payload['env']['PR_NUMBER'] == 'false'
def test_weekly_performance_api_build_keeps_scheduled_baseline_policy():
import os
import subprocess
import tempfile
lane = (REPO_ROOT / '.buildkite/scripts/lanes/performance.sh').read_text()
lane = lane.replace('export PERF_REPORTS_DIR=/workspace/artifacts/performance',
'export PERF_REPORTS_DIR="$TEST_REPORTS_DIR"')
cases = (
('main', 'api', 'fastvideo-performance-lane', 'direct', 'false', 'scheduled_main', 'always'),
('main', 'schedule', 'fastvideo-performance-lane', 'direct', 'false', 'scheduled_main', 'always'),
('main', 'api', 'ci', 'full', 'false', 'scheduled_main', 'always'),
('main', 'api', 'ci', 'direct', 'false', 'unknown', 'pass'),
('feature', 'api', 'fastvideo-performance-lane', 'direct', 'false', 'unknown', 'pass'),
('main', 'api', 'fastvideo-performance-lane', 'direct', '1958', 'pr', 'never'),
)
with tempfile.TemporaryDirectory() as root:
directory = Path(root)
mocks = {
'nvidia-smi': '#!/bin/sh\nexit 0\n',
'pytest': '#!/bin/sh\nexit "$TEST_PYTEST_RC"\n',
'python': '#!/bin/sh\nprintf "%s|%s|%s|%s\\n" "$1" "$PERF_RUN_SOURCE" '
'"$PERF_UPLOAD_POLICY" "${PERF_PYTEST_RC:-}" >> "$TEST_CALLS_FILE"\n',
}
for name, contents in mocks.items():
executable = directory / name
executable.write_text(contents)
executable.chmod(0o755)
calls = directory / 'calls.txt'
for branch, source, pipeline, scope, pr, run_source, policy in cases:
for pytest_rc in (0, 1):
calls.unlink(missing_ok=True)
env = {
'PATH': root + os.pathsep + os.environ['PATH'],
'BUILDKITE_BRANCH': branch,
'BUILDKITE_SOURCE': source,
'BUILDKITE_PIPELINE_SLUG': pipeline,
'BUILDKITE_PULL_REQUEST': pr,
'TEST_SCOPE': scope,
'TEST_PYTEST_RC': str(pytest_rc),
'TEST_CALLS_FILE': str(calls),
'TEST_REPORTS_DIR': str(directory / 'reports'),
}
result = subprocess.run(['bash', '-c', lane], cwd=root, env=env, text=True, capture_output=True)
assert result.returncode == pytest_rc, result.stderr
records = [line.split('|') for line in calls.read_text().splitlines()]
assert all(record[1:3] == [run_source, policy] for record in records)
comparisons = [record for record in records if record[0].endswith('/compare_baseline.py')]
if pytest_rc == 0 or policy == 'always':
assert len(comparisons) == 1
assert comparisons[0][3] == str(pytest_rc)
else:
assert not comparisons
def test_active_pipeline_has_no_modal_or_untrusted_compute_path():
pipeline_text = (REPO_ROOT / ".buildkite/pipeline.yml").read_text()
steps = _pipeline_steps()
@@ -360,3 +457,17 @@ def test_allowlist_entries_are_still_real_directories():
# A stale allowlist hides regressions; entries must track reality.
missing = [name for name in ALLOWLIST if name != "modal" and not (TESTS_ROOT / name).is_dir()]
assert not missing, (f"Allowlisted directories no longer exist — remove them: {missing}")
def test_arm64_chromium_filters_generation_mode_playback_but_keeps_uploads():
lane = (REPO_ROOT / '.buildkite/scripts/lanes/dreamverse.sh').read_text()
pattern = re.search(r"--grep-invert='([^']+)'", lane).group(1)
for mode in ('t2va', 'fl2va', 'ref2va'):
title = f'{mode} sends validated assets and plays a clearly labeled sample'
assert re.search(pattern, title), f'ARM64 Chromium cannot decode this playback test: {title}'
assert not re.search(pattern, "proxies a media upload larger than Next's default 10 MiB body limit")
assert not re.search(pattern, 'accepts a custom prompt and enters the streaming state')
arm_branch = lane.rsplit('if [[ $machine_arch =~ ^(aarch64|arm64)$ ]]; then', 1)[1].split('\nelse')[0]
firefox_command = re.search(r'npm run e2e(?:\\\n|[^\n])*', arm_branch).group(0)
assert '--project=firefox' in firefox_command
assert '--grep' not in firefox_command
fastvideo/tests/contract/test_golden_environment.py
+37
View File
@@ -0,0 +1,37 @@
# SPDX-License-Identifier: Apache-2.0
import pytest
import fastvideo.envs as envs
from fastvideo.tests.golden_gate._harness import GateSpec, assert_env_fingerprint_matches, env_fingerprint
@pytest.mark.parametrize("value", [None, False, True])
def test_block_fingerprint_records_effective_fa4(monkeypatch, value):
import torch
monkeypatch.setattr(torch.cuda, "get_device_name", lambda index: "NVIDIA GB200")
spec = GateSpec("test", "test/repo", lambda layer: None, lambda device, seed: {})
with envs.FASTVIDEO_FA4.override(value):
assert env_fingerprint(spec)["fastvideo_fa4"] == str(int(envs.FASTVIDEO_FA4.get()))
def test_legacy_unset_fa4_matches_disabled_without_changing_reference():
reference = {"fastvideo_fa4": "<unset>", "torch": "pinned"}
assert_env_fingerprint_matches(reference, {"fastvideo_fa4": "0", "torch": "pinned"})
assert reference == {"fastvideo_fa4": "<unset>", "torch": "pinned"}
@pytest.mark.parametrize("reference_fa4", ["<unset>", "0", "1"])
def test_fa4_compute_path_changes_still_fail(reference_fa4):
current_fa4 = "0" if reference_fa4 == "1" else "1"
with pytest.raises(AssertionError, match="fastvideo_fa4"):
assert_env_fingerprint_matches({"fastvideo_fa4": reference_fa4}, {"fastvideo_fa4": current_fa4})
@pytest.mark.parametrize("field", ["torch", "cuda", "cudnn", "flash_attn", "device_name", "attention_backend",
"tf32_matmul", "tf32_cudnn"])
def test_other_runtime_changes_still_fail(field):
with pytest.raises(AssertionError, match=field):
assert_env_fingerprint_matches({"fastvideo_fa4": "<unset>", field: "old"},
{"fastvideo_fa4": "0", field: "new"})
fastvideo/tests/eval/test_registry_optional_deps.py
+21
View File
@@ -0,0 +1,21 @@
# SPDX-License-Identifier: Apache-2.0
import subprocess
import sys
def test_dynamic_degree_stays_registered_without_easydict():
script = """
import sys
sys.modules['easydict'] = None
from fastvideo.eval.registry import get_metric, list_metrics
assert 'vbench.dynamic_degree' in list_metrics()
try:
get_metric('vbench.dynamic_degree')
except ImportError as error:
assert 'easydict' in str(error)
else:
raise AssertionError('Missing optional dependency was not reported')
"""
result = subprocess.run([sys.executable, "-c", script], text=True, capture_output=True)
assert result.returncode == 0, result.stderr
fastvideo/tests/golden_gate/_harness.py
+12 -12
View File
@@ -74,7 +74,7 @@ def env_fingerprint(spec: GateSpec | str | None) -> dict[str, str]:
"flash_attn": fa_version,
"device_name": torch.cuda.get_device_name(0),
"attention_backend": backend or "NONE",
"fastvideo_fa4": str(int(envs.FASTVIDEO_FA4.get())) if envs.FASTVIDEO_FA4.is_set() else "<unset>",
"fastvideo_fa4": str(int(envs.FASTVIDEO_FA4.get())),
"tf32_matmul": str(torch.backends.cuda.matmul.allow_tf32),
"tf32_cudnn": str(torch.backends.cudnn.allow_tf32),
}
@@ -83,14 +83,19 @@ def env_fingerprint(spec: GateSpec | str | None) -> dict[str, str]:
# part of the codec compute path. Keep torch/CUDA/cuDNN/device/TF32 pins.
result.pop("flash_attn")
result.pop("fastvideo_fa4")
elif not isinstance(spec, GateSpec):
# Tensor gates record the effective switch, so unset and "0" both record
# "0". Block gates record "<unset>" when FASTVIDEO_FA4 is unset, and "1"
# or "0" when it is set.
result["fastvideo_fa4"] = str(int(envs.FASTVIDEO_FA4.get()))
return result
def assert_env_fingerprint_matches(expected: dict[str, str], actual: dict[str, str]) -> None:
"""Legacy block goldens recorded disabled FA4 as '<unset>'; compare its effective value."""
expected = dict(expected)
if expected.get("fastvideo_fa4") == "<unset>":
expected["fastvideo_fa4"] = "0"
mismatches = {key: (value, actual.get(key)) for key, value in expected.items() if actual.get(key) != value}
assert not mismatches, (f"environment changed since golden was minted — re-mint deliberately rather "
f"than chasing bit drift. golden -> current: {mismatches}")
def _component_dir(spec: GateSpec) -> Path:
"""Local checkout wins; otherwise fetch the index + only the shards holding
this layer (or the single weight file for index-less checkpoints)."""
@@ -228,12 +233,7 @@ def _run_block_against_golden(spec: GateSpec) -> None:
)
golden = torch.load(golden_path, weights_only=True)
mismatches = {
key: (value, meta["env"].get(key))
for key, value in golden["metadata"]["env"].items() if meta["env"].get(key) != value
}
assert not mismatches, (f"environment changed since golden was minted — re-mint deliberately rather "
f"than chasing bit drift. golden -> current: {mismatches}")
assert_env_fingerprint_matches(golden["metadata"]["env"], meta["env"])
drift = (out - golden["output"]).abs()
print(f"golden-gate[{spec.name}] drift: max_abs={drift.max().item():.10f} "
fastvideo/tests/golden_gate/test_flux.py
+2
View File
@@ -14,6 +14,7 @@ is a GATED HF repo; FASTVIDEO_TEST_FLUX_T2I_MODEL_DIR can point at a local check
from __future__ import annotations
import pytest
import torch
import fastvideo.envs as envs
@@ -82,5 +83,6 @@ SPEC = GateSpec(
)
@pytest.mark.skip(reason="FLUX.1-dev golden gate is paused while gated-model access is not provisioned for CI.")
def test_flux_golden_gate(distributed_runtime) -> None:
run_gate(SPEC)
fastvideo/tests/golden_gate/test_minimax_h3_t2v.py
+3 -7
View File
@@ -38,6 +38,7 @@ import torch
from torch.testing import assert_close
import fastvideo.envs as envs
from fastvideo.tests.golden_gate._harness import assert_env_fingerprint_matches
MODEL_REPO_ID = "MiniMaxAI/MiniMax-H3"
SEED = 20260805
@@ -98,7 +99,7 @@ def _env_fingerprint() -> dict[str, str]:
"flash_attn": fa_version,
"device_name": torch.cuda.get_device_name(0),
"attention_backend": envs.FASTVIDEO_ATTENTION_BACKEND.get() or "<unset>",
"fastvideo_fa4": str(int(envs.FASTVIDEO_FA4.get())) if envs.FASTVIDEO_FA4.is_set() else "<unset>",
"fastvideo_fa4": str(int(envs.FASTVIDEO_FA4.get())),
"tf32_matmul": str(torch.backends.cuda.matmul.allow_tf32),
"tf32_cudnn": str(torch.backends.cudnn.allow_tf32),
}
@@ -286,12 +287,7 @@ def test_minimax_h3_t2v_golden_gate(env_overrides) -> None:
)
golden = torch.load(golden_path, weights_only=True)
mismatches = {
key: (value, meta["env"].get(key))
for key, value in golden["metadata"]["env"].items() if meta["env"].get(key) != value
}
assert not mismatches, (f"environment changed since golden was minted — re-mint deliberately rather "
f"than chasing bit drift. golden -> current: {mismatches}")
assert_env_fingerprint_matches(golden["metadata"]["env"], meta["env"])
drift = (out - golden["output"]).abs()
print(f"golden-gate drift: max_abs={drift.max().item():.10f} "
fastvideo/tests/golden_gate/test_sd35.py
+2
View File
@@ -16,6 +16,7 @@ tensors only — no RoPE, no mask; sincos pos embed is added before the blocks
from __future__ import annotations
import pytest
import torch
from fastvideo.tests.golden_gate._harness import GateSpec, distributed_runtime, run_gate
@@ -73,5 +74,6 @@ SPEC = GateSpec(
)
@pytest.mark.skip(reason="SD3.5 golden gate is paused while gated-model access is not provisioned for CI.")
def test_sd35_golden_gate(distributed_runtime) -> None:
run_gate(SPEC)
fastvideo/tests/inference/vmoba/test_vmoba_inference.py
+3 -1
View File
@@ -94,4 +94,6 @@ def test_inference_vmoba(env_overrides):
if __name__ == "__main__":
test_inference_vmoba()
import pytest
raise SystemExit(pytest.main([__file__, "-vs"]))
fastvideo/tests/layers/test_triton_fused_norm.py
+20
View File
@@ -12,12 +12,18 @@ import pytest
import torch
from fastvideo.layers.layernorm import ScaleResidualLayerNormScaleShift
import fastvideo.envs as envs
cuda_only = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
B, S, H = 1, 9360, 1536 # the Wan2.1-T2V-1.3B sp2 shapes
@pytest.fixture(autouse=True)
def opt_into_fused_norm(env_overrides):
env_overrides.enter_context(envs.FASTVIDEO_ENABLE_FUSED_NORM.override(True))
def _make_modules(elementwise_affine: bool, seed: int = 0):
"""One eager module and one fused module with identical weights."""
torch.manual_seed(seed)
@@ -175,6 +181,20 @@ def test_env_kill_switch(env_overrides):
assert out.dtype == torch.float32 # eager promotion, not the fused bf16
@cuda_only
def test_fused_norm_requires_explicit_opt_in(env_overrides):
env_overrides.enter_context(envs.FASTVIDEO_ENABLE_FUSED_NORM.override(False))
eager, fused = _make_modules(elementwise_affine=True)
residual, x = _inputs(torch.bfloat16, seed=6)
gate = torch.randn(1, 1, H, device="cuda", dtype=torch.float32)
with torch.no_grad():
expected, expected_residual = eager(residual, x, gate, None, None)
actual, actual_residual = fused(residual, x, gate, None, None)
assert actual.dtype == torch.float32
assert torch.equal(actual, expected)
assert torch.equal(actual_residual, expected_residual)
@cuda_only
def test_mixed_none_still_raises():
_, fused = _make_modules(elementwise_affine=True)
fastvideo/tests/modal/test_pr_test.py
+1 -1
View File
@@ -392,7 +392,7 @@ def test_wave1_lane_functions_use_shared_scripts(monkeypatch):
lanes_dir = Path(__file__).resolve().parents[3] / ".buildkite/scripts/lanes"
for script, payload in {
"kernel_tests.sh": "pytest fastvideo-kernel/tests/ -vs",
"inference_vmoba.sh": "python fastvideo/tests/inference/vmoba/test_vmoba_inference.py",
"inference_vmoba.sh": "exec pytest ./fastvideo/tests/inference/vmoba/test_vmoba_inference.py -vs",
"golden_gate.sh": 'exec pytest "$golden_root" -xvs',
"encoder.sh": "pytest ./fastvideo/tests/encoders -vs",
"vae.sh": "pytest ./fastvideo/tests/vaes -vs",
fastvideo/tests/performance/compare_baseline.py
+1 -3
View File
@@ -591,9 +591,7 @@ def main() -> int:
upload_policy = _normalized_upload_policy()
performance_pytest_failed = _performance_pytest_failed()
# Strict on upload-enabled runs: silent sync failure would make comparison
# and upload state ambiguous.
sync_from_hf(TRACKING_ROOT, strict=persist_tracking)
sync_from_hf(TRACKING_ROOT, strict=persist_tracking or _detect_run_source() == "pr")
current_results = _load_current_results()
if not current_results:
fastvideo/tests/ssim/test_flux2_similarity.py
+2
View File
@@ -87,6 +87,8 @@ def test_flux2_similarity(
attention_backend_name: str,
model_id: str,
) -> None:
if model_id == "black-forest-labs/FLUX.2-klein-9B":
pytest.skip("FLUX.2-klein-9B SSIM is paused while gated-model access is not provisioned for CI.")
_ = run_text_to_latent_similarity_test(
logger=logger,
script_dir=os.path.dirname(os.path.abspath(__file__)),
fastvideo/tests/ssim/test_flux_t2i_similarity.py
+1
View File
@@ -84,6 +84,7 @@ FLUX_FULL_QUALITY_MODEL_TO_PARAMS = {
}
@pytest.mark.skip(reason="FLUX.1-dev SSIM is paused while gated-model access is not provisioned for CI.")
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="FLUX T2I SSIM test requires CUDA",
fastvideo/tests/ssim/test_hunyuan15_i2v_similarity.py
+1
View File
@@ -88,5 +88,6 @@ def test_hunyuan15_i2v_inference_similarity(
model_id=model_id,
default_params_map=HUNYUAN15_I2V_MODEL_TO_PARAMS,
full_quality_params_map=FULL_QUALITY_HUNYUAN15_I2V_MODEL_TO_PARAMS,
init_kwargs_override={"revision": "854c04a4c8a53d990b418c7478f0802c0fc8c726"},
min_acceptable_ssim=0.95,
)
fastvideo/tests/ssim/test_kandinsky6_similarity.py
+1
View File
@@ -91,6 +91,7 @@ def test_kandinsky6_ti2va_inference_similarity(
# examples/inference/basic/basic_kandinsky6_ti2va.py): no FSDP inference, text encoder offloaded
# to CPU between encodes.
init_kwargs_override={
"revision": "7be18aed6fbd64653c0c4872fddfe4a3e3e109f5",
"use_fsdp_inference": False,
"text_encoder_cpu_offload": True,
"pin_cpu_memory": True,
fastvideo/tests/ssim/test_lingbot_fast_similarity.py
+3 -3
View File
@@ -115,10 +115,10 @@ def test_lingbot_fast_i2v_similarity(prompt: str, ATTENTION_BACKEND: str, env_ov
model_id,
ATTENTION_BACKEND,
)
output_video_name = f"{prompt[:100].strip()}.mp4"
os.makedirs(output_dir, exist_ok=True)
init_kwargs = {
"revision": "98583eb0fd66f481f73383428d65d2867c878e51",
"num_gpus": params["num_gpus"],
"use_fsdp_inference": True,
"dit_cpu_offload": True,
@@ -142,12 +142,12 @@ def test_lingbot_fast_i2v_similarity(prompt: str, ATTENTION_BACKEND: str, env_ov
try:
generator = VideoGenerator.from_pretrained(
model_path=params["model_path"], **init_kwargs)
generator.generate_video(prompt, **generation_kwargs)
result = generator.generate_video(prompt, **generation_kwargs)
finally:
if generator is not None:
generator.shutdown()
generated_video_path = os.path.join(output_dir, output_video_name)
generated_video_path = result["video_path"]
assert os.path.exists(generated_video_path), (
f"Output video was not generated at {generated_video_path}")
fastvideo/tests/ssim/test_matrixgame3_similarity.py
+5 -6
View File
@@ -84,6 +84,7 @@ def test_matrixgame3_similarity(prompt, ATTENTION_BACKEND, model_id, env_overrid
and compares the output to reference videos using SSIM.
"""
env_overrides.enter_context(envs.FASTVIDEO_ATTENTION_BACKEND.override(ATTENTION_BACKEND))
env_overrides.enter_context(envs.FASTVIDEO_FA4.override(False))
script_dir = os.path.dirname(os.path.abspath(__file__))
@@ -126,7 +127,7 @@ def test_matrixgame3_similarity(prompt, ATTENTION_BACKEND, model_id, env_overrid
}
generator = VideoGenerator.from_pretrained(model_path=BASE_PARAMS["model_path"], **init_kwargs)
generator.generate_video(prompt, **generation_kwargs)
result = generator.generate_video(prompt, **generation_kwargs)
generator.executor.shutdown()
@@ -156,13 +157,11 @@ def test_matrixgame3_similarity(prompt, ATTENTION_BACKEND, model_id, env_overrid
logger.error(f"Reference video not found for model: {model_id} with backend: {ATTENTION_BACKEND}")
raise FileNotFoundError("Reference video missing")
generated_video_name = _find_mp4(output_dir)
if not generated_video_name:
logger.error(f"Generated video not found for model: {model_id} with backend: {ATTENTION_BACKEND}")
raise FileNotFoundError(f"Generated video missing in {output_dir}")
assert isinstance(result, dict), f"Expected one MatrixGame3 generation result, got {type(result)}"
generated_video_path = result["video_path"]
assert generated_video_path and os.path.isfile(generated_video_path), "Generated MatrixGame3 video missing"
reference_video_path = os.path.join(reference_folder, reference_video_name)
generated_video_path = os.path.join(output_dir, generated_video_name)
logger.info(f"Computing SSIM between {reference_video_path} and {generated_video_path}")
ssim_values = compute_video_ssim_torchvision(reference_video_path, generated_video_path, use_ms_ssim=True)
fastvideo/tests/ssim/test_sd35_similarity.py
+1
View File
@@ -82,6 +82,7 @@ SD35_MODEL_TO_PARAMS = {
}
@pytest.mark.skip(reason="SD3.5 SSIM is paused while gated-model access is not provisioned for CI.")
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="SD3.5 SSIM test requires CUDA",
fastvideo/tests/ssim/test_stable_audio_similarity.py
+4 -1
View File
@@ -106,5 +106,8 @@ def test_stable_audio_inference_similarity(
# FSDP + @torch.inference_mode in StableAudioDenoisingStage hits
# "Inference tensors do not track version counter" on single-GPU
# unshard. SA-1.0 fits on one B200 anyway.
init_kwargs_override={"use_fsdp_inference": False},
init_kwargs_override={
"use_fsdp_inference": False,
"revision": "26609645c363fc81829282fcfe6356340346c1d4",
},
)
fastvideo/tests/training/VSA/gb200_reference_wandb_summary_VSA.json
+21
View File
@@ -0,0 +1,21 @@
{
"avg_step_time": 3.832678238349035,
"grad_norm": 1.9089118242263794,
"step_time": 1.1276781709166244,
"train_loss": 0.3086116909980774,
"vsa_sparsity": 0.05,
"provenance": {
"source_commit": "d5253fd71055927f12aa6c4aa679b93a00238851",
"device": "NVIDIA GB200",
"num_gpus": 2,
"torch": "2.12.0+cu130",
"cuda": "13.0",
"flash_attn": "2.8.3",
"fastvideo_fa4": false,
"model_revision": "0fad780a534b6463e45facd96134c9f345acfa5b",
"dataset_revision": "e97e41d3db696c534d90424f7efea514a63bfde5",
"validation_runs": 2,
"second_run_grad_norm": 1.9349586963653564,
"second_run_train_loss": 0.30886438488960266
}
}
fastvideo/tests/training/VSA/test_training_loss_VSA.py
+3 -2
View File
@@ -16,7 +16,6 @@ from fastvideo.training.runner import main
from fastvideo.utils import build_parser
wandb_name = "test_training_loss_VSA"
reference_wandb_summary_file = "fastvideo/tests/training/VSA/h100_reference_wandb_summary_VSA.json"
NUM_NODES = "1"
NUM_GPUS_PER_NODE = "2"
@@ -83,7 +82,9 @@ def test_distributed_training():
if "H100" not in device_name and "B200" not in device_name:
raise ValueError(f"VSA training regression supports H100 and the GB200 Slurm CI target, got: {device_name}")
reference_wandb_summary = json.load(open(reference_wandb_summary_file))
reference_name = "gb200_reference_wandb_summary_VSA.json" if "B200" in device_name else "h100_reference_wandb_summary_VSA.json"
reference_path = current_file.with_name(reference_name)
reference_wandb_summary = json.load(open(reference_path))
wandb_summary = json.load(open(summary_file))
fields_and_thresholds = {'avg_step_time': 15, 'grad_norm': 0.5, 'step_time': 15, 'train_loss': 0.04}
tests/local_tests/performance/test_compare_baseline_policy.py
+20 -5
View File
@@ -1618,10 +1618,13 @@ def test_v2_identity_lookup_ignores_model_id_and_gpu_display_string(tmp_path):
assert records == [baseline]
def test_main_v2_identity_lookup_ignores_model_id_and_gpu_display_string(
@pytest.mark.parametrize("baseline_latency, expected_rc", [(10.0, 0), (1.0, 1)])
def test_pr_compares_baseline_without_upload_credentials(
monkeypatch,
env_overrides,
tmp_path,
baseline_latency,
expected_rc,
):
tracking_root = tmp_path / "tracking"
results_dir = tmp_path / "results"
@@ -1637,7 +1640,7 @@ def test_main_v2_identity_lookup_ignores_model_id_and_gpu_display_string(
"timestamp": "2026-06-15T00:00:00+00:00",
"success": True,
"baseline_eligible": True,
"latency": 10.0,
"latency": baseline_latency,
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
@@ -1654,17 +1657,29 @@ def test_main_v2_identity_lookup_ignores_model_id_and_gpu_display_string(
monkeypatch.setattr(compare_baseline, "RESULTS_DIR", str(results_dir))
monkeypatch.setattr(compare_baseline, "PERF_REPORTS_DIR", str(reports_dir))
monkeypatch.setattr(compare_baseline, "UPLOAD_POLICY", "never")
monkeypatch.setattr(compare_baseline, "sync_from_hf", lambda local_dir, strict=False: local_dir)
sync_calls = []
def sync(local_dir, strict=False):
sync_calls.append(strict)
return local_dir
def upload(*args, **kwargs):
pytest.fail("PR comparisons must not require upload credentials")
monkeypatch.setattr(compare_baseline, "sync_from_hf", sync)
monkeypatch.setattr(compare_baseline, "upload_record", upload)
env_overrides.enter_context(envs.override_external("PERF_PYTEST_RC", None))
assert compare_baseline.main() == 0
assert compare_baseline.main() == expected_rc
assert sync_calls == [True]
artifacts = list((reports_dir / "results").glob("normalized_perf_*.json"))
assert len(artifacts) == 1
with open(artifacts[0], encoding="utf-8") as f:
normalized = json.load(f)
assert normalized["comparison_status"] == compare_baseline.STATUS_PASS
expected_status = compare_baseline.STATUS_PASS if expected_rc == 0 else compare_baseline.STATUS_REGRESSION
assert normalized["comparison_status"] == expected_status
assert normalized["baseline_status"] == "compared"
tests/local_tests/test_cookbook_serving.py
-27
View File
@@ -4,7 +4,6 @@ import ast
import copy
import json
from pathlib import Path
from html.parser import HTMLParser
import pytest
import yaml
@@ -233,32 +232,6 @@ def test_wan21_i2v_stays_python_only():
assert "serving" not in recipe
def test_h3_command_blocks_have_unique_copy_targets():
class CodeBlocks(HTMLParser):
def __init__(self):
super().__init__()
self.ids = []
def handle_starttag(self, tag, attrs):
if tag == "pre":
block_id = dict(attrs).get("id")
if block_id:
self.ids.append(block_id)
parser = CodeBlocks()
parser.feed((ROOT / "docs/cookbook/minimax-h3.md").read_text())
assert parser.ids == [
"cookbook-local-command",
"cookbook-server-install",
"cookbook-server-prepare",
"cookbook-server-command",
"cookbook-health-command",
"cookbook-client-install",
"cookbook-client-code",
]
assert len(set(parser.ids)) == len(parser.ids)
def test_kandinsky6_cookbook_recipes_match_maintained_examples():
recipes = json.loads(COOKBOOK_DATA.read_text())["recipes"]
family = [recipe for recipe in recipes if recipe["family"] == "kandinsky6"]