Compare commits

..
131 changed files with 1129 additions and 7961 deletions
@@ -69,7 +69,7 @@ approval, then upload reviewed accepted baseline records.
| `model_id` | Yes | Benchmark id, e.g. `wan-t2v-1.3b-2gpu`. This maps to the HF subdirectory after `sanitize(model_id)`. |
| `gpu_type` | Yes | Exact GPU device string from the performance record, e.g. the L40S device name emitted by CI. Baselines are GPU-specific. |
| `source_results` | Yes | One or more local paths or Buildkite artifact URLs for accepted shifted performance JSONs. Prefer normalized `normalized_perf_*.json` artifacts emitted by `compare_baseline.py`. Accept `source_result` as an alias only for a single JSON. |
| `max_intra_batch_regression` | No | Maximum allowed regression of any source JSON against the source batch median. Default: `0.05` (5%). |
| `max_intra_batch_regression` | No | Maximum allowed regression of any source JSON against the source batch median. Default: `PERF_MAX_REGRESSION` if set, otherwise `0.05` (5%). |
| `intent_rationale` | Yes | One-line explanation for why the baseline shift is legitimate. This is written into provenance and should be reused in the PR. |
Hardcoded defaults:
@@ -148,7 +148,8 @@ For each metric with at least two non-null source values:
4. Stop if any source record regresses against the batch median by more than
`max_intra_batch_regression`.
Default `max_intra_batch_regression` to `0.05`. Print a table with per-source values, batch median, and
Default `max_intra_batch_regression` to `PERF_MAX_REGRESSION` when set,
otherwise `0.05`. Print a table with per-source values, batch median, and
worst intra-batch regression.
This check prevents uploading a mixed batch where one JSON is materially
@@ -182,7 +183,7 @@ present, that run is not a valid source for baseline reseeding.
### 2. Sync and back up existing HF records under /tmp
Use `fastvideo/performance/hf_store.py` helpers directly. Do **not** use
Use `fastvideo/tests/performance/hf_store.py` helpers directly. Do **not** use
`compare_baseline.py` as a sync shortcut; on full main runs it can persist
records, while this step must only fetch and back up existing history.
@@ -191,7 +192,7 @@ The sync command pattern is:
```bash
export PERFORMANCE_TRACKING_ROOT="${PERFORMANCE_TRACKING_ROOT:-/tmp/perf-tracking}"
export HF_REPO_ID="${HF_REPO_ID:-FastVideo/performance-tracking}"
python -c 'from fastvideo.performance.hf_store import sync_from_hf; import os; sync_from_hf(os.environ["PERFORMANCE_TRACKING_ROOT"], strict=True)'
PYTHONPATH=fastvideo/tests/performance python -c 'from hf_store import sync_from_hf; import os; sync_from_hf(os.environ["PERFORMANCE_TRACKING_ROOT"], strict=True)'
```
Then back up only the sanitized model directory under `/tmp`:
@@ -199,8 +200,8 @@ Then back up only the sanitized model directory under `/tmp`:
```bash
SHORT_COMMIT=$(git rev-parse --short=12 HEAD)
TIMESTAMP=$(date -u +%Y%m%d_%H%M%S)
MODEL_SAFE=$(python - <<'PY'
from fastvideo.performance.hf_store import sanitize
MODEL_SAFE=$(PYTHONPATH=fastvideo/tests/performance python - <<'PY'
from hf_store import sanitize
print(sanitize("<model_id>"))
PY
)
@@ -234,7 +235,7 @@ first baseline seed. Continue, but report that baseline history was empty.
Load the last 5 successful records for the target:
```python
from fastvideo.performance.hf_store import load_records_for_model
from hf_store import load_records_for_model
records = load_records_for_model(
"/tmp/perf-tracking",
@@ -371,7 +372,7 @@ prepared records plus backup on disk.
Use the shared storage helper so the path and repo type match CI:
```python
from fastvideo.performance.hf_store import upload_record
from hf_store import upload_record
upload_record("<local_record_path>", record, strict=True)
```
@@ -459,7 +460,7 @@ directories created for this reseed. Never remove unrelated `/tmp` contents.
intentional baseline replacement.
- `fastvideo/tests/performance/compare_baseline.py` — normalization, rolling
median comparison, and persistence rules.
- `fastvideo/performance/hf_store.py` — HF sync, record loading,
- `fastvideo/tests/performance/hf_store.py` — HF sync, record loading,
`sanitize()`, and `upload_record()`.
- `fastvideo/tests/performance/test_inference_performance.py` — source result
JSON schema.
@@ -1,9 +1,9 @@
{
"benchmark_id": "wan-t2v-1.3b-2gpu",
"config_schema_version": 2,
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"workload_id": "wan-t2v-1.3b",
"variant_id": "canonical",
"benchmark_version": 1,
"description": "Wan2.1 T2V 1.3B inference performance",
"model": {
"model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
+2 -19
View File
@@ -1,11 +1,7 @@
name: pre-commit
on:
# pull_request_target instead of pull_request: the workflow definition and
# the hook config are always taken from the BASE branch, so fork /
# first-time-contributor PRs run immediately without a maintainer clicking
# "Approve and run". The PR head is checked out as data only.
pull_request_target:
pull_request:
branches: [main]
workflow_call:
inputs:
@@ -19,25 +15,12 @@ permissions:
jobs:
pre-commit:
if: github.event.pull_request.draft != true
if: github.event_name == 'workflow_call' || github.event.pull_request.draft != true
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || '' }}
# For PR events, lint the PR head — but keep the hook definitions from
# the base branch so an untrusted PR cannot alter what gets executed.
- name: Save trusted hook config
if: github.event_name == 'pull_request_target'
run: cp .pre-commit-config.yaml "$RUNNER_TEMP/trusted-pre-commit-config.yaml"
- uses: actions/checkout@v4
if: github.event_name == 'pull_request_target'
with:
ref: ${{ github.event.pull_request.head.sha }}
persist-credentials: false
- name: Restore trusted hook config
if: github.event_name == 'pull_request_target'
run: cp "$RUNNER_TEMP/trusted-pre-commit-config.yaml" .pre-commit-config.yaml
- uses: actions/setup-python@v5
with:
python-version: "3.12"
+3 -8
View File
@@ -12,17 +12,13 @@ Defaults:
- `HF_REPO_ID=FastVideo/performance-tracking`
- `PERFORMANCE_TRACKING_ROOT=/tmp/fastvideo-perf-dashboard`
- `PERF_MAX_REGRESSION=0.05`
Records can include source metadata and rolling-baseline policy context:
Records can include source metadata:
- `run_source`: `pr`, `local`, `scheduled_main`, or `unknown`
- `baseline_eligible`: only successful scheduled-main records should be true
- Buildkite metadata such as branch, PR number, build URL, build ID, and job ID
- `regression_thresholds`: per-metric rolling-baseline percent and absolute
floors used for recomputed status context
Dashboard/API metric payloads expose `threshold_exceeded` for raw threshold
crossings; `regressed` remains the gated CI-failure signal.
Set one of `HF_API_KEY`, `HUGGINGFACE_HUB_TOKEN`, or `HF_TOKEN` if the
configured dataset repo requires authenticated access:
@@ -93,8 +89,7 @@ Trend charts show metric-specific axes and exact point details on hover/focus:
- PR number, branch, and Buildkite URL when present
The latest status table uses the stored JSON `success` value. Recomputed
baseline context applies each metric's percent and absolute regression floors
and does not override stored status.
baseline context is shown separately and does not override stored status.
## API
@@ -1,7 +1,6 @@
import { useEffect, useMemo, useState } from "react";
import { fetchSummary, fetchTrends, refreshData } from "./api";
import type { CohortValue, RunSource, SummaryResponse, TrendGroup, TrendPoint } from "./api";
import { fetchSummary, fetchTrends, refreshData, RunSource, SummaryResponse, TrendGroup, TrendPoint } from "./api";
const METRIC_KEYS = ["latency", "throughput", "memory", "text_encoder_time_s", "dit_time_s", "vae_decode_time_s"];
const RUN_SOURCES: Array<{ value: "" | RunSource; label: string }> = [
@@ -110,61 +109,6 @@ function metricLabel(metricKey: string) {
return METRIC_DEFINITIONS[metricKey]?.label ?? metricKey;
}
type CohortFields = {
model_id: string;
gpu_type: string;
workload_id: CohortValue;
variant_id: CohortValue;
benchmark_version: CohortValue;
recipe_fingerprint: CohortValue;
hardware_profile_id: CohortValue;
software_profile_id: CohortValue;
};
function cohortValue(value: CohortValue) {
if (value === null || value === undefined || value === "") {
return "legacy";
}
return String(value);
}
function shortCohortValue(value: CohortValue) {
const text = cohortValue(value);
if (text === "legacy" || text.length <= 14) {
return text;
}
return text.slice(0, 12);
}
function cohortKey(cohort: CohortFields) {
return [
cohort.model_id,
cohort.gpu_type,
cohortValue(cohort.workload_id),
cohortValue(cohort.variant_id),
cohortValue(cohort.benchmark_version),
cohortValue(cohort.recipe_fingerprint),
cohortValue(cohort.hardware_profile_id),
cohortValue(cohort.software_profile_id)
].join("|");
}
function cohortTitle(cohort: CohortFields) {
const workload = cohortValue(cohort.workload_id);
const variant = cohortValue(cohort.variant_id);
const version = cohortValue(cohort.benchmark_version);
const versionLabel = version === "legacy" ? version : `v${version}`;
return `${workload} / ${variant} / ${versionLabel}`;
}
function cohortDetail(cohort: CohortFields) {
return [
`recipe ${shortCohortValue(cohort.recipe_fingerprint)}`,
shortCohortValue(cohort.hardware_profile_id),
shortCohortValue(cohort.software_profile_id)
].join(" | ");
}
function formatMetricValue(metricKey: string, value: number | null | undefined, tooltip = false) {
const definition = METRIC_DEFINITIONS[metricKey];
if (!definition) {
@@ -227,9 +171,7 @@ function TrendChart({ group, metricKey }: { group: TrendGroup; metricKey: string
top: `${(activePoint.y / height) * 100}%`
}
: undefined;
const ariaLabel = `${metricLabel(metricKey)} trend for ${group.model_id} on ${group.gpu_type}, ${cohortTitle(
group
)}`;
const ariaLabel = `${metricLabel(metricKey)} trend for ${group.model_id} on ${group.gpu_type}`;
return (
<div className="chart-shell">
@@ -477,7 +419,7 @@ export default function App() {
<section className="panel">
<div className="panel-header">
<h2>Latest Status</h2>
<span>{latestRows.length} comparison cohorts</span>
<span>{latestRows.length} model/GPU groups</span>
</div>
{latestRows.length === 0 ? (
<div className="empty">No records match the selected filters.</div>
@@ -490,7 +432,6 @@ export default function App() {
<th>Recomputed</th>
<th>Model</th>
<th>GPU</th>
<th>Cohort</th>
<th>Commit</th>
<th>Source</th>
<th>Baseline</th>
@@ -499,13 +440,11 @@ export default function App() {
<th>Throughput</th>
<th>Memory</th>
<th>Worst</th>
<th>Exceeded</th>
<th>Failing</th>
</tr>
</thead>
<tbody>
{latestRows.map((row) => (
<tr key={cohortKey(row)}>
<tr key={`${row.model_id}-${row.gpu_type}`}>
<td>
<span className={`badge ${row.status}`}>{row.status}</span>
</td>
@@ -516,12 +455,6 @@ export default function App() {
</td>
<td>{row.model_id}</td>
<td>{row.gpu_type}</td>
<td>
<div className="cohort-cell">
<strong>{cohortTitle(row)}</strong>
<span>{cohortDetail(row)}</span>
</div>
</td>
<td>{shortSha(row.commit_sha)}</td>
<td>
<span className={`source-badge source-${row.run_source}`}>{runSourceLabel(row.run_source)}</span>
@@ -532,12 +465,6 @@ export default function App() {
<td>{formatNumber(row.metrics.throughput?.current, 3)}</td>
<td>{formatNumber(row.metrics.memory?.current, 1)}</td>
<td>{formatNumber(row.worst_regression_pct, 1)}%</td>
<td>
{row.threshold_exceeded_metrics.length
? row.threshold_exceeded_metrics.join(", ")
: "none"}
</td>
<td>{row.failing_metrics.length ? row.failing_metrics.join(", ") : "none"}</td>
</tr>
))}
</tbody>
@@ -560,13 +487,11 @@ export default function App() {
) : (
trends.map((group) =>
METRIC_KEYS.map((metricKey) => (
<article className="trend-card" key={`${cohortKey(group)}-${metricKey}`}>
<article className="trend-card" key={`${group.model_id}-${group.gpu_type}-${metricKey}`}>
<div>
<h3>{metricLabel(metricKey)}</h3>
<p>
{group.model_id} | {group.gpu_type}
<span>{cohortTitle(group)}</span>
<span>{cohortDetail(group)}</span>
</p>
</div>
<TrendChart group={group} metricKey={metricKey} />
+4 -22
View File
@@ -2,28 +2,11 @@ export type MetricValue = {
current: number | null;
baseline: number | null;
regression_pct: number | null;
absolute_delta: number | null;
threshold_percent: number;
threshold_absolute: number;
gated: boolean;
threshold_exceeded: boolean;
regressed: boolean;
label: string;
lower_is_better: boolean;
precision: number;
};
export type CohortValue = string | number | null;
export type ComparisonCohort = {
workload_id: CohortValue;
variant_id: CohortValue;
benchmark_version: CohortValue;
recipe_fingerprint: CohortValue;
hardware_profile_id: CohortValue;
software_profile_id: CohortValue;
};
export type SummaryRow = {
model_id: string;
gpu_type: string;
@@ -32,8 +15,7 @@ export type SummaryRow = {
success: boolean;
baseline_n: number;
worst_regression_pct: number | null;
threshold_exceeded_metrics: string[];
failing_metrics: string[];
regression_threshold_pct: number;
computed_regression_status: "pass" | "fail";
status: "pass" | "fail";
run_source: RunSource;
@@ -45,7 +27,7 @@ export type SummaryRow = {
build_id: string;
job_id: string;
metrics: Record<string, MetricValue>;
} & ComparisonCohort;
};
export type RunSource = "pr" | "local" | "scheduled_main" | "unknown";
@@ -79,13 +61,13 @@ export type TrendPoint = {
build_id: string;
job_id: string;
metrics: Record<string, number | null>;
} & ComparisonCohort;
};
export type TrendGroup = {
model_id: string;
gpu_type: string;
points: TrendPoint[];
} & ComparisonCohort;
};
export type TrendsResponse = {
groups: TrendGroup[];
@@ -149,11 +149,6 @@ h3 {
font-size: 0.82rem;
}
.trend-card p {
display: grid;
gap: 2px;
}
.stat strong {
display: block;
margin-top: 8px;
@@ -191,7 +186,7 @@ h3 {
table {
width: 100%;
min-width: 1260px;
min-width: 1120px;
border-collapse: collapse;
}
@@ -214,25 +209,6 @@ td {
font-size: 0.9rem;
}
.cohort-cell {
display: grid;
gap: 2px;
}
.cohort-cell strong,
.trend-card p span {
color: #1b2836;
font-size: 0.78rem;
font-weight: 700;
}
.cohort-cell span,
.trend-card p span + span {
color: #607080;
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace;
font-size: 0.72rem;
}
.badge {
display: inline-flex;
align-items: center;
+2 -2
View File
@@ -68,7 +68,7 @@ ARG FLASH_ATTN_WHEEL_RELEASE_ARM64=https://github.com/mjun0812/flash-attention-p
# cutlass-4.4 `cute.core.ThrMma` API, which crashes on the cutlass-dsl 4.5 that
# flashinfer/quack pull in. After the wheel install we overlay this cutlass-4.5-safe
# upstream cute (flash-attn-4) so the image runs FA4 instead of the FA2 fallback.
ARG FA4_CUTE_REF=82d6441eec5d4dfec120153db2c0145ae855a083
ARG FA4_CUTE_REF=940cd9680f3315f2f06b43ab5bea2c2cf2d96806
# Provided automatically by BuildKit/buildx (e.g. "amd64" / "arm64") and used to
# select the prebuilt flash-attn wheel. Empty under a plain `docker build` without
@@ -161,7 +161,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \
uv pip install flash-attn==${FLASH_ATTN_VERSION} --no-build-isolation; \
fi
# Overlay the CuTe-DSL-4.6-compatible upstream FA4 cute (FA4_CUTE_REF) over the
# Overlay the cutlass-4.5-safe upstream FA4 cute (FA4_CUTE_REF) over the
# wheel/source one so the image runs FA4, not the FA2 fallback. This pulls the FA4
# runtime stack (cutlass-dsl, quack-kernels, apache-tvm-ffi, torch-c-dlpack-ext) --
# the same deps the [dreamverse] extra already installs in CI; the installed torch
+61 -197
View File
@@ -72,18 +72,14 @@ fastvideo/tests/performance/
│ writes Markdown summary + (optionally) uploads new records
├── dashboard.py
│ └── builds time-series Plotly HTML from HF history
fastvideo/performance/
├── hf_store.py # shared HF I/O + DataFrame helpers
└── metric_policy.py # shared rolling-baseline threshold policy
└── hf_store.py # shared HF I/O + DataFrame helpers
```
The HF dataset (`FastVideo/performance-tracking` by default) holds one
normalized JSON per run. For v2 records, the rolling baseline is the median of
the last 5 successful, baseline-eligible records in the same comparison cohort:
`model_id`, `gpu_type`, `workload_id`, `variant_id`, `benchmark_version`,
`recipe_fingerprint`, `hardware_profile_id`, and `software_profile_id`. PR and
local records are visible in the dashboard but are not baseline eligible.
normalized JSON per `(model_id, gpu_type, run)` tuple. The rolling baseline is
the median of the last 5 successful, baseline-eligible records for that
model+GPU. PR and local records are visible in the dashboard but are not
baseline eligible.
## Planned Coverage
@@ -96,28 +92,25 @@ and recipe changes instead of treating all records for a model as equivalent.
## Metrics
Each benchmark records six metrics. The rolling-baseline comparator also has a
per-metric policy with direction, percent threshold, absolute threshold, and a
`gated` flag.
Each benchmark records six metrics:
| Metric | Raw key | Normalized key | Direction | Default rolling policy |
|---|---|---|---|---|
| End-to-end generation latency | `avg_generation_time_s` | `latency` | Lower is better | 8% and 0.5 s |
| Video throughput | `throughput_fps` | `throughput` | Higher is better | 8% and 0.05 FPS |
| Peak GPU memory | `max_peak_memory_mb` | `memory` | Lower is better | 5% and 256 MB |
| Text encoder time | `text_encoder_time_s` | `text_encoder_time_s` | Lower is better | 5% and 0.25 s |
| DiT denoising time | `dit_time_s` | `dit_time_s` | Lower is better | 5% and 0.25 s |
| VAE decode time | `vae_decode_time_s` | `vae_decode_time_s` | Lower is better | 5% and 0.25 s |
| Metric | Raw key | Normalized key | Direction |
|---|---|---|---|
| End-to-end generation latency | `avg_generation_time_s` | `latency` | Lower is better |
| Video throughput | `throughput_fps` | `throughput` | Higher is better |
| Peak GPU memory | `max_peak_memory_mb` | `memory` | Lower is better |
| Text encoder time | `text_encoder_time_s` | `text_encoder_time_s` | Lower is better |
| DiT denoising time | `dit_time_s` | `dit_time_s` | Lower is better |
| VAE decode time | `vae_decode_time_s` | `vae_decode_time_s` | Lower is better |
`test_inference_performance.py` temporarily sets `FASTVIDEO_STAGE_LOGGING=1`
while it runs so pipeline stage execution times are available in
`generate_video(...).logging_info`. Stage logs use pipeline-unique keys such as
`prompt_encoding_stage` so duplicate stage classes do not collide. For
`PipelineStage` entries, shared component stage bases emit a stable
`component_metric`: text encoding stages map to `text_encoder_time_s`,
denoising stages and subclasses map to `dit_time_s`, and decoding stages map to
`vae_decode_time_s`. The extractor falls back to known `stage_class` names for
older logs that do not include `component_metric` or that used the class name as
`PipelineStage` entries, the extractor maps the `stage_class` field:
`TextEncodingStage` maps to `text_encoder_time_s`, `DenoisingStage` and
`DmdDenoisingStage` map to `dit_time_s`, and `DecodingStage` maps to
`vae_decode_time_s`, with a fallback for older logs that used the class name as
the stage key. Generator-side timings such as `PostDecodeFrameProcessStage`,
`VideoSaveStage`, and `AudioMuxStage` are intentionally ignored. If a pipeline
does not report one of the mapped stages, that component metric is stored as
@@ -159,29 +152,13 @@ unrealistic memory growth, and optionally large component-specific slowdowns
even when the rolling baseline is empty. They are hand-set with generous
headroom and almost never need touching.
### Rolling baseline (per comparison cohort)
### Rolling baseline (per `(model_id, gpu_type)`)
`compare_baseline.py` loads the last 5 successful, baseline-eligible records
for the same comparison cohort from the HF dataset, computes the median for
each available metric, and evaluates the current run with the metric's
rolling regression policy. For v2 records, that cohort is `model_id`,
`gpu_type`, `workload_id`, `variant_id`, `benchmark_version`,
`recipe_fingerprint`, `hardware_profile_id`, and `software_profile_id`. For
latency, memory, and component times, higher values are regressions. For
throughput, lower values are regressions.
A metric exceeds its rolling threshold when both of these are true:
```text
percent_delta > threshold_percent
absolute_delta > threshold_absolute
```
Gated metrics fail CI when that threshold crossing happens. Set `gated: false`
for metrics that should remain visible in reports and the dashboard without
failing CI. Dashboard/API payloads expose `threshold_exceeded` separately from
`regressed`, where `regressed` means a gated CI failure. Missing or `null`
metrics are skipped.
for the same `(model_id, gpu_type)` from the HF dataset, computes the median
for each available metric, and fails if the current run regresses by more than
`PERF_MAX_REGRESSION` (default 5%). For latency, memory, and component times,
higher values are regressions. For throughput, lower values are regressions.
This is the **drift detector** — it catches sub-threshold regressions that
slowly add up. Only scheduled-main successful records are baseline eligible.
@@ -205,9 +182,9 @@ configs and remain loadable. New or migrated configs should use
{
"benchmark_id": "wan-t2v-1.3b-2gpu",
"config_schema_version": 2,
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2
"workload_id": "wan-t2v-1.3b",
"variant_id": "canonical",
"benchmark_version": 1
}
```
@@ -218,27 +195,21 @@ metadata that make the measured workload explicit:
| Field | Purpose |
|---|---|
| `workload_id` | Stable benchmark family, such as `wan-t2v`. |
| `variant_id` | Intentional recipe family, including model size and parallelism config, such as `1.3b-sp2`. |
| `workload_id` | Stable benchmark family, such as `wan-t2v-1.3b`. |
| `variant_id` | Intentional recipe family, such as `canonical`. |
| `benchmark_version` | Version of the measurement protocol and comparison policy. |
If a config declares `config_schema_version: 2`, loading fails clearly when any
required v2 identity field is missing. If v2 identity or metadata fields are
added without `config_schema_version: 2`, loading also fails so partial
migrations do not silently run as v1 configs. Optional v2 metadata fields
reserved for follow-up work, such as `metric_threshold_policy` and
`quality_metadata`, must be JSON objects when present. (`recipe` is emitted
by the harness and is not config-declarable.)
reserved for follow-up work, such as `recipe`, `metric_threshold_policy`, and
`quality_metadata`, must be JSON objects when present.
Recipe fingerprinting, hardware/software profile IDs, exact-identity
comparison, and dashboard cohort grouping land with this change: v2 records
compare only within their identity cohort, and a record that opens a NEW
cohort is marked `baseline_status: "initialized_new_cohort"` (regression
gating starts once that cohort accumulates history). Legacy v1 configs still
run and are normalized for reporting, but their records skip rolling-baseline
comparison entirely (`baseline_status: "skipped_missing_identity"`, never
baseline eligible); only static thresholds gate them. Metric-specific
threshold policies and promoted baselines remain separate follow-ups.
comparison, metric-specific threshold policy behavior, promoted baselines, and
dashboard regrouping are separate follow-up changes. Until those land, rolling
baseline comparison remains keyed by `(model_id, gpu_type)`.
### Raw record (`results/perf_*.json`)
@@ -247,10 +218,10 @@ Written by `test_inference_performance.py`. One file per benchmark run.
```jsonc
{
"benchmark_id": "wan-t2v-1.3b-2gpu",
"result_schema_version": 2,
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"config_schema_version": 2,
"workload_id": "wan-t2v-1.3b",
"variant_id": "canonical",
"benchmark_version": 1,
"model_short_name": "Wan2.1-T2V-1.3B-Diffusers",
"device": "NVIDIA L40S",
"num_gpus": 2,
@@ -268,62 +239,12 @@ Written by `test_inference_performance.py`. One file per benchmark run.
"max_dit_time_s": 10.0,
"max_vae_decode_time_s": 10.0
},
"regression_thresholds": {
"latency": {
"threshold_percent": 0.10,
"threshold_absolute": 1.0,
"gated": true
}
},
"commit": "<full sha>",
"run_source": "pr",
"branch": "feature/perf-change",
"pr_number": "1234",
"test_scope": "direct",
"build_url": "https://buildkite.example/build",
"build_id": "<buildkite-build-id>",
"job_id": "<buildkite-job-id>",
"timestamp": "2026-05-08T22:00:00+00:00",
"quality_metadata": { "quality_status": "canonical" },
"text_encoder_time_s": 2.141,
"dit_time_s": 8.437,
"vae_decode_time_s": 3.208,
"recipe": {
"recipe_schema_version": 1,
"benchmark": {
"benchmark_id": "wan-t2v-1.3b-2gpu",
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2
},
"model": { "model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" },
"init_kwargs": { "num_gpus": 2, "sp_size": 2, "tp_size": 1 },
"generation_kwargs": { "height": 480, "width": 832, "num_frames": 45 },
"inputs": { "prompt_count": 1, "prompt_sha256": ["<measured-prompt-sha256>"] },
"attention": { "requested_backend": "FLASH_ATTN", "resolved_backend": "FLASH_ATTN" }
},
"recipe_fingerprint": "<sha256>",
"hardware_profile": {
"device_type": "cuda",
"gpu_count": 2,
"gpus": [{ "name": "NVIDIA L40S", "memory_gb": 48, "compute_capability": "8.9" }],
"interconnect": "none_or_partial"
},
"hardware_profile_id": "hw-<sha256-prefix>",
"software_profile": {
"python": "3.12",
"pytorch": "2.12",
"cuda": "13.0",
"packages": {
"fastvideo_kernel": "0.3.2",
"flashinfer": "0.2.11",
"nvidia_cutlass_dsl": "4.5.0",
"triton": "3.4.1"
}
},
"software_profile_id": "sw-<sha256-prefix>",
"environment_metadata": { "env": { "IMAGE_VERSION": "py3.12-cuda13.0.0" } },
"environment_fingerprint": "env-<sha256-prefix>"
"vae_decode_time_s": 3.208
}
```
@@ -335,10 +256,6 @@ result, used as the rolling-baseline source of truth.
```jsonc
{
"model_id": "wan-t2v-1.3b-2gpu",
"result_schema_version": 2,
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"timestamp": "2026-05-08T22:00:00+00:00",
"commit_sha": "<full sha>",
"gpu_type": "NVIDIA L40S",
@@ -348,69 +265,34 @@ result, used as the rolling-baseline source of truth.
"text_encoder_time_s": 2.141,
"dit_time_s": 8.437,
"vae_decode_time_s": 3.208,
"regression_thresholds": {
"latency": {
"threshold_percent": 0.08,
"threshold_absolute": 0.5,
"gated": true
}
},
"recipe_fingerprint": "<sha256>",
"hardware_profile_id": "hw-<sha256-prefix>",
"software_profile_id": "sw-<sha256-prefix>",
"environment_fingerprint": "env-<sha256-prefix>",
"run_source": "pr",
"branch": "feature/perf-change",
"pr_number": "1234",
"test_scope": "direct",
"build_url": "https://buildkite.example/build",
"build_id": "<buildkite-build-id>",
"job_id": "<buildkite-job-id>",
"quality_metadata": { "quality_status": "canonical" },
"success": true
}
```
### Compatibility with legacy records
Older records in the HF dataset may not have `result_schema_version`,
component timing fields, or v2 identity/profile fields. Records without
`result_schema_version` are treated as v1. The comparator ignores missing or
`null` metrics when computing a median, and the dashboard lists skipped plots
for metric series that have no non-null values. Records missing both
`run_source` and `baseline_eligible` are treated as legacy successful
main/full-suite uploads and remain eligible for rolling baselines.
Current `perf_*.json` artifacts that lack the v2 comparison identity are
normalized for reporting but skip rolling-baseline comparison and are not marked
baseline eligible.
New records compare only against the same `model_id`, `gpu_type`,
`workload_id`, `variant_id`, `benchmark_version`, `recipe_fingerprint`,
`hardware_profile_id`, and `software_profile_id` cohort.
`environment_metadata` and `environment_fingerprint` are audit data and are not
part of the comparison key.
The recipe prompt digests describe the prompts actually measured by the
benchmark run; extra configured prompts are ignored unless the benchmark runner
executes them.
Software profile package cohorts keep exact versions for relevant
attention/kernel packages, including FastVideo kernels, FlashAttention,
FlashInfer, Cutlass DSL, SageAttention, Triton, and xFormers when installed.
Older records in the HF dataset may not have component timing fields. The
comparator ignores missing or `null` metrics when computing a median, and the
dashboard lists skipped plots for metric series that have no non-null values.
Records missing both `run_source` and `baseline_eligible` are treated as legacy
successful main/full-suite uploads and remain eligible for rolling baselines.
## Environment variable reference
| Variable | Default | Used by | Purpose |
|---|---|---|---|
| `PERF_MAX_REGRESSION` | `0.05` | `compare_baseline.py` | Per-metric regression fraction that fails the build. |
| `PERFORMANCE_TRACKING_ROOT` | `/tmp/perf-tracking` | `compare_baseline.py`, `dashboard.py` | Local directory the HF dataset is synced to. |
| `PERF_REPORTS_DIR` | `/root/data/perf_reports` | `compare_baseline.py`, `dashboard.py` | Where the Markdown summary and Plotly HTML get written for Buildkite to pick up. |
| `HF_REPO_ID` | `FastVideo/performance-tracking` | `fastvideo/performance/hf_store.py` | HF dataset repo holding rolling-baseline records. |
| `HF_API_KEY`, `HUGGINGFACE_HUB_TOKEN`, `HF_TOKEN` | unset | `fastvideo/performance/hf_store.py` | Required for upload or private dataset reads. |
| `PERF_RUN_SOURCE` | inferred | `compare_baseline.py`, `test_inference_performance.py` | Source metadata for uploaded records: `pr`, `local`, `scheduled_main`, or `unknown`. |
| `HF_REPO_ID` | `FastVideo/performance-tracking` | `hf_store.py` | HF dataset repo holding rolling-baseline records. |
| `HF_API_KEY`, `HUGGINGFACE_HUB_TOKEN`, `HF_TOKEN` | unset | `hf_store.py` | Required for upload or private dataset reads. |
| `PERF_RUN_SOURCE` | inferred | `compare_baseline.py` | Source metadata for uploaded records: `pr`, `local`, `scheduled_main`, or `unknown`. |
| `PERF_UPLOAD_POLICY` | `never` | `compare_baseline.py` | Upload policy: `never`, `pass`, or `always`. |
| `PERF_PYTEST_RC` | unset | `compare_baseline.py` | Static-threshold pytest exit code, used so scheduled-main failures can be uploaded with `success=false`. |
| `TEST_SCOPE` | unset | `compare_baseline.py` | CI context used to infer scheduled-main runs together with `BUILDKITE_BRANCH=main`. |
| `BUILDKITE_BRANCH`, `BUILDKITE_COMMIT`, `BUILDKITE_PULL_REQUEST` | unset | `compare_baseline.py`, `test_inference_performance.py` | CI metadata stamped into records. |
| `DASHBOARD_DAYS` | `30` | `dashboard.py` | Lookback window for the Plotly trend pages. |
| `PERFORMANCE_TRACKING_SYNC_REUSE_TTL_SECONDS` | `3600` | `fastvideo/performance/hf_store.py` | Freshness window for reusing an existing HF sync when requested by dashboard consumers. |
| `PERFORMANCE_TRACKING_SYNC_REUSE_TTL_SECONDS` | `3600` | `hf_store.py` | Freshness window for reusing an existing HF sync when requested by dashboard consumers. |
| `FASTVIDEO_STAGE_LOGGING` | set by the pytest test | `test_inference_performance.py` | Enables pipeline stage timing capture for component metrics during benchmark runs. |
## CI integration
@@ -421,22 +303,17 @@ point is `fastvideo/tests/modal/pr_test.py:run_performance_tests` and the
Buildkite artifact upload is in
`.buildkite/scripts/pr_test.sh:upload_performance_artifacts`.
Each performance build runs pytest first. PR and direct runs only continue to
`compare_baseline.py` when that fixed-threshold phase passes; if pytest fails,
Markdown summaries and normalized JSON artifacts are not emitted. Scheduled
main runs set `PERF_UPLOAD_POLICY=always`, so they still run
`compare_baseline.py` (with `PERF_PYTEST_RC` set) after a fixed-threshold
failure. Those failed scheduled main runs emit summaries and normalized
records, upload records with `success=false`, and are excluded from future
rolling baselines. The dashboard still runs best-effort for observability.
When the rolling-baseline phase runs, it emits:
Each performance build runs pytest first. If that fixed-threshold phase fails,
`compare_baseline.py` is skipped, so Markdown summaries and normalized JSON
artifacts are not emitted. The dashboard still runs best-effort for
observability. When pytest passes, the rolling-baseline phase emits:
* **Markdown summary** — appended to `$GITHUB_STEP_SUMMARY` when that variable
is set, and written as `perf_<sha>_<ts>.md` for Buildkite upload. Contains a
per-benchmark row with current vs. baseline values for latency, throughput,
memory, text encoder time, DiT time, and VAE decode time.
* **Plotly dashboard** — `dashboard_<sha>_<ts>.html` showing time-series for
each metric grouped by comparison cohort.
each metric grouped by `(model_id, gpu_type)`.
* **Normalized records** — `normalized_perf_*.json`, one per benchmark.
Useful as input to the
[`reseed-performance-baseline`](https://github.com/hao-ai-lab/FastVideo/blob/main/.agents/skills/reseed-performance-baseline/SKILL.md)
@@ -453,7 +330,7 @@ When the rolling-baseline phase runs, it emits:
"benchmark_id": "<unique-id>",
"config_schema_version": 2,
"workload_id": "<stable-workload-id>",
"variant_id": "<variant, e.g. 1.3b-sp2>",
"variant_id": "canonical",
"benchmark_version": 1,
"model": { "model_path": "...", "model_short_name": "..." },
"init_kwargs": { "num_gpus": 1, ... },
@@ -470,19 +347,13 @@ When the rolling-baseline phase runs, it emits:
"max_vae_decode_time_s": 10.0
},
"default": { "max_generation_time_s": 120.0, "max_peak_memory_mb": 30000.0 }
},
"regression_thresholds": {
"latency": { "threshold_percent": 0.10, "threshold_absolute": 1.0, "gated": true }
}
}
```
}
```
Legacy v1 configs without `config_schema_version` still load, but should not
gain v2 identity or metadata fields until they are migrated to
`config_schema_version: 2`. For v2 configs, `workload_id`, `variant_id`,
and `benchmark_version` are part of the comparison key; benchmark runs
fail if any of these identity fields are missing.
`config_schema_version: 2`.
2. The pytest test auto-discovers all configs — no test code needed. CI
picks it up on the next `/test performance` run.
@@ -501,17 +372,10 @@ When the rolling-baseline phase runs, it emits:
a useful fixed gate. The rolling baseline will still track component times
when static component thresholds are omitted.
6. Omit `regression_thresholds` to use the default rolling-baseline policy, or
include only benchmark-specific deviations. Tune these independently from
the fixed thresholds when a metric is noisy or should be informational. The
fixed `thresholds` block is an absolute pytest ceiling. The
`regression_thresholds` block controls rolling-baseline comparisons against
recent scheduled-main records.
## Troubleshooting
**"No baseline for ... Initializing"** — first run for this comparison cohort.
Run will pass and (if persisting) seed the first record.
**"No baseline for ... Initializing"** — first run for this `(model_id,
gpu_type)`. Run will pass and (if persisting) seed the first record.
**Persistent failure right after a torch / kernel / image upgrade** —
genuine regression *or* baseline drift. Compare the failing normalized record
@@ -524,5 +388,5 @@ pipelines that did not report a mapped component stage.
**Component timing is `null`** — the generated result did not include a mapped
stage in `logging_info.stages`. Check that the pipeline emits stage logging
and that the stage emits `component_metric` or is covered by the legacy
`STAGE_METRIC_MAP` fallback in `test_inference_performance.py`.
and that the stage name is listed in `STAGE_METRIC_MAP` in
`test_inference_performance.py`.
+2 -2
View File
@@ -58,8 +58,8 @@ pipeline initialization and sampling.
| FastWan2.1 T2V 1.3B | `FastVideo/FastWan2.1-T2V-1.3B-Diffusers` | 480P | ⭕ | ⭕ | ⭕ | ✅ | ⭕ |
| FastWan2.2 TI2V 5B Full Attn* | `FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers` | 720P | ⭕ | ⭕ | ⭕ | ✅ | ⭕ |
| Wan2.2 TI2V 5B | `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | 720P | ⭕ | ⭕ | ✅ | ⭕ | ⭕ |
| DreamX-World 5B Cam | `FastVideo/DreamX-World-5B-Cam-Diffusers` | 480P | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ |
| DreamX-World 5B AR | `FastVideo/DreamX-World-5B-Diffusers` | 704px1280p | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ |
| DreamX-World 5B Cam | `GD-ML/DreamX-World-5B-Cam` | 480P | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ |
| DreamX-World 5B AR | `GD-ML/DreamX-World-5B` | 704px1280p | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ |
| Lucy Edit Dev 5B*** | `decart-ai/Lucy-Edit-Dev` | 480P | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ |
| Wan2.2 T2V A14B | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 480P<br>720P | ❌ | ❌ | ✅ | ⭕ | ⭕ |
| Wan2.2 I2V A14B | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | 480P<br>720P | ❌ | ❌ | ✅ | ⭕ | ⭕ |
-91
View File
@@ -1,91 +0,0 @@
# Training Trackers
FastVideo can send training metrics and validation media to Weights & Biases
or SwanLab. Tracking runs only on global rank 0, and local tracker files are
stored under `<output_dir>/tracker`.
## Supported Trackers
| Value | Backend | Installation |
|-------|---------|--------------|
| `wandb` | Weights & Biases | Included with FastVideo |
| `swanlab` | SwanLab | Install the optional `swanlab` dependency |
| `none` | Disable external tracking | No additional package |
You can enable more than one backend, for example `trackers: [wandb, swanlab]`.
Metrics and validation media are converted to the artifact type required by
each backend.
## Install SwanLab
For a published FastVideo installation, install the SwanLab extra:
```bash
uv pip install "fastvideo[swanlab]"
```
For an editable source checkout, include the same extra during installation:
```bash
uv pip install -e ".[swanlab]"
```
If FastVideo is already installed, you can install the compatible SDK directly:
```bash
uv pip install "swanlab>=0.6.7"
```
Authenticate once before starting a training run:
```bash
swanlab login
```
See the [SwanLab login documentation](https://docs.swanlab.cn/en/api/cli-swanlab-login.html)
for non-interactive and self-hosted setups.
## Configure Tracking
Select SwanLab in the YAML config used by the modular training framework:
```yaml
training:
checkpoint:
output_dir: outputs/my_run
tracker:
trackers: [swanlab]
project_name: my_project
run_name: my_run
```
To log to both supported services:
```yaml
training:
tracker:
trackers: [wandb, swanlab]
project_name: my_project
run_name: my_run
```
An empty or omitted `trackers` list selects W&B when `project_name` is set.
Use an explicit `none` entry to disable external tracking:
```yaml
training:
tracker:
trackers: [none]
```
## Validation Videos
SwanLab currently accepts GIF video artifacts. FastVideo converts validation
MP4 files and in-memory video arrays to GIF automatically before logging them.
For video files, FastVideo uses the sampling frame rate supplied by the caller,
or the source file's frame rate when no value is supplied. In-memory arrays use
the frame rate supplied by the caller. Both forms fall back to 16 FPS when no
frame rate is available.
For details about configuring validation callbacks, see
[Training Infrastructure](train_infra.md#callbacks-pluggable-hooks).
-3
View File
@@ -161,9 +161,6 @@ training:
decay_interval_steps: 0
```
See [Training Trackers](trackers.md) to configure Weights & Biases or SwanLab,
including SwanLab installation and authentication.
### `callbacks` — Pluggable hooks
Callbacks run at specific points in the training loop (before/after optimizer
@@ -15,7 +15,7 @@ def _env_float(name: str, default: float) -> float:
def main():
model_name = os.getenv("DREAMX_WORLD_MODEL_DIR", "FastVideo/DreamX-World-5B-Cam-Diffusers")
model_name = os.getenv("DREAMX_WORLD_MODEL_DIR", "GD-ML/DreamX-World-5B-Cam")
generator = VideoGenerator.from_pretrained(
model_name,
num_gpus=1,
@@ -1,37 +0,0 @@
from fastvideo import VideoGenerator
OUTPUT_PATH = "video_samples_kandinsky5_i2v"
IMAGE_PATH = "assets/girl.png"
def main():
generator = VideoGenerator.from_pretrained(
"kandinskylab/Kandinsky-5.0-I2V-Pro-distilled-5s-Diffusers",
# "kandinskylab/Kandinsky-5.0-I2V-Pro-sft-5s-Diffusers"
# "kandinskylab/Kandinsky-5.0-I2V-Lite-5s-Diffusers"
num_gpus=1,
use_fsdp_inference=False,
dit_cpu_offload=False,
vae_cpu_offload=False,
text_encoder_cpu_offload=True,
pin_cpu_memory=True,
# image_encoder_cpu_offload=False,
)
prompt = (
"A woman stands up and walks away"
)
_ = generator.generate_video(
prompt,
image_path=IMAGE_PATH,
output_path=OUTPUT_PATH,
save_video=True,
height=1024,
width=1024,
num_frames=121,
)
if __name__ == "__main__":
main()
@@ -1,37 +0,0 @@
from fastvideo import VideoGenerator
OUTPUT_PATH = "video_samples_kandinsky5_t2v"
def main():
generator = VideoGenerator.from_pretrained(
"kandinskylab/Kandinsky-5.0-T2V-Lite-sft-5s-Diffusers",
# "kandinskylab/Kandinsky-5.0-T2V-Pro-sft-5s-Diffusers"
# "kandinskylab/Kandinsky-5.0-T2V-Lite-distilled16steps-5s-Diffusers"
# "kandinskylab/Kandinsky-5.0-T2V-Pro-distilled-5s-Diffusers"
num_gpus=1,
use_fsdp_inference=False,
dit_cpu_offload=False,
vae_cpu_offload=False,
text_encoder_cpu_offload=True,
pin_cpu_memory=True,
# image_encoder_cpu_offload=False,
)
prompt = (
"A curious raccoon peers through a vibrant field of yellow sunflowers, its eyes "
"wide with interest. The playful yet serene atmosphere is complemented by soft "
"natural light filtering through the petals. Mid-shot, warm and cheerful tones."
)
_ = generator.generate_video(prompt, output_path=OUTPUT_PATH, save_video=True,height=512, width=768, num_frames=121)
prompt2 = (
"A majestic lion strides across the golden savanna, its powerful frame "
"glistening under the warm afternoon sun. The tall grass ripples gently in "
"the breeze, enhancing the lion's commanding presence. The tone is vibrant, "
"embodying the raw energy of the wild. Low angle, steady tracking shot, "
"cinematic.")
_ = generator.generate_video(prompt2, output_path=OUTPUT_PATH, save_video=True, height=512, width=768, num_frames=121)
if __name__ == "__main__":
main()
-69
View File
@@ -1,69 +0,0 @@
# LTX-2.3 distilled inference configs
Ready-to-run `fastvideo generate` run configs for the LTX-2.3
distilled model (`FastVideo/LTX-2.3-Distilled-Diffusers`), covering both
workloads (t2v / i2v), both two-stage step schedules (`5+2`, `8+3` = denoise
+ refine), and four resolutions.
```bash
fastvideo generate --config examples/inference/ltx2_3/t2v_8s3_1280x832.yaml
```
Each config is self-contained (no preset registry needed): the two-stage
refine is wired via `generator.pipeline.preset_overrides.refine`, and the
base sampling knobs live under `request.sampling`. The refine upsampler
auto-resolves from the model's `spatial_upscaler`.
## Configs
| workload | schedule | resolution (HxW) | file |
|---|---|---|---|
| t2v | 5+2 | 1280x832 | `t2v_5s2_1280x832.yaml` |
| t2v | 5+2 | 1024x1536 | `t2v_5s2_1024x1536.yaml` |
| t2v | 5+2 | 768x1280 | `t2v_5s2_768x1280.yaml` |
| t2v | 5+2 | 512x768 | `t2v_5s2_512x768.yaml` |
| t2v | 8+3 | 1280x832 | `t2v_8s3_1280x832.yaml` |
| t2v | 8+3 | 1024x1536 | `t2v_8s3_1024x1536.yaml` |
| t2v | 8+3 | 768x1280 | `t2v_8s3_768x1280.yaml` |
| t2v | 8+3 | 512x768 | `t2v_8s3_512x768.yaml` |
| i2v | 5+2 | 1280x832 | `i2v_5s2_1280x832.yaml` |
| i2v | 5+2 | 1024x1536 | `i2v_5s2_1024x1536.yaml` |
| i2v | 5+2 | 768x1280 | `i2v_5s2_768x1280.yaml` |
| i2v | 5+2 | 512x768 | `i2v_5s2_512x768.yaml` |
| i2v | 8+3 | 1280x832 | `i2v_8s3_1280x832.yaml` |
| i2v | 8+3 | 1024x1536 | `i2v_8s3_1024x1536.yaml` |
| i2v | 8+3 | 768x1280 | `i2v_8s3_768x1280.yaml` |
| i2v | 8+3 | 512x768 | `i2v_8s3_512x768.yaml` |
## Overriding without editing a file
Dotted overrides (prefixes `generator.` / `request.`) let you tweak any field:
```bash
# swap prompt
fastvideo generate --config examples/inference/ltx2_3/t2v_8s3_1280x832.yaml \
--request.prompt "a red fox running through fresh snow"
# change output path / gpu count
fastvideo generate --config examples/inference/ltx2_3/t2v_5s2_512x768.yaml \
--request.output.output_path outputs/preview.mp4 \
--generator.engine.num_gpus 4
```
## i2v
The `i2v_*` configs take a first-frame image via
`request.extensions.ltx2_images` (`[[path, frame_offset, weight]]`). Edit the
path in the file, or override it:
```bash
fastvideo generate --config examples/inference/ltx2_3/i2v_8s3_1280x832.yaml \
--request.extensions.ltx2_images '[["/data/portrait.jpg", 0, 1.0]]'
```
## Schedules
`5+2` is the fast preview schedule; `8+3` is the higher-quality distilled
recipe. Refine (`preset_overrides.refine.num_inference_steps`) only accepts 2
or 3 steps.
@@ -1,38 +0,0 @@
# LTX-2.3 distilled i2v — 5+2 two-stage at 1024x1536.
#
# Run:
# fastvideo generate --config examples/inference/ltx2_3/i2v_5s2_1024x1536.yaml
#
# Stage 1 denoises for 5 steps at half resolution; the latents are then
# spatially upsampled and refined for 2 steps (refine only supports 2 or 3).
# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
generator:
model_path: FastVideo/LTX-2.3-Distilled-Diffusers
engine:
num_gpus: 1
pipeline:
workload_type: i2v
preset_overrides:
refine:
enabled: true
num_inference_steps: 2
guidance_scale: 1.0
add_noise: true
request:
prompt: >-
The subject slowly turns toward the camera with a soft, natural expression, hair and clothing swaying gently, shallow depth of field, subtle cinematic motion.
sampling:
height: 1024
width: 1536
num_frames: 121
fps: 24
guidance_scale: 1.0
num_inference_steps: 5
# i2v conditioning — replace the path with your own first-frame image.
extensions:
ltx2_images:
- ["/path/to/your/first_frame.jpg", 0, 1.0]
ltx2_image_crf: 0.0
output:
output_path: outputs/ltx2_3_i2v_5s2_1024x1536.mp4
save_video: true
@@ -1,38 +0,0 @@
# LTX-2.3 distilled i2v — 5+2 two-stage at 1280x832.
#
# Run:
# fastvideo generate --config examples/inference/ltx2_3/i2v_5s2_1280x832.yaml
#
# Stage 1 denoises for 5 steps at half resolution; the latents are then
# spatially upsampled and refined for 2 steps (refine only supports 2 or 3).
# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
generator:
model_path: FastVideo/LTX-2.3-Distilled-Diffusers
engine:
num_gpus: 1
pipeline:
workload_type: i2v
preset_overrides:
refine:
enabled: true
num_inference_steps: 2
guidance_scale: 1.0
add_noise: true
request:
prompt: >-
The subject slowly turns toward the camera with a soft, natural expression, hair and clothing swaying gently, shallow depth of field, subtle cinematic motion.
sampling:
height: 1280
width: 832
num_frames: 121
fps: 24
guidance_scale: 1.0
num_inference_steps: 5
# i2v conditioning — replace the path with your own first-frame image.
extensions:
ltx2_images:
- ["/path/to/your/first_frame.jpg", 0, 1.0]
ltx2_image_crf: 0.0
output:
output_path: outputs/ltx2_3_i2v_5s2_1280x832.mp4
save_video: true
@@ -1,38 +0,0 @@
# LTX-2.3 distilled i2v — 5+2 two-stage at 512x768.
#
# Run:
# fastvideo generate --config examples/inference/ltx2_3/i2v_5s2_512x768.yaml
#
# Stage 1 denoises for 5 steps at half resolution; the latents are then
# spatially upsampled and refined for 2 steps (refine only supports 2 or 3).
# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
generator:
model_path: FastVideo/LTX-2.3-Distilled-Diffusers
engine:
num_gpus: 1
pipeline:
workload_type: i2v
preset_overrides:
refine:
enabled: true
num_inference_steps: 2
guidance_scale: 1.0
add_noise: true
request:
prompt: >-
The subject slowly turns toward the camera with a soft, natural expression, hair and clothing swaying gently, shallow depth of field, subtle cinematic motion.
sampling:
height: 512
width: 768
num_frames: 121
fps: 24
guidance_scale: 1.0
num_inference_steps: 5
# i2v conditioning — replace the path with your own first-frame image.
extensions:
ltx2_images:
- ["/path/to/your/first_frame.jpg", 0, 1.0]
ltx2_image_crf: 0.0
output:
output_path: outputs/ltx2_3_i2v_5s2_512x768.mp4
save_video: true
@@ -1,38 +0,0 @@
# LTX-2.3 distilled i2v — 5+2 two-stage at 768x1280.
#
# Run:
# fastvideo generate --config examples/inference/ltx2_3/i2v_5s2_768x1280.yaml
#
# Stage 1 denoises for 5 steps at half resolution; the latents are then
# spatially upsampled and refined for 2 steps (refine only supports 2 or 3).
# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
generator:
model_path: FastVideo/LTX-2.3-Distilled-Diffusers
engine:
num_gpus: 1
pipeline:
workload_type: i2v
preset_overrides:
refine:
enabled: true
num_inference_steps: 2
guidance_scale: 1.0
add_noise: true
request:
prompt: >-
The subject slowly turns toward the camera with a soft, natural expression, hair and clothing swaying gently, shallow depth of field, subtle cinematic motion.
sampling:
height: 768
width: 1280
num_frames: 121
fps: 24
guidance_scale: 1.0
num_inference_steps: 5
# i2v conditioning — replace the path with your own first-frame image.
extensions:
ltx2_images:
- ["/path/to/your/first_frame.jpg", 0, 1.0]
ltx2_image_crf: 0.0
output:
output_path: outputs/ltx2_3_i2v_5s2_768x1280.mp4
save_video: true
@@ -1,38 +0,0 @@
# LTX-2.3 distilled i2v — 8+3 two-stage at 1024x1536.
#
# Run:
# fastvideo generate --config examples/inference/ltx2_3/i2v_8s3_1024x1536.yaml
#
# Stage 1 denoises for 8 steps at half resolution; the latents are then
# spatially upsampled and refined for 3 steps (refine only supports 2 or 3).
# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
generator:
model_path: FastVideo/LTX-2.3-Distilled-Diffusers
engine:
num_gpus: 1
pipeline:
workload_type: i2v
preset_overrides:
refine:
enabled: true
num_inference_steps: 3
guidance_scale: 1.0
add_noise: true
request:
prompt: >-
The subject slowly turns toward the camera with a soft, natural expression, hair and clothing swaying gently, shallow depth of field, subtle cinematic motion.
sampling:
height: 1024
width: 1536
num_frames: 121
fps: 24
guidance_scale: 1.0
num_inference_steps: 8
# i2v conditioning — replace the path with your own first-frame image.
extensions:
ltx2_images:
- ["/path/to/your/first_frame.jpg", 0, 1.0]
ltx2_image_crf: 0.0
output:
output_path: outputs/ltx2_3_i2v_8s3_1024x1536.mp4
save_video: true
@@ -1,38 +0,0 @@
# LTX-2.3 distilled i2v — 8+3 two-stage at 1280x832.
#
# Run:
# fastvideo generate --config examples/inference/ltx2_3/i2v_8s3_1280x832.yaml
#
# Stage 1 denoises for 8 steps at half resolution; the latents are then
# spatially upsampled and refined for 3 steps (refine only supports 2 or 3).
# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
generator:
model_path: FastVideo/LTX-2.3-Distilled-Diffusers
engine:
num_gpus: 1
pipeline:
workload_type: i2v
preset_overrides:
refine:
enabled: true
num_inference_steps: 3
guidance_scale: 1.0
add_noise: true
request:
prompt: >-
The subject slowly turns toward the camera with a soft, natural expression, hair and clothing swaying gently, shallow depth of field, subtle cinematic motion.
sampling:
height: 1280
width: 832
num_frames: 121
fps: 24
guidance_scale: 1.0
num_inference_steps: 8
# i2v conditioning — replace the path with your own first-frame image.
extensions:
ltx2_images:
- ["/path/to/your/first_frame.jpg", 0, 1.0]
ltx2_image_crf: 0.0
output:
output_path: outputs/ltx2_3_i2v_8s3_1280x832.mp4
save_video: true
@@ -1,38 +0,0 @@
# LTX-2.3 distilled i2v — 8+3 two-stage at 512x768.
#
# Run:
# fastvideo generate --config examples/inference/ltx2_3/i2v_8s3_512x768.yaml
#
# Stage 1 denoises for 8 steps at half resolution; the latents are then
# spatially upsampled and refined for 3 steps (refine only supports 2 or 3).
# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
generator:
model_path: FastVideo/LTX-2.3-Distilled-Diffusers
engine:
num_gpus: 1
pipeline:
workload_type: i2v
preset_overrides:
refine:
enabled: true
num_inference_steps: 3
guidance_scale: 1.0
add_noise: true
request:
prompt: >-
The subject slowly turns toward the camera with a soft, natural expression, hair and clothing swaying gently, shallow depth of field, subtle cinematic motion.
sampling:
height: 512
width: 768
num_frames: 121
fps: 24
guidance_scale: 1.0
num_inference_steps: 8
# i2v conditioning — replace the path with your own first-frame image.
extensions:
ltx2_images:
- ["/path/to/your/first_frame.jpg", 0, 1.0]
ltx2_image_crf: 0.0
output:
output_path: outputs/ltx2_3_i2v_8s3_512x768.mp4
save_video: true
@@ -1,38 +0,0 @@
# LTX-2.3 distilled i2v — 8+3 two-stage at 768x1280.
#
# Run:
# fastvideo generate --config examples/inference/ltx2_3/i2v_8s3_768x1280.yaml
#
# Stage 1 denoises for 8 steps at half resolution; the latents are then
# spatially upsampled and refined for 3 steps (refine only supports 2 or 3).
# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
generator:
model_path: FastVideo/LTX-2.3-Distilled-Diffusers
engine:
num_gpus: 1
pipeline:
workload_type: i2v
preset_overrides:
refine:
enabled: true
num_inference_steps: 3
guidance_scale: 1.0
add_noise: true
request:
prompt: >-
The subject slowly turns toward the camera with a soft, natural expression, hair and clothing swaying gently, shallow depth of field, subtle cinematic motion.
sampling:
height: 768
width: 1280
num_frames: 121
fps: 24
guidance_scale: 1.0
num_inference_steps: 8
# i2v conditioning — replace the path with your own first-frame image.
extensions:
ltx2_images:
- ["/path/to/your/first_frame.jpg", 0, 1.0]
ltx2_image_crf: 0.0
output:
output_path: outputs/ltx2_3_i2v_8s3_768x1280.mp4
save_video: true
@@ -1,33 +0,0 @@
# LTX-2.3 distilled t2v — 5+2 two-stage at 1024x1536.
#
# Run:
# fastvideo generate --config examples/inference/ltx2_3/t2v_5s2_1024x1536.yaml
#
# Stage 1 denoises for 5 steps at half resolution; the latents are then
# spatially upsampled and refined for 2 steps (refine only supports 2 or 3).
# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
generator:
model_path: FastVideo/LTX-2.3-Distilled-Diffusers
engine:
num_gpus: 1
pipeline:
workload_type: t2v
preset_overrides:
refine:
enabled: true
num_inference_steps: 2
guidance_scale: 1.0
add_noise: true
request:
prompt: >-
A cinematic drone shot flying over dramatic coastal cliffs at sunrise, golden light spilling across the water, gentle waves breaking on the rocks below, ultra-detailed, smooth camera motion.
sampling:
height: 1024
width: 1536
num_frames: 121
fps: 24
guidance_scale: 1.0
num_inference_steps: 5
output:
output_path: outputs/ltx2_3_t2v_5s2_1024x1536.mp4
save_video: true
@@ -1,33 +0,0 @@
# LTX-2.3 distilled t2v — 5+2 two-stage at 1280x832.
#
# Run:
# fastvideo generate --config examples/inference/ltx2_3/t2v_5s2_1280x832.yaml
#
# Stage 1 denoises for 5 steps at half resolution; the latents are then
# spatially upsampled and refined for 2 steps (refine only supports 2 or 3).
# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
generator:
model_path: FastVideo/LTX-2.3-Distilled-Diffusers
engine:
num_gpus: 1
pipeline:
workload_type: t2v
preset_overrides:
refine:
enabled: true
num_inference_steps: 2
guidance_scale: 1.0
add_noise: true
request:
prompt: >-
A cinematic drone shot flying over dramatic coastal cliffs at sunrise, golden light spilling across the water, gentle waves breaking on the rocks below, ultra-detailed, smooth camera motion.
sampling:
height: 1280
width: 832
num_frames: 121
fps: 24
guidance_scale: 1.0
num_inference_steps: 5
output:
output_path: outputs/ltx2_3_t2v_5s2_1280x832.mp4
save_video: true
@@ -1,33 +0,0 @@
# LTX-2.3 distilled t2v — 5+2 two-stage at 512x768.
#
# Run:
# fastvideo generate --config examples/inference/ltx2_3/t2v_5s2_512x768.yaml
#
# Stage 1 denoises for 5 steps at half resolution; the latents are then
# spatially upsampled and refined for 2 steps (refine only supports 2 or 3).
# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
generator:
model_path: FastVideo/LTX-2.3-Distilled-Diffusers
engine:
num_gpus: 1
pipeline:
workload_type: t2v
preset_overrides:
refine:
enabled: true
num_inference_steps: 2
guidance_scale: 1.0
add_noise: true
request:
prompt: >-
A cinematic drone shot flying over dramatic coastal cliffs at sunrise, golden light spilling across the water, gentle waves breaking on the rocks below, ultra-detailed, smooth camera motion.
sampling:
height: 512
width: 768
num_frames: 121
fps: 24
guidance_scale: 1.0
num_inference_steps: 5
output:
output_path: outputs/ltx2_3_t2v_5s2_512x768.mp4
save_video: true
@@ -1,33 +0,0 @@
# LTX-2.3 distilled t2v — 5+2 two-stage at 768x1280.
#
# Run:
# fastvideo generate --config examples/inference/ltx2_3/t2v_5s2_768x1280.yaml
#
# Stage 1 denoises for 5 steps at half resolution; the latents are then
# spatially upsampled and refined for 2 steps (refine only supports 2 or 3).
# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
generator:
model_path: FastVideo/LTX-2.3-Distilled-Diffusers
engine:
num_gpus: 1
pipeline:
workload_type: t2v
preset_overrides:
refine:
enabled: true
num_inference_steps: 2
guidance_scale: 1.0
add_noise: true
request:
prompt: >-
A cinematic drone shot flying over dramatic coastal cliffs at sunrise, golden light spilling across the water, gentle waves breaking on the rocks below, ultra-detailed, smooth camera motion.
sampling:
height: 768
width: 1280
num_frames: 121
fps: 24
guidance_scale: 1.0
num_inference_steps: 5
output:
output_path: outputs/ltx2_3_t2v_5s2_768x1280.mp4
save_video: true
@@ -1,33 +0,0 @@
# LTX-2.3 distilled t2v — 8+3 two-stage at 1024x1536.
#
# Run:
# fastvideo generate --config examples/inference/ltx2_3/t2v_8s3_1024x1536.yaml
#
# Stage 1 denoises for 8 steps at half resolution; the latents are then
# spatially upsampled and refined for 3 steps (refine only supports 2 or 3).
# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
generator:
model_path: FastVideo/LTX-2.3-Distilled-Diffusers
engine:
num_gpus: 1
pipeline:
workload_type: t2v
preset_overrides:
refine:
enabled: true
num_inference_steps: 3
guidance_scale: 1.0
add_noise: true
request:
prompt: >-
A cinematic drone shot flying over dramatic coastal cliffs at sunrise, golden light spilling across the water, gentle waves breaking on the rocks below, ultra-detailed, smooth camera motion.
sampling:
height: 1024
width: 1536
num_frames: 121
fps: 24
guidance_scale: 1.0
num_inference_steps: 8
output:
output_path: outputs/ltx2_3_t2v_8s3_1024x1536.mp4
save_video: true
@@ -1,33 +0,0 @@
# LTX-2.3 distilled t2v — 8+3 two-stage at 1280x832.
#
# Run:
# fastvideo generate --config examples/inference/ltx2_3/t2v_8s3_1280x832.yaml
#
# Stage 1 denoises for 8 steps at half resolution; the latents are then
# spatially upsampled and refined for 3 steps (refine only supports 2 or 3).
# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
generator:
model_path: FastVideo/LTX-2.3-Distilled-Diffusers
engine:
num_gpus: 1
pipeline:
workload_type: t2v
preset_overrides:
refine:
enabled: true
num_inference_steps: 3
guidance_scale: 1.0
add_noise: true
request:
prompt: >-
A cinematic drone shot flying over dramatic coastal cliffs at sunrise, golden light spilling across the water, gentle waves breaking on the rocks below, ultra-detailed, smooth camera motion.
sampling:
height: 1280
width: 832
num_frames: 121
fps: 24
guidance_scale: 1.0
num_inference_steps: 8
output:
output_path: outputs/ltx2_3_t2v_8s3_1280x832.mp4
save_video: true
@@ -1,33 +0,0 @@
# LTX-2.3 distilled t2v — 8+3 two-stage at 512x768.
#
# Run:
# fastvideo generate --config examples/inference/ltx2_3/t2v_8s3_512x768.yaml
#
# Stage 1 denoises for 8 steps at half resolution; the latents are then
# spatially upsampled and refined for 3 steps (refine only supports 2 or 3).
# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
generator:
model_path: FastVideo/LTX-2.3-Distilled-Diffusers
engine:
num_gpus: 1
pipeline:
workload_type: t2v
preset_overrides:
refine:
enabled: true
num_inference_steps: 3
guidance_scale: 1.0
add_noise: true
request:
prompt: >-
A cinematic drone shot flying over dramatic coastal cliffs at sunrise, golden light spilling across the water, gentle waves breaking on the rocks below, ultra-detailed, smooth camera motion.
sampling:
height: 512
width: 768
num_frames: 121
fps: 24
guidance_scale: 1.0
num_inference_steps: 8
output:
output_path: outputs/ltx2_3_t2v_8s3_512x768.mp4
save_video: true
@@ -1,33 +0,0 @@
# LTX-2.3 distilled t2v — 8+3 two-stage at 768x1280.
#
# Run:
# fastvideo generate --config examples/inference/ltx2_3/t2v_8s3_768x1280.yaml
#
# Stage 1 denoises for 8 steps at half resolution; the latents are then
# spatially upsampled and refined for 3 steps (refine only supports 2 or 3).
# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
generator:
model_path: FastVideo/LTX-2.3-Distilled-Diffusers
engine:
num_gpus: 1
pipeline:
workload_type: t2v
preset_overrides:
refine:
enabled: true
num_inference_steps: 3
guidance_scale: 1.0
add_noise: true
request:
prompt: >-
A cinematic drone shot flying over dramatic coastal cliffs at sunrise, golden light spilling across the water, gentle waves breaking on the rocks below, ultra-detailed, smooth camera motion.
sampling:
height: 768
width: 1280
num_frames: 121
fps: 24
guidance_scale: 1.0
num_inference_steps: 8
output:
output_path: outputs/ltx2_3_t2v_8s3_768x1280.mp4
save_video: true
@@ -1,78 +0,0 @@
# Causal Consistency Distillation: Wan 2.1 T2V 1.3B Causal
#
# ODE-data-free distillation. A frozen teacher takes a single CFG Euler step;
# the student matches an EMA copy of itself at the next timestep, all under
# clean-history teacher forcing.
#
# All three roles initialize from the SAME checkpoint (the teacher-forcing
# AR-diffusion model). Point init_from at that checkpoint for a real run.
models:
student:
_target_: fastvideo.train.models.wan.WanCausalModel
init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers
trainable: true
teacher:
_target_: fastvideo.train.models.wan.WanCausalModel
init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers
trainable: false
ema:
_target_: fastvideo.train.models.wan.WanCausalModel
init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers
trainable: false
method:
_target_: fastvideo.train.methods.consistency_model.causal_cd.CausalConsistencyDistillationMethod
discrete_cd_N: 48
guidance_scale: 3.0
ema_decay: 0.99
ema_start_step: 200
training:
distributed:
num_gpus: 8
sp_size: 1
tp_size: 1
hsdp_replicate_dim: 1
hsdp_shard_dim: 8
data:
data_path: data/Wan-Syn_77x448x832_600k
dataloader_num_workers: 4
train_batch_size: 1
training_cfg_rate: 0.0
seed: 1000
num_latent_t: 18
num_height: 448
num_width: 832
num_frames: 69
optimizer:
learning_rate: 2.0e-6
betas: [0.9, 0.999]
weight_decay: 0.01
lr_scheduler: constant
lr_warmup_steps: 0
loop:
max_train_steps: 3000
gradient_accumulation_steps: 1
checkpoint:
output_dir: outputs/wan2.1_causal_cd
training_state_checkpointing_steps: 1000
checkpoints_total_limit: 3
tracker:
project_name: distillation_wan_r
run_name: wan2.1_causal_cd_shift5
model:
enable_gradient_checkpointing_type: full
callbacks:
grad_clip:
max_grad_norm: 1.0
pipeline:
flow_shift: 5
@@ -1,73 +0,0 @@
# DFSFT (Diffusion-Forcing SFT), frame-wise: Wan 2.1 T2V 1.3B Causal
#
# - Student: trainable causal Wan model with a block size of 1 frame
# - Training: each frame gets its own independent noise level (frame-wise
# diffusion forcing), versus the chunk-wise variant that shares one noise
# level across num_frames_per_block frames.
models:
student:
_target_: fastvideo.train.models.wan.WanCausalModel
init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers
trainable: true
num_frames_per_block: 1
method:
_target_: fastvideo.train.methods.fine_tuning.dfsft.DiffusionForcingSFTMethod
chunk_size: 1
training:
distributed:
num_gpus: 8
sp_size: 1
tp_size: 1
hsdp_replicate_dim: 1
hsdp_shard_dim: 8
data:
data_path: data/Wan-Syn_77x448x832_600k
dataloader_num_workers: 4
train_batch_size: 1
training_cfg_rate: 0.0
seed: 1000
num_latent_t: 18
num_height: 448
num_width: 832
num_frames: 69
optimizer:
learning_rate: 2.0e-6
betas: [0.9, 0.999]
weight_decay: 0.01
lr_scheduler: constant
lr_warmup_steps: 0
loop:
max_train_steps: 4000
gradient_accumulation_steps: 1
checkpoint:
output_dir: outputs/wan2.1_causal_dfsft_framewise
training_state_checkpointing_steps: 1000
checkpoints_total_limit: 3
tracker:
project_name: distillation_wan_r
run_name: wan2.1_causal_dfsft_framewise_shift5_gauss_weight
model:
enable_gradient_checkpointing_type: full
callbacks:
grad_clip:
max_grad_norm: 1.0
validation:
pipeline_target: fastvideo.pipelines.basic.wan.wan_causal_pipeline.WanCausalPipeline
dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json
every_steps: 50
sampling_steps: [40]
guidance_scale: 6.0
num_frames: 69
pipeline:
flow_shift: 5
@@ -1,72 +0,0 @@
# TFSFT (Teacher-Forcing SFT): Wan 2.1 T2V 1.3B Causal
#
# - Student: trainable causal Wan model
# - Training: inhomogeneous timesteps per chunk, but the causal transformer
# denoises the current block while attending to *clean* history (clean_x),
# not its own noisy rollout.
models:
student:
_target_: fastvideo.train.models.wan.WanCausalModel
init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers
trainable: true
method:
_target_: fastvideo.train.methods.fine_tuning.tfsft.TeacherForcingSFTMethod
chunk_size: 3
training:
distributed:
num_gpus: 8
sp_size: 1
tp_size: 1
hsdp_replicate_dim: 1
hsdp_shard_dim: 8
data:
data_path: data/Wan-Syn_77x448x832_600k
dataloader_num_workers: 4
train_batch_size: 1
training_cfg_rate: 0.0
seed: 1000
num_latent_t: 18
num_height: 448
num_width: 832
num_frames: 69
optimizer:
learning_rate: 2.0e-6
betas: [0.9, 0.999]
weight_decay: 0.01
lr_scheduler: constant
lr_warmup_steps: 0
loop:
max_train_steps: 4000
gradient_accumulation_steps: 1
checkpoint:
output_dir: outputs/wan2.1_causal_tfsft
training_state_checkpointing_steps: 1000
checkpoints_total_limit: 3
tracker:
project_name: distillation_wan_r
run_name: wan2.1_causal_tfsft_shift5_gauss_weight
model:
enable_gradient_checkpointing_type: full
callbacks:
grad_clip:
max_grad_norm: 1.0
validation:
pipeline_target: fastvideo.pipelines.basic.wan.wan_causal_pipeline.WanCausalPipeline
dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json
every_steps: 50
sampling_steps: [40]
guidance_scale: 6.0
num_frames: 69
pipeline:
flow_shift: 5
+1 -5
View File
@@ -338,11 +338,7 @@ class FlashAttentionImpl(AttentionImpl):
)
qkv = torch.stack([query, key, value], dim=2)
key_padding_mask = _key_padding_mask_from_attn_mask(attn_mask, attn_mask.shape[-1]).to(device=query.device)
if key_padding_mask.shape[-1] > qkv.shape[1]:
raise ValueError("Invalid key padding mask length for FLASH_ATTN: "
f"expected at most {qkv.shape[1]}, got {key_padding_mask.shape[-1]}")
attn_mask_padded = F.pad(key_padding_mask, (qkv.shape[1] - key_padding_mask.shape[-1], 0), value=True)
attn_mask_padded = F.pad(attn_mask, (qkv.shape[1] - attn_mask.shape[1], 0), value=True)
output = flash_attn_no_pad(qkv, attn_mask_padded, causal=False, dropout_p=0, softmax_scale=None)
elif self.nvfp4_fa4:
output = self._forward_nvfp4(query, key, value)
-147
View File
@@ -1,147 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""NABLA block-sparse flex-attention backend (Kandinsky5 "nabla" checkpoints).
The block mask is data-dependent: nablaT_v2 mean-pools 64-token blocks of the
fractal-ordered sequence, thresholds the softmaxed block map, and ORs it with a
precomputed spatio-temporal-window (STA) mask carried on the attention
metadata. The mask spans the full sequence, so this backend does not support
sequence parallelism — use it via LocalAttention only.
"""
import math
from dataclasses import dataclass
from typing import Any
import torch
try:
from torch.nn.attention.flex_attention import BlockMask, flex_attention
flex_attention = torch.compile(flex_attention, dynamic=False, mode="max-autotune-no-cudagraphs")
CAN_USE_FLEX_ATTN = True
except ImportError:
CAN_USE_FLEX_ATTN = False
from fastvideo.attention.backends.abstract import (
AttentionBackend,
AttentionImpl,
AttentionMetadata,
AttentionMetadataBuilder,
)
def nablaT_v2(
q: torch.Tensor,
k: torch.Tensor,
sta: torch.Tensor,
thr: float = 0.9,
) -> "BlockMask":
q = q.transpose(1, 2).contiguous()
k = k.transpose(1, 2).contiguous()
# Map estimation
B, h, S, D = q.shape
s1 = S // 64
qa = q.reshape(B, h, s1, 64, D).mean(-2)
ka = k.reshape(B, h, s1, 64, D).mean(-2).transpose(-2, -1)
map = qa @ ka
map = torch.softmax(map / math.sqrt(D), dim=-1)
# Map binarization
vals, inds = map.sort(-1)
cvals = vals.cumsum_(-1)
mask = (cvals >= 1 - thr).int()
mask = mask.gather(-1, inds.argsort(-1))
mask = torch.logical_or(mask, sta)
# BlockMask creation
kv_nb = mask.sum(-1).to(torch.int32)
kv_inds = mask.argsort(dim=-1, descending=True).to(torch.int32)
return BlockMask.from_kv_blocks(torch.zeros_like(kv_nb), kv_inds, kv_nb, kv_inds, BLOCK_SIZE=64, mask_mod=None)
class NablaAttentionBackend(AttentionBackend):
@staticmethod
def get_name() -> str:
return "NABLA_ATTN"
@staticmethod
def get_impl_cls() -> type["NablaAttentionImpl"]:
return NablaAttentionImpl
@staticmethod
def get_metadata_cls() -> type["NablaAttentionMetadata"]:
return NablaAttentionMetadata
@staticmethod
def get_builder_cls() -> type["NablaAttentionMetadataBuilder"]:
return NablaAttentionMetadataBuilder
@dataclass
class NablaAttentionMetadata(AttentionMetadata):
# Block-level STA window mask [1, 1, S/64, S/64], precomputed once per run.
sta_mask: torch.Tensor = None # type: ignore[assignment]
# Cumulative-probability threshold for block-map binarization.
P: float = 0.9
visual_shape: tuple[int, int, int] = (0, 0, 0)
class NablaAttentionMetadataBuilder(AttentionMetadataBuilder):
def __init__(self) -> None:
pass
def prepare(self) -> None:
pass
def build(
self,
current_timestep: int,
sta_mask: torch.Tensor,
P: float,
visual_shape: tuple[int, int, int],
**kwargs: Any,
) -> NablaAttentionMetadata:
return NablaAttentionMetadata(
current_timestep=current_timestep,
sta_mask=sta_mask,
P=P,
visual_shape=visual_shape,
)
class NablaAttentionImpl(AttentionImpl):
def __init__(
self,
num_heads: int,
head_size: int,
softmax_scale: float,
causal: bool = False,
num_kv_heads: int | None = None,
prefix: str = "",
**extra_impl_args,
) -> None:
if not CAN_USE_FLEX_ATTN:
raise RuntimeError("NABLA attention requires torch.nn.attention.flex_attention, "
"which is unavailable in this PyTorch build.")
if causal:
raise ValueError("NABLA attention does not support causal masking.")
def forward(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: NablaAttentionMetadata,
) -> torch.Tensor:
# q/k/v: [B, S, heads, head_dim], fractal-ordered by the model; S % 64 == 0.
block_mask = nablaT_v2(query, key, attn_metadata.sta_mask, thr=attn_metadata.P)
return flex_attention(
query=query.transpose(1, 2),
key=key.transpose(1, 2),
value=value.transpose(1, 2),
block_mask=block_mask,
).transpose(1, 2)
+1 -45
View File
@@ -2,7 +2,6 @@
import torch
from dataclasses import dataclass
from torch.nn import functional as F
from fastvideo.attention.backends.abstract import ( # FlashAttentionMetadata,
AttentionBackend, AttentionImpl, AttentionMetadata, AttentionMetadataBuilder)
from fastvideo.logger import init_logger
@@ -20,7 +19,7 @@ class SDPABackend(AttentionBackend):
@staticmethod
def get_name() -> str:
return "TORCH_SDPA"
return "SDPA"
@staticmethod
def get_impl_cls() -> type["SDPAImpl"]:
@@ -50,51 +49,9 @@ class SDPAMetadataBuilder(AttentionMetadataBuilder):
current_timestep: int,
attn_mask: torch.Tensor,
) -> SDPAMetadata:
# Store the mask exactly as passed. The metadata is cross-backend:
# call sites (HYWorld, HunyuanVideo15) build SDPAMetadata while the
# layer's selector may pick FLASH_ATTN, and the shared convention for
# padding masks is the tokenizer-style 2D [batch, key_len]. Any
# reshaping for torch.sdpa happens inside the SDPA impl
# (_normalize_attn_mask_for_sdpa).
return SDPAMetadata(current_timestep=current_timestep, attn_mask=attn_mask)
def _normalize_attn_mask_for_sdpa(
attn_mask: torch.Tensor | None,
query: torch.Tensor,
key: torch.Tensor,
) -> torch.Tensor | None:
if attn_mask is None:
return None
attn_mask = attn_mask.to(device=query.device)
# F.scaled_dot_product_attention only accepts bool or float masks;
# tokenizers commonly produce int64 0/1 padding masks.
if attn_mask.dtype != torch.bool and not attn_mask.dtype.is_floating_point:
attn_mask = attn_mask != 0
key_len = key.shape[-2]
if attn_mask.shape[-1] > key_len:
raise ValueError("Invalid attention mask length for SDPA: "
f"expected at most {key_len}, got {attn_mask.shape[-1]}")
if attn_mask.shape[-1] < key_len:
# Front-pad as "attend": double-stream layouts (HYWorld) prepend
# non-text tokens the tokenizer mask does not cover.
valid_value = True if attn_mask.dtype == torch.bool else 0.0
attn_mask = F.pad(attn_mask, (key_len - attn_mask.shape[-1], 0), value=valid_value)
if attn_mask.dim() == 2:
# In-tree producers pass 2D [batch, key_len] padding masks; lift to a
# broadcastable [batch, 1, 1, key_len] here so torch.sdpa does not
# reinterpret 2D as its documented [query_len, key_len] broadcast.
return attn_mask[:, None, None, :]
if attn_mask.dim() == 3:
return attn_mask[:, None, :, :]
if attn_mask.dim() == 4:
return attn_mask
raise ValueError(f"Unsupported attention mask shape for SDPA: {attn_mask.shape}")
class SDPAImpl(AttentionImpl):
def __init__(
@@ -125,7 +82,6 @@ class SDPAImpl(AttentionImpl):
attn_mask = attn_metadata.attn_mask if (attn_metadata is not None
and hasattr(attn_metadata, "attn_mask")) else None
attn_mask = _normalize_attn_mask_for_sdpa(attn_mask, query, key)
attn_kwargs = {
"attn_mask": attn_mask,
"dropout_p": self.dropout,
+1 -5
View File
@@ -252,7 +252,6 @@ class LocalAttention(nn.Module):
causal: bool = False,
supported_attention_backends: tuple[AttentionBackendEnum, ...]
| None = None,
default_backend: AttentionBackendEnum | None = None,
**extra_impl_args) -> None:
super().__init__()
if softmax_scale is None:
@@ -263,10 +262,7 @@ class LocalAttention(nn.Module):
num_kv_heads = num_heads
dtype = get_compute_dtype()
attn_backend = get_attn_backend(head_size,
dtype,
supported_attention_backends=supported_attention_backends,
default_backend=default_backend)
attn_backend = get_attn_backend(head_size, dtype, supported_attention_backends=supported_attention_backends)
impl_cls = attn_backend.get_impl_cls()
self.attn_impl = impl_cls(num_heads=num_heads,
head_size=head_size,
+1 -9
View File
@@ -84,9 +84,8 @@ def get_attn_backend(
dtype: torch.dtype,
supported_attention_backends: tuple[AttentionBackendEnum, ...]
| None = None,
default_backend: AttentionBackendEnum | None = None,
) -> type[AttentionBackend]:
return _cached_get_attn_backend(head_size, dtype, supported_attention_backends, default_backend)
return _cached_get_attn_backend(head_size, dtype, supported_attention_backends)
@cache
@@ -95,7 +94,6 @@ def _cached_get_attn_backend(
dtype: torch.dtype,
supported_attention_backends: tuple[AttentionBackendEnum, ...]
| None = None,
default_backend: AttentionBackendEnum | None = None,
) -> type[AttentionBackend]:
# Check whether a particular choice of backend was
# previously forced.
@@ -114,12 +112,6 @@ def _cached_get_attn_backend(
if backend_by_env_var is not None:
selected_backend = backend_name_to_enum(backend_by_env_var)
# Layer-level default (e.g. a checkpoint that requires a specific sparse
# backend). Lower precedence than the global force and the env var, so
# users can still override it.
if selected_backend is None and default_backend is not None:
selected_backend = default_backend
# get device-specific attn_backend
from fastvideo.platforms import current_platform
+4 -14
View File
@@ -2,24 +2,14 @@
from dataclasses import dataclass, field
from fastvideo.configs.models.dits.base import DiTArchConfig, DiTConfig
from fastvideo.platforms import AttentionBackendEnum
def _is_kandinsky5_transformer_block(n: str, m) -> bool:
return ("text_transformer_blocks" in n or "visual_transformer_blocks" in n) and n.split(".")[-1].isdigit()
@dataclass
class Kandinsky5ArchConfig(DiTArchConfig):
_fsdp_shard_conditions: list = field(default_factory=lambda: [_is_kandinsky5_transformer_block])
# NABLA block-sparse attention for attention_type="nabla" checkpoints, plus
# the dense backends every DiT supports.
_supported_attention_backends: tuple[AttentionBackendEnum, ...] = (
AttentionBackendEnum.NABLA_ATTN,
AttentionBackendEnum.FLASH_ATTN,
AttentionBackendEnum.TORCH_SDPA,
)
_fsdp_shard_conditions: list = field(default_factory=lambda: [
lambda n, m:
("text_transformer_blocks" in n or "visual_transformer_blocks" in n) and n.split(".")[-1].isdigit()
])
# Native FastVideo implementation uses the same parameter names as diffusers
# except FFN internals: Diffusers FFN uses `in_layer/out_layer`, while
+1 -6
View File
@@ -43,16 +43,11 @@ class TextEncoderArchConfig(EncoderArchConfig):
require_processor: bool = False
def __post_init__(self) -> None:
# update_model_arch re-runs __post_init__ after pipeline configs may
# have customized tokenizer_kwargs (e.g. kandinsky5/gen3c/longcat set
# "padding"); rebuilding the dict here would silently wipe those
# customizations, so only fill in defaults for keys not already set.
defaults = {
self.tokenizer_kwargs = {
"truncation": True,
"max_length": self.text_len,
"return_tensors": "pt",
}
self.tokenizer_kwargs = defaults | self.tokenizer_kwargs
@dataclass
+2 -3
View File
@@ -6,7 +6,6 @@ from fastvideo.configs.pipelines.hunyuan import FastHunyuanConfig, HunyuanConfig
from fastvideo.configs.pipelines.hunyuan15 import Hunyuan15T2V480PConfig, Hunyuan15T2V720PConfig
from fastvideo.configs.pipelines.hunyuangamecraft import HunyuanGameCraftPipelineConfig
from fastvideo.configs.pipelines.hyworld import HYWorldConfig
from fastvideo.configs.pipelines.kandinsky5 import Kandinsky5I2VConfig, Kandinsky5T2VConfig
from fastvideo.configs.pipelines.matrixgame2 import MatrixGame2I2V480PConfig
from fastvideo.configs.pipelines.matrixgame3 import MatrixGame3I2V720PConfig
from fastvideo.pipelines.basic.ltx2.pipeline_configs import LTX2T2VConfig
@@ -18,6 +17,6 @@ __all__ = [
"HunyuanConfig", "FastHunyuanConfig", "HunyuanGameCraftPipelineConfig", "PipelineConfig", "Hunyuan15T2V480PConfig",
"Hunyuan15T2V720PConfig", "WanT2V480PConfig", "WanI2V480PConfig", "WanT2V720PConfig", "WanI2V720PConfig",
"SelfForcingWanT2V480PConfig", "LucyEditDevConfig", "CosmosConfig", "Cosmos25Config", "LTX2T2VConfig",
"DreamXWorld5BCamPipelineConfig", "DreamXWorld5BARPipelineConfig", "HYWorldConfig", "Kandinsky5T2VConfig",
"Kandinsky5I2VConfig", "MatrixGame2I2V480PConfig", "MatrixGame3I2V720PConfig", "get_pipeline_config_cls_from_name"
"DreamXWorld5BCamPipelineConfig", "DreamXWorld5BARPipelineConfig", "HYWorldConfig", "MatrixGame2I2V480PConfig",
"MatrixGame3I2V720PConfig", "get_pipeline_config_cls_from_name"
]
-122
View File
@@ -1,122 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Any
import torch
from fastvideo.configs.models import DiTConfig, EncoderConfig, VAEConfig
from fastvideo.configs.models.dits import Kandinsky5VideoConfig
from fastvideo.configs.models.encoders import BaseEncoderOutput, CLIPTextConfig
from fastvideo.configs.models.encoders.reason1 import Reason1Config
from fastvideo.configs.models.vaes import HunyuanVAEConfig
from fastvideo.configs.pipelines.base import PipelineConfig, preprocess_text
# Byte-exact copy of the upstream Kandinsky5/diffusers template, including the
# "promt"/"scren" typos: the checkpoints were trained with this exact system # codespell:ignore promt,scren
# prompt, and ENCODE_START_IDX below is the tokenized length of everything
# before the user prompt. Fixing the typos shifts user content to index 127
# and mis-conditions every generation.
KANDINSKY5_PROMPT_TEMPLATE = "\n".join([
"<|im_start|>system\nYou are a promt engineer. Describe the video in detail.", # codespell:ignore promt
"Describe how the camera moves or shakes, describe the zoom and view angle, whether it follows the objects.",
"Describe the location of the video, main characters or objects and their action.",
"Describe the dynamism of the video and presented actions.",
"Name the visual style of the video: whether it is a professional footage, user generated content, some kind of animation, video game or scren content.", # codespell:ignore scren
"Describe the visual effects, postprocessing and transitions if they are presented in the video.",
"Pay attention to the order of key actions shown in the scene.<|im_end|>",
"<|im_start|>user\n{}<|im_end|>",
])
KANDINSKY5_PROMPT_TEMPLATE_ENCODE_START_IDX = 129
def kandinsky5_qwen_preprocess_text(prompt: str) -> str:
if not prompt.strip():
prompt = "."
return KANDINSKY5_PROMPT_TEMPLATE.format(prompt)
def kandinsky5_qwen_postprocess_text(outputs: BaseEncoderOutput,
mask: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
if outputs.hidden_states is None:
raise RuntimeError("Kandinsky5 Qwen prompt embeddings require hidden_states.")
hidden_states = outputs.hidden_states[-1]
prompt_embeds = hidden_states[:, KANDINSKY5_PROMPT_TEMPLATE_ENCODE_START_IDX:]
mask = mask[:, KANDINSKY5_PROMPT_TEMPLATE_ENCODE_START_IDX:]
if prompt_embeds.shape[1] == 0:
prompt_embeds = hidden_states[:, -1:]
mask = torch.ones((mask.shape[0], 1), dtype=mask.dtype, device=mask.device)
return prompt_embeds, mask
def kandinsky5_clip_postprocess_text(outputs: BaseEncoderOutput) -> torch.Tensor:
if outputs.pooler_output is None:
raise RuntimeError("Kandinsky5 CLIP pooled output is required.")
return outputs.pooler_output
@dataclass
class Kandinsky5T2VConfig(PipelineConfig):
"""Kandinsky-5.0 Lite text-to-video pipeline configuration."""
dit_config: DiTConfig = field(default_factory=Kandinsky5VideoConfig)
vae_config: VAEConfig = field(default_factory=HunyuanVAEConfig)
text_encoder_configs: tuple[EncoderConfig, ...] = field(default_factory=lambda: (Reason1Config(), CLIPTextConfig()))
preprocess_text_funcs: tuple[Callable[[str], Any], ...] = field(
default_factory=lambda: (kandinsky5_qwen_preprocess_text, preprocess_text))
postprocess_text_funcs: tuple[Callable[..., Any], ...] = field(
default_factory=lambda: (kandinsky5_qwen_postprocess_text, kandinsky5_clip_postprocess_text))
dit_precision: str = "bf16"
vae_precision: str = "bf16"
text_encoder_precisions: tuple[str, ...] = field(default_factory=lambda: ("bf16", "bf16"))
text_encoder_max_lengths: tuple[int, ...] = field(
default_factory=lambda: (KANDINSKY5_PROMPT_TEMPLATE_ENCODE_START_IDX + 512, 77))
flow_shift: float | None = 5.0
vae_tiling: bool = True
def __post_init__(self) -> None:
if len(self.text_encoder_configs) != 2:
raise ValueError(f"Kandinsky5 pipeline requires exactly 2 text encoders (qwen and clip), "
f"but got {len(self.text_encoder_configs)} encoder(s).")
if len(self.text_encoder_precisions) != 2:
raise ValueError("Kandinsky5 pipeline requires exactly 2 text encoder precisions, "
f"but got {len(self.text_encoder_precisions)}.")
if len(self.text_encoder_max_lengths) != 2:
raise ValueError("Kandinsky5 pipeline requires exactly 2 text encoder max lengths, "
f"but got {len(self.text_encoder_max_lengths)}.")
self.vae_config.load_encoder = False
self.vae_config.load_decoder = True
qwen_cfg = self.text_encoder_configs[0]
qwen_cfg.arch_config.output_hidden_states = True
qwen_cfg.arch_config.tokenizer_kwargs.update({
"padding": True,
"truncation": True,
"return_tensors": "pt",
})
clip_cfg = self.text_encoder_configs[1]
clip_cfg.arch_config.tokenizer_kwargs.update({
"padding": "max_length",
"max_length": 77,
"truncation": True,
"add_special_tokens": True,
"return_tensors": "pt",
})
@dataclass
class Kandinsky5I2VConfig(Kandinsky5T2VConfig):
"""Kandinsky-5.0 image-to-video pipeline configuration."""
def __post_init__(self) -> None:
super().__post_init__()
# I2V needs the VAE encoder to encode the conditioning image.
self.vae_config.load_encoder = True
-8
View File
@@ -23,7 +23,6 @@ If you only need to use the distributed environment without model parallelism,
you can skip the model parallel initialization and destruction steps.
"""
import contextlib
import gc
import os
import pickle
import weakref
@@ -1007,13 +1006,6 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
if shutdown_ray:
import ray # Lazy import Ray
ray.shutdown()
# Actually free GPU memory, as the name promises. FSDP-wrapped modules sit
# in reference cycles (module <-> FSDP state <-> hooks), so dropping the
# last user reference does not free their parameters until a gc pass runs.
# Without this, back-to-back model-load tests in one pytest session OOM.
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
def get_same_node_ranks(pg: ProcessGroup | StatelessProcessGroup, source_rank: int = 0) -> list[int]:
+11 -108
View File
@@ -437,7 +437,6 @@ class CausalWanTransformer3DModel(BaseDiT):
# Causal-specific
self.block_mask = None
self.teacher_forcing_block_mask = None
self.num_frame_per_block = config.arch_config.num_frames_per_block
assert self.num_frame_per_block <= 3
self.independent_first_frame = False
@@ -501,70 +500,6 @@ class CausalWanTransformer3DModel(BaseDiT):
return block_mask
@staticmethod
def _prepare_teacher_forcing_mask(
device: torch.device | str, num_frames: int = 21,
frame_seqlen: int = 1560, num_frame_per_block=1, local_attn_size=-1
) -> BlockMask:
"""Attention mask for the teacher-forcing ``[clean | noisy]`` sequence.
A noisy token attends to its own block plus the clean context of all
strictly previous blocks; clean tokens are block-wise causal.
"""
if local_attn_size != -1:
raise NotImplementedError(
f"Teacher forcing ignores local_attn_size={local_attn_size}: "
"unlike the block-wise causal mask, this mask always attends "
"to the full clean context. Windowed teacher forcing is not "
"implemented; use local_attn_size=-1 for teacher-forcing "
"training.")
total_length = num_frames * frame_seqlen * 2
padded_length = math.ceil(total_length / 128) * 128 - total_length
clean_ends = num_frames * frame_seqlen
context_ends = torch.zeros(total_length + padded_length, device=device, dtype=torch.long)
noise_context_starts = torch.zeros(total_length + padded_length, device=device, dtype=torch.long)
noise_context_ends = torch.zeros(total_length + padded_length, device=device, dtype=torch.long)
noise_noise_starts = torch.zeros(total_length + padded_length, device=device, dtype=torch.long)
noise_noise_ends = torch.zeros(total_length + padded_length, device=device, dtype=torch.long)
attention_block_size = frame_seqlen * num_frame_per_block
frame_indices = torch.arange(
start=0, end=num_frames * frame_seqlen,
step=attention_block_size, device=device, dtype=torch.long
)
for start in frame_indices:
context_ends[start:start + attention_block_size] = start + attention_block_size
noisy_image_start_list = torch.arange(
num_frames * frame_seqlen, total_length,
step=attention_block_size, device=device, dtype=torch.long
)
noisy_image_end_list = noisy_image_start_list + attention_block_size
for block_index, (start, end) in enumerate(zip(noisy_image_start_list, noisy_image_end_list)):
noise_noise_starts[start:end] = start
noise_noise_ends[start:end] = end
noise_context_ends[start:end] = block_index * attention_block_size
def attention_mask(b, h, q_idx, kv_idx):
clean_mask = (q_idx < clean_ends) & (kv_idx < context_ends[q_idx])
c1 = (kv_idx < noise_noise_ends[q_idx]) & (kv_idx >= noise_noise_starts[q_idx])
c2 = (kv_idx < noise_context_ends[q_idx]) & (kv_idx >= noise_context_starts[q_idx])
noise_mask = (q_idx >= clean_ends) & (c1 | c2)
eye_mask = q_idx == kv_idx
return eye_mask | clean_mask | noise_mask
block_mask = create_block_mask(
attention_mask, B=None, H=None,
Q_LEN=total_length + padded_length, KV_LEN=total_length + padded_length,
_compile=False, device=device)
if not dist.is_initialized() or dist.get_rank() == 0:
print(f" cache a teacher-forcing mask with block size of {num_frame_per_block} frames")
print(block_mask)
return block_mask
def _forward_inference(
self,
hidden_states: torch.Tensor,
@@ -693,12 +628,9 @@ class CausalWanTransformer3DModel(BaseDiT):
encoder_hidden_states_image: torch.Tensor | list[torch.Tensor]
| None = None,
start_frame: int = 0,
clean_x: torch.Tensor | None = None,
aug_t: torch.Tensor | None = None,
**kwargs) -> torch.Tensor:
orig_dtype = hidden_states.dtype
teacher_forcing = clean_x is not None
if not isinstance(encoder_hidden_states, torch.Tensor):
encoder_hidden_states = encoder_hidden_states[0]
if isinstance(encoder_hidden_states_image,
@@ -731,26 +663,15 @@ class CausalWanTransformer3DModel(BaseDiT):
freqs_cis = (freqs_cos,
freqs_sin) if freqs_cos is not None else None
if teacher_forcing:
if self.teacher_forcing_block_mask is None:
self.teacher_forcing_block_mask = self._prepare_teacher_forcing_mask(
device=hidden_states.device,
num_frames=num_frames,
frame_seqlen=post_patch_height * post_patch_width,
num_frame_per_block=self.num_frame_per_block,
local_attn_size=self.local_attn_size,
)
block_mask = self.teacher_forcing_block_mask
else:
if self.block_mask is None:
self.block_mask = self._prepare_blockwise_causal_attn_mask(
device=hidden_states.device,
num_frames=num_frames,
frame_seqlen=post_patch_height * post_patch_width,
num_frame_per_block=self.num_frame_per_block,
local_attn_size=self.local_attn_size
)
block_mask = self.block_mask
# Construct blockwise causal attn mask
if self.block_mask is None:
self.block_mask = self._prepare_blockwise_causal_attn_mask(
device=hidden_states.device,
num_frames=num_frames,
frame_seqlen=post_patch_height * post_patch_width,
num_frame_per_block=self.num_frame_per_block,
local_attn_size=self.local_attn_size
)
hidden_states = self.patch_embedding(hidden_states)
grid_sizes = torch.stack(
@@ -758,7 +679,6 @@ class CausalWanTransformer3DModel(BaseDiT):
hidden_states = hidden_states.flatten(2).transpose(1, 2)
encoder_hidden_states = torch.cat([encoder_hidden_states, encoder_hidden_states.new_zeros(1, self.text_len - encoder_hidden_states.size(1), encoder_hidden_states.size(2))], dim=1)
encoder_hidden_states_text = encoder_hidden_states
temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = self.condition_embedder(
timestep.flatten(), encoder_hidden_states, encoder_hidden_states_image)
@@ -774,35 +694,18 @@ class CausalWanTransformer3DModel(BaseDiT):
assert encoder_hidden_states.dtype == orig_dtype
if teacher_forcing:
# Tile RoPE/modulation so clean frame i and noisy frame i share a position.
clean_tokens = self.patch_embedding(clean_x).flatten(2).transpose(1, 2)
hidden_states = torch.cat([clean_tokens, hidden_states], dim=1)
if aug_t is None:
aug_t = torch.zeros_like(timestep)
_, timestep_proj_clean, _, _ = self.condition_embedder(
aug_t.flatten(), encoder_hidden_states_text, None)
timestep_proj_clean = timestep_proj_clean.unflatten(
1, (6, self.hidden_size)).unflatten(dim=0, sizes=timestep.shape)
timestep_proj = torch.cat([timestep_proj_clean, timestep_proj], dim=1)
freqs_cis = (torch.cat([freqs_cos, freqs_cos], dim=0),
torch.cat([freqs_sin, freqs_sin], dim=0))
# 4. Transformer blocks
if torch.is_grad_enabled() and self.gradient_checkpointing:
for block in self.blocks:
hidden_states = self._gradient_checkpointing_func(
block, hidden_states, encoder_hidden_states,
timestep_proj, freqs_cis,
block_mask=block_mask)
block_mask=self.block_mask)
else:
for block in self.blocks:
hidden_states = block(hidden_states, encoder_hidden_states,
timestep_proj, freqs_cis,
block_mask=block_mask)
if teacher_forcing:
hidden_states = hidden_states[:, hidden_states.shape[1] // 2:]
block_mask=self.block_mask)
# 5. Output norm, projection & unpatchify
temb = temb.unflatten(dim=0, sizes=timestep.shape).unsqueeze(2)
+2 -2
View File
@@ -699,8 +699,8 @@ class IndividualTokenRefinerBlock(nn.Module):
if mask is None:
attn_output = self.attn(q, k, v)
else:
from fastvideo.attention.backends.sdpa import SDPAMetadataBuilder
attn_metadata = SDPAMetadataBuilder().build(
from fastvideo.attention.backends.flash_attn import FlashAttnMetadataBuilder
attn_metadata = FlashAttnMetadataBuilder().build(
current_timestep=0,
attn_mask=mask,
)
+9 -7
View File
@@ -173,11 +173,8 @@ class HYWorldDoubleStreamBlock(MMDoubleStreamBlock):
img_v_prope = img_v_prope.permute(0, 2, 1, 3) # [batch, seqlen, num_heads, head_dim]
# end hyworld
# The metadata only carries the text padding mask; the executing
# attention kernel is still whatever the layer's selector picked
# (flash-attn when installed), not necessarily torch SDPA.
from fastvideo.attention.backends.sdpa import SDPAMetadataBuilder
attn_metadata = SDPAMetadataBuilder().build(
from fastvideo.attention.backends.flash_attn import FlashAttnMetadataBuilder
attn_metadata = FlashAttnMetadataBuilder().build(
current_timestep=0,
attn_mask=encoder_attention_mask,
)
@@ -195,9 +192,14 @@ class HYWorldDoubleStreamBlock(MMDoubleStreamBlock):
)
# begin hyworld
# attention with prope (same text mask, so reuse the metadata)
# attention with prope
from fastvideo.attention.backends.flash_attn import FlashAttnMetadataBuilder
attn_metadata_prope = FlashAttnMetadataBuilder().build(
current_timestep=0,
attn_mask=encoder_attention_mask,
)
# NOTE: Do NOT pass freqs_cis to prope attention - HY-WorldPlay does not apply RoPE to prope
with set_forward_context(current_timestep=0, attn_metadata=attn_metadata):
with set_forward_context(current_timestep=0, attn_metadata=attn_metadata_prope):
img_attn_prope, _ = self.attn(
img_q_prope,
img_k_prope,
+25 -76
View File
@@ -10,21 +10,13 @@ import torch.nn as nn
import torch.nn.functional as F
from fastvideo.attention import LocalAttention
from fastvideo.attention.backends.nabla import CAN_USE_FLEX_ATTN, flex_attention, nablaT_v2
from fastvideo.configs.models.dits import Kandinsky5VideoConfig
from fastvideo.layers.layernorm import LayerNormScaleShift
from fastvideo.layers.linear import ReplicatedLinear
from fastvideo.layers.mlp import MLP
from fastvideo.logger import init_logger
from fastvideo.models.dits.base import BaseDiT
from fastvideo.platforms import AttentionBackendEnum
logger = init_logger(__name__)
if not CAN_USE_FLEX_ATTN:
logger.warning("torch.nn.attention.flex_attention is unavailable in this PyTorch build; "
"Kandinsky5 NABLA sparse attention (Pro checkpoints) cannot be used.")
FRACTAL_PIXEL_SIZE = 8
_ARCH_CONFIG_DEFAULTS = Kandinsky5VideoConfig().arch_config
@@ -271,9 +263,10 @@ class Kandinsky5Modulation(nn.Module):
def _apply_rotary(x: torch.Tensor, rope: torch.Tensor) -> torch.Tensor:
orig_dtype = x.dtype
x_ = x.reshape(*x.shape[:-1], -1, 1, 2).to(torch.float32)
x_out = (rope * x_).sum(dim=-1)
return x_out.reshape(*x.shape).to(x.dtype)
return x_out.reshape(*x.shape).to(orig_dtype)
class Kandinsky5Attention(nn.Module):
@@ -284,7 +277,6 @@ class Kandinsky5Attention(nn.Module):
head_dim: int,
supported_attention_backends: tuple[AttentionBackendEnum, ...] | None,
prefix: str = "",
use_nabla: bool = False,
):
super().__init__()
assert num_channels % head_dim == 0
@@ -314,17 +306,6 @@ class Kandinsky5Attention(nn.Module):
causal=False,
supported_attention_backends=supported_attention_backends,
)
# NABLA checkpoints get a second attention layer whose backend defaults
# to NABLA_ATTN; FASTVIDEO_ATTENTION_BACKEND still overrides it.
self.nabla_attention = None
if use_nabla:
self.nabla_attention = LocalAttention(
num_heads=self.num_heads,
head_size=head_dim,
causal=False,
supported_attention_backends=supported_attention_backends,
default_backend=AttentionBackendEnum.NABLA_ATTN,
)
def forward(
self,
@@ -358,54 +339,27 @@ class Kandinsky5Attention(nn.Module):
key = _apply_rotary(key, rotary_emb).type_as(key)
if sparse_params is not None:
if self.nabla_attention is None:
raise RuntimeError("sparse_params passed to an attention layer built without use_nabla; "
"this checkpoint/config combination is inconsistent.")
try:
# Backend impl reads sta_mask/P from the forward-context
# attention metadata built by the denoising stage.
hidden_states = self.nabla_attention(query, key, value)
except AssertionError as exc:
# Standalone parity tests call the model without a pipeline
# forward context; run the NABLA kernel directly.
if "Forward context is not set" not in str(exc):
raise
attn_mask = nablaT_v2(query, key, sparse_params["sta_mask"], thr=sparse_params["P"])
hidden_states = flex_attention(
query=query.transpose(1, 2),
key=key.transpose(1, 2),
value=value.transpose(1, 2),
block_mask=attn_mask,
).transpose(1, 2)
else:
try:
hidden_states = self.local_attention(query, key, value)
raise NotImplementedError(
"Sparse attention is not yet supported for Kandinsky5 in FastVideo."
)
except AssertionError as exc:
# LocalAttention requires pipeline forward context. Standalone
# parity tests call the model directly, so fallback to Torch SDPA.
if "Forward context is not set" not in str(exc):
raise
query_shape = query.shape[:-2]
key_shape = key.shape[:-2]
query = query.reshape(query_shape[0], -1, self.num_heads,
query.shape[-1]).transpose(1, 2)
key = key.reshape(key_shape[0], -1, self.num_heads,
key.shape[-1]).transpose(1, 2)
value = value.reshape(key_shape[0], -1, self.num_heads,
value.shape[-1]).transpose(1, 2)
hidden_states = F.scaled_dot_product_attention(
query,
key,
value,
attn_mask=None,
is_causal=False,
)
hidden_states = hidden_states.transpose(1, 2).reshape(
*query_shape, self.num_heads, -1)
hidden_states = hidden_states.flatten(-2, -1)
try:
hidden_states = self.local_attention(query, key, value)
except AssertionError as exc:
# LocalAttention requires pipeline forward context. Standalone
# parity tests call the model directly, so fallback to Torch SDPA.
if "Forward context is not set" not in str(exc):
raise
query = query.transpose(1, 2)
key = key.transpose(1, 2)
value = value.transpose(1, 2)
hidden_states = F.scaled_dot_product_attention(query,
key,
value,
attn_mask=None,
is_causal=False)
hidden_states = hidden_states.transpose(1, 2)
hidden_states = hidden_states.flatten(2)
hidden_states, _ = self.out_layer(hidden_states)
return hidden_states
@@ -522,8 +476,7 @@ class Kandinsky5TransformerDecoderBlock(nn.Module):
head_dim: int,
supported_attention_backends: tuple[AttentionBackendEnum, ...]
| None = None,
prefix: str = "",
use_nabla: bool = False):
prefix: str = ""):
super().__init__()
self.visual_modulation = Kandinsky5Modulation(time_dim, model_dim, 9)
@@ -538,8 +491,7 @@ class Kandinsky5TransformerDecoderBlock(nn.Module):
model_dim,
head_dim,
supported_attention_backends=supported_attention_backends,
prefix=f"{prefix}.self_attention",
use_nabla=use_nabla)
prefix=f"{prefix}.self_attention")
self.cross_attention_norm = LayerNormScaleShift(
model_dim,
@@ -672,8 +624,7 @@ class Kandinsky5Transformer3DModel(BaseDiT):
arch.ff_dim,
head_dim,
self._supported_attention_backends,
prefix=f"{config.prefix}.visual_transformer_blocks.{i}",
use_nabla=arch.attention_type == "nabla")
prefix=f"{config.prefix}.visual_transformer_blocks.{i}")
for i in range(arch.num_visual_blocks)
])
@@ -743,7 +694,6 @@ class Kandinsky5Transformer3DModel(BaseDiT):
scale_factor)
to_fractal = sparse_params[
"to_fractal"] if sparse_params is not None else False
visual_embed, visual_rope = fractal_flatten(visual_embed, visual_rope,
visual_shape,
block_mask=to_fractal)
@@ -774,7 +724,6 @@ class Kandinsky5Transformer3DModel(BaseDiT):
if return_dict:
return Kandinsky5TransformerOutput(sample=x)
return x
def materialize_non_persistent_buffers(self, device: torch.device,
+2 -44
View File
@@ -467,18 +467,6 @@ class Qwen2_5_VisionTransformerPretrainedModel(nn.Module):
return hidden_states
def _compute_default_rope_parameters(config, device=None, seq_len=None, **kwargs):
# transformers>=5 removes the "default" entry from ROPE_INIT_FUNCTIONS and
# moves rope_theta inside rope_parameters; replicate the 4.x default init.
rope_params = getattr(config, "rope_parameters", None) or getattr(config, "rope_scaling", None) or {}
base = rope_params.get("rope_theta", getattr(config, "rope_theta", 10000.0))
head_dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads
partial_rotary_factor = getattr(config, "partial_rotary_factor", 1.0)
dim = int(head_dim * partial_rotary_factor)
inv_freq = 1.0 / (base**(torch.arange(0, dim, 2, dtype=torch.int64).float().to(device) / dim))
return inv_freq, 1.0
class Qwen2_5_VLRotaryEmbedding(nn.Module):
def __init__(self, config: Qwen2_5_VLConfig, device=None):
super().__init__()
@@ -491,14 +479,7 @@ class Qwen2_5_VLRotaryEmbedding(nn.Module):
self.original_max_seq_len = config.max_position_embeddings
self.config = config
if self.rope_type in ROPE_INIT_FUNCTIONS:
self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
elif self.rope_type == "default":
# transformers>=5 drops the "default" entry from ROPE_INIT_FUNCTIONS.
self.rope_init_fn = _compute_default_rope_parameters
else:
raise KeyError(f"Unsupported rope_type '{self.rope_type}'; available: "
f"{['default', *ROPE_INIT_FUNCTIONS]}")
self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
self.register_buffer("inv_freq", inv_freq, persistent=False)
@@ -967,16 +948,6 @@ QWEN2_5_VL_ATTENTION_CLASSES = {
# If FlashAttention2 is not available, transparently fall back to SDPA.
if not is_flash_attn_2_available():
QWEN2_5_VL_ATTENTION_CLASSES["flash_attention_2"] = Qwen2_5_VLSdpaAttention
else:
# transformers>=5 only resolves the flash-attn functions when the model
# preloads them via its attention interface; this module bypasses
# PreTrainedModel, so _flash_attention_forward(implementation=None) raises
# unless we preload here.
try:
from transformers.modeling_flash_attention_utils import lazy_import_flash_attention
lazy_import_flash_attention("flash_attention_2")
except (ImportError, ValueError):
QWEN2_5_VL_ATTENTION_CLASSES["flash_attention_2"] = Qwen2_5_VLSdpaAttention
class Qwen2_5_VLDecoderLayer(nn.Module):
def __init__(self, config: Qwen2_5_VLConfig, layer_idx: int):
@@ -1064,7 +1035,7 @@ class Qwen2_5_VLModel(nn.Module):
def __init__(self, config: Qwen2_5_VLConfig):
super().__init__()
self.config = config
self.padding_idx = getattr(config, "pad_token_id", None)
self.padding_idx = config.pad_token_id
self.vocab_size = config.vocab_size
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
@@ -1437,18 +1408,6 @@ class Qwen2_5_VLCausalLMOutputWithPast(ModelOutput):
rope_deltas: Optional[torch.LongTensor] = None
def _flatten_text_config(config):
# transformers>=5 stops forwarding text-model attributes (hidden_size,
# vocab_size, rope_scaling, ...) from the composite Qwen2_5_VLConfig to
# config.text_config; this module reads them from the top level.
text_config = getattr(config, "text_config", None)
if text_config is not None:
for key, value in text_config.to_dict().items():
if not hasattr(config, key):
setattr(config, key, value)
return config
class Qwen2_5_VLForConditionalGenerationSimple(nn.Module):
_tied_weights_keys = ["lm_head.weight"]
config_class = Qwen2_5_VLConfig
@@ -1456,7 +1415,6 @@ class Qwen2_5_VLForConditionalGenerationSimple(nn.Module):
def __init__(self, config):
super().__init__()
config = _flatten_text_config(config)
self.config = config
self.visual = Qwen2_5_VisionTransformerPretrainedModel(config.vision_config)
+1 -16
View File
@@ -320,19 +320,6 @@ class TextEncoderLoader(ComponentLoader):
f"text encoder index {idx} out of range for text_encoder_configs (len={len(encoder_configs)}), model_path={model_path}"
)
encoder_config = encoder_configs[idx]
if (
model_config.get("architectures") == ["CLIPModel"]
and isinstance(model_config.get("text_config"), dict)
):
valid_arch_fields = {
f.name for f in dataclasses.fields(encoder_config.arch_config)
}
model_config = {
key: value
for key, value in deepcopy(model_config["text_config"]).items()
if key in valid_arch_fields
}
model_config["architectures"] = ["CLIPTextModel"]
encoder_config.update_model_arch(model_config)
if idx < 0 or idx >= len(encoder_precisions):
raise IndexError(
@@ -417,9 +404,7 @@ class TextEncoderLoader(ComponentLoader):
else:
loaded_weights: set[str] = model.load_weights(
self._get_all_weights(
model,
model_path,
to_cpu=fastvideo_args.text_encoder_cpu_offload,
model, model_path, to_cpu=use_cpu_offload
)
) # type: ignore
+1 -40
View File
@@ -1,9 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
import io
import os
import tempfile
import time
from collections.abc import Callable
from typing import Any
from urllib.parse import unquote, urlparse
@@ -18,10 +16,6 @@ import torch.nn.functional as F
import torchvision.transforms.functional as TF
from packaging import version
from fastvideo.logger import init_logger
logger = init_logger(__name__)
if version.parse(version.parse(
PIL.__version__).base_version) >= version.parse("9.1.0"):
PIL_INTERPOLATION = {
@@ -95,39 +89,6 @@ def normalize(images: np.ndarray | torch.Tensor) -> np.ndarray | torch.Tensor:
return 2.0 * images - 1.0
def _fetch_image_bytes(url: str,
attempts: int = 3,
timeout: float = 30.0) -> bytes:
"""Download ``url`` fully, retrying transient network errors.
Reads the whole body via ``response.content`` instead of handing PIL a
``.raw`` stream: a connection dropped mid-body (e.g. an HF-CDN
``IncompleteRead``) then fails here, where it can be retried, instead of
surfacing as a truncated image inside PIL.
"""
for attempt in range(attempts):
try:
response = requests.get(url, timeout=timeout)
response.raise_for_status()
return response.content
except requests.RequestException as e:
status = getattr(getattr(e, "response", None), "status_code",
None)
# Permanent client errors (4xx except 408/429) won't heal on
# retry — fail fast.
if (status is not None and 400 <= status < 500
and status not in (408, 429)):
raise
if attempt == attempts - 1:
raise
backoff = 2**attempt
logger.warning(
"Failed to download image %s (attempt %d/%d): %s. "
"Retrying in %ds.", url, attempt + 1, attempts, e, backoff)
time.sleep(backoff)
raise AssertionError("unreachable")
# adapted from diffusers.utils import load_image
def load_image(
image: str | PIL.Image.Image,
@@ -149,7 +110,7 @@ def load_image(
"""
if isinstance(image, str):
if image.startswith("http://") or image.startswith("https://"):
image = PIL.Image.open(io.BytesIO(_fetch_image_bytes(image)))
image = PIL.Image.open(requests.get(image, stream=True).raw)
elif os.path.isfile(image):
image = PIL.Image.open(image)
else:
-2
View File
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Performance benchmark and dashboard utilities."""
-116
View File
@@ -1,116 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Metric policy for rolling performance baseline comparisons."""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any
@dataclass(frozen=True)
class MetricPolicy:
key: str
label: str
precision: int
lower_is_better: bool
threshold_percent: float
threshold_absolute: float
gated: bool = True
@dataclass(frozen=True)
class MetricDelta:
absolute: float
percent: float
threshold_exceeded: bool
regressed: bool
DEFAULT_METRIC_POLICIES: tuple[MetricPolicy, ...] = (
MetricPolicy("latency", "Latency", 3, True, 0.08, 0.5),
MetricPolicy("throughput", "Throughput", 3, False, 0.08, 0.05),
MetricPolicy("memory", "Memory", 1, True, 0.05, 256.0),
MetricPolicy("text_encoder_time_s", "Text Enc", 3, True, 0.05, 0.25),
MetricPolicy("dit_time_s", "DiT", 3, True, 0.05, 0.25),
MetricPolicy("vae_decode_time_s", "VAE Decode", 3, True, 0.05, 0.25),
)
def _optional_float(value: Any) -> float | None:
if value is None or isinstance(value, bool):
return None
try:
return float(value)
except (TypeError, ValueError):
return None
def _optional_bool(value: Any) -> bool | None:
if isinstance(value, bool):
return value
if isinstance(value, str):
normalized = value.strip().lower()
if normalized in {"1", "true", "yes", "on"}:
return True
if normalized in {"0", "false", "no", "off"}:
return False
return None
def resolve_metric_policies(threshold_overrides: Mapping[str, Any] | None, ) -> tuple[MetricPolicy, ...]:
"""Return default metric policies with optional per-metric overrides."""
if not isinstance(threshold_overrides, Mapping):
threshold_overrides = {}
policies: list[MetricPolicy] = []
for base_policy in DEFAULT_METRIC_POLICIES:
raw_override = threshold_overrides.get(base_policy.key, {})
if not isinstance(raw_override, Mapping):
raw_override = {}
threshold_percent = _optional_float(raw_override.get("threshold_percent"))
threshold_absolute = _optional_float(raw_override.get("threshold_absolute"))
gated = _optional_bool(raw_override.get("gated"))
policies.append(
MetricPolicy(
key=base_policy.key,
label=base_policy.label,
precision=base_policy.precision,
lower_is_better=base_policy.lower_is_better,
threshold_percent=(base_policy.threshold_percent if threshold_percent is None else threshold_percent),
threshold_absolute=(base_policy.threshold_absolute
if threshold_absolute is None else threshold_absolute),
gated=base_policy.gated if gated is None else gated,
))
return tuple(policies)
def serialize_metric_thresholds(policies: tuple[MetricPolicy, ...], ) -> dict[str, dict[str, float | bool]]:
return {
policy.key: {
"threshold_percent": policy.threshold_percent,
"threshold_absolute": policy.threshold_absolute,
"gated": policy.gated,
}
for policy in policies
}
def regression_delta(
policy: MetricPolicy,
current: float,
baseline: float,
) -> MetricDelta | None:
if baseline <= 0:
return None
absolute_delta = current - baseline if policy.lower_is_better else baseline - current
percent_delta = absolute_delta / baseline
threshold_exceeded = (percent_delta > policy.threshold_percent and absolute_delta > policy.threshold_absolute)
return MetricDelta(
absolute=absolute_delta,
percent=percent_delta,
threshold_exceeded=threshold_exceeded,
regressed=policy.gated and threshold_exceeded,
)
+2 -1
View File
@@ -13,7 +13,7 @@ from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse
from fastapi.staticfiles import StaticFiles
from fastvideo.performance import hf_store
from fastvideo.tests.performance import hf_store
from .service import build_latest_summary, build_trends, filter_records
@@ -150,6 +150,7 @@ def create_app(store: PerformanceDataStore | None = None) -> FastAPI:
filtered = filter_records(loaded, model_id=model_id, gpu_type=gpu_type)
rows = build_latest_summary(
filtered,
max_regression=float(os.environ.get("PERF_MAX_REGRESSION", "0.05")),
run_source=run_source,
)
return {
+20 -2
View File
@@ -1,8 +1,26 @@
# SPDX-License-Identifier: Apache-2.0
"""Metric definitions shared by the performance dashboard backend."""
from fastvideo.performance.metric_policy import DEFAULT_METRIC_POLICIES
from __future__ import annotations
METRICS = DEFAULT_METRIC_POLICIES
from dataclasses import dataclass
@dataclass(frozen=True)
class MetricDefinition:
key: str
label: str
precision: int
lower_is_better: bool
METRICS: tuple[MetricDefinition, ...] = (
MetricDefinition("latency", "Latency", 3, True),
MetricDefinition("throughput", "Throughput", 3, False),
MetricDefinition("memory", "Memory", 1, True),
MetricDefinition("text_encoder_time_s", "Text Encoder", 3, True),
MetricDefinition("dit_time_s", "DiT", 3, True),
MetricDefinition("vae_decode_time_s", "VAE Decode", 3, True),
)
METRIC_BY_KEY = {metric.key: metric for metric in METRICS}
+56 -108
View File
@@ -13,19 +13,11 @@ from collections import defaultdict
from datetime import datetime, timezone
from typing import Any
from fastvideo.performance.hf_store import is_baseline_eligible_record, safe_float
from fastvideo.performance.metric_policy import regression_delta, resolve_metric_policies
from fastvideo.tests.performance.hf_store import is_baseline_eligible_record, safe_float
from .metrics import METRICS
Record = dict[str, Any]
CohortKey = tuple[str, str, str, str, str, str, str, str]
COMPARISON_COHORT_KEYS = (
"workload_id",
"variant_id",
"benchmark_version",
"recipe_fingerprint",
"hardware_profile_id",
"software_profile_id",
)
def parse_timestamp(value: Any) -> datetime | None:
@@ -86,56 +78,15 @@ def record_metadata(record: Record) -> Record:
}
def _cohort_metadata_value(value: Any) -> Any:
if value is None:
return ""
if isinstance(value, str) and not value.strip():
return ""
return value
def _cohort_key_value(value: Any) -> str:
return str(_cohort_metadata_value(value))
def record_comparison_metadata(record: Record) -> Record:
return {key: _cohort_metadata_value(record.get(key)) for key in COMPARISON_COHORT_KEYS}
def comparison_cohort_key(record: Record) -> CohortKey:
return (
str(record.get("model_id") or "unknown"),
str(record.get("gpu_type") or "unknown"),
_cohort_key_value(record.get("workload_id")),
_cohort_key_value(record.get("variant_id")),
_cohort_key_value(record.get("benchmark_version")),
_cohort_key_value(record.get("recipe_fingerprint")),
_cohort_key_value(record.get("hardware_profile_id")),
_cohort_key_value(record.get("software_profile_id")),
)
def group_by_comparison_cohort(records: list[Record]) -> dict[CohortKey, list[Record]]:
groups: dict[CohortKey, list[Record]] = defaultdict(list)
def group_by_model_gpu(records: list[Record]) -> dict[tuple[str, str], list[Record]]:
groups: dict[tuple[str, str], list[Record]] = defaultdict(list)
for record in records:
groups[comparison_cohort_key(record)].append(record)
model_id = str(record.get("model_id") or "unknown")
gpu_type = str(record.get("gpu_type") or "unknown")
groups[(model_id, gpu_type)].append(record)
return {key: sorted(value, key=record_sort_key) for key, value in groups.items()}
def comparison_sort_key(record: Record) -> CohortKey:
return comparison_cohort_key(record)
def latest_row_sort_key(row: Record) -> tuple[Any, ...]:
return (row["status"] != "fail", *comparison_sort_key(row))
def group_identity(key: CohortKey) -> tuple[str, str]:
model_id = key[0]
gpu_type = key[1]
return model_id, gpu_type
def baseline_value(records: list[Record], metric_key: str) -> float | None:
values = [safe_float(record.get(metric_key)) for record in records]
values = [value for value in values if value is not None]
@@ -144,13 +95,22 @@ def baseline_value(records: list[Record], metric_key: str) -> float | None:
return float(statistics.median(values))
def regression_percent(metric_key: str, current: float | None, baseline: float | None) -> float | None:
if current is None or baseline is None or baseline <= 0:
return None
metric = next(metric for metric in METRICS if metric.key == metric_key)
if metric.lower_is_better:
return (current - baseline) / baseline * 100.0
return (baseline - current) / baseline * 100.0
def build_latest_summary(records: list[Record],
*,
baseline_window: int = 5,
max_regression: float = 0.05,
run_source: str | None = None) -> list[Record]:
rows: list[Record] = []
for key, group in group_by_comparison_cohort(records).items():
model_id, gpu_type = group_identity(key)
for (model_id, gpu_type), group in group_by_model_gpu(records).items():
latest_candidates = group
if run_source:
latest_candidates = [record for record in group if record_run_source(record) == run_source]
@@ -158,93 +118,81 @@ def build_latest_summary(records: list[Record],
continue
latest = latest_candidates[-1]
latest_index = next(index for index, record in enumerate(group) if record is latest)
baseline_pool = [
record for record in group[:latest_index]
if record.get("success", True) and is_baseline_eligible_record(record)
record for record in group
if record is not latest and record.get("success", True) and is_baseline_eligible_record(record)
]
baseline_records = baseline_pool[-baseline_window:]
metric_policies = resolve_metric_policies(latest.get("regression_thresholds"))
metrics: dict[str, Record] = {}
regressions: list[float] = []
failing_metrics: list[str] = []
threshold_exceeded_metrics: list[str] = []
for policy in metric_policies:
current = safe_float(latest.get(policy.key))
baseline = baseline_value(baseline_records, policy.key)
delta = None
if current is not None and baseline is not None:
delta = regression_delta(policy, current, baseline)
regression = None if delta is None else delta.percent * 100.0
metrics[policy.key] = {
for metric in METRICS:
current = safe_float(latest.get(metric.key))
baseline = baseline_value(baseline_records, metric.key)
regression = regression_percent(metric.key, current, baseline)
metrics[metric.key] = {
"current": current,
"baseline": baseline,
"regression_pct": regression,
"absolute_delta": None if delta is None else delta.absolute,
"threshold_percent": policy.threshold_percent * 100.0,
"threshold_absolute": policy.threshold_absolute,
"gated": policy.gated,
"threshold_exceeded": False if delta is None else delta.threshold_exceeded,
"regressed": False if delta is None else delta.regressed,
"label": policy.label,
"lower_is_better": policy.lower_is_better,
"precision": policy.precision,
"label": metric.label,
"lower_is_better": metric.lower_is_better,
"precision": metric.precision,
}
if regression is not None:
regressions.append(regression)
if delta is not None and delta.threshold_exceeded:
threshold_exceeded_metrics.append(policy.key)
if delta is not None and delta.regressed:
failing_metrics.append(policy.key)
worst_regression = max(regressions) if regressions else None
success = bool(latest.get("success", True))
status = "pass" if success else "fail"
rows.append({
"model_id": model_id,
"gpu_type": gpu_type,
"timestamp": latest.get("timestamp"),
"commit_sha": latest.get("commit_sha"),
"model_id":
model_id,
"gpu_type":
gpu_type,
"timestamp":
latest.get("timestamp"),
"commit_sha":
latest.get("commit_sha"),
**record_metadata(latest),
**record_comparison_metadata(latest),
"success": success,
"baseline_n": len(baseline_records),
"worst_regression_pct": worst_regression,
"threshold_exceeded_metrics": threshold_exceeded_metrics,
"failing_metrics": failing_metrics,
"computed_regression_status": "fail" if failing_metrics else "pass",
"status": status,
"metrics": metrics,
"success":
success,
"baseline_n":
len(baseline_records),
"worst_regression_pct":
worst_regression,
"regression_threshold_pct":
max_regression * 100.0,
"computed_regression_status":
"fail" if worst_regression is not None and worst_regression > max_regression * 100.0 else "pass",
"status":
status,
"metrics":
metrics,
})
return sorted(rows, key=latest_row_sort_key)
return sorted(rows, key=lambda row: (row["status"] != "fail", row["model_id"], row["gpu_type"]))
def build_trends(records: list[Record]) -> list[Record]:
trends: list[Record] = []
for key, group in group_by_comparison_cohort(records).items():
model_id, gpu_type = group_identity(key)
for (model_id, gpu_type), group in group_by_model_gpu(records).items():
points = []
for record in group:
metric_policies = resolve_metric_policies(record.get("regression_thresholds"))
point = {
"timestamp": record.get("timestamp"),
"commit_sha": record.get("commit_sha"),
**record_metadata(record),
**record_comparison_metadata(record),
"success": bool(record.get("success", True)),
"metrics": {
policy.key: safe_float(record.get(policy.key))
for policy in metric_policies
metric.key: safe_float(record.get(metric.key))
for metric in METRICS
},
}
points.append(point)
trends.append({
"model_id": model_id,
"gpu_type": gpu_type,
**record_comparison_metadata(group[-1]),
"points": points,
})
return sorted(trends, key=comparison_sort_key)
return sorted(trends, key=lambda trend: (trend["model_id"], trend["gpu_type"]))
@@ -1 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
@@ -1,89 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
from fastvideo.fastvideo_args import FastVideoArgs
from fastvideo.pipelines.composed_pipeline_base import ComposedPipelineBase
from fastvideo.pipelines.stages.input_validation import InputValidationStage
from fastvideo.pipelines.stages.kandinsky5 import (
Kandinsky5DecodingStage,
Kandinsky5DenoisingStage,
Kandinsky5ImageEncodingStage,
Kandinsky5LatentPreparationStage,
Kandinsky5NormalizationStage,
)
from fastvideo.pipelines.stages.text_encoding import TextEncodingStage
from fastvideo.pipelines.stages.timestep_preparation import TimestepPreparationStage
class Kandinsky5I2VPipeline(ComposedPipelineBase):
"""Kandinsky-5.0 image-to-video pipeline."""
_required_config_modules = [
"scheduler",
"text_encoder",
"text_encoder_2",
"tokenizer",
"tokenizer_2",
"transformer",
"vae",
]
def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None:
self.add_stage(stage_name="input_validation_stage", stage=InputValidationStage())
self.add_stage(
stage_name="text_encoding_stage",
stage=TextEncodingStage(
text_encoders=[
self.get_module("text_encoder"),
self.get_module("text_encoder_2"),
],
tokenizers=[
self.get_module("tokenizer"),
self.get_module("tokenizer_2"),
],
),
)
self.add_stage(
stage_name="timestep_preparation_stage",
stage=TimestepPreparationStage(scheduler=self.get_module("scheduler")),
)
self.add_stage(
stage_name="latent_preparation_stage",
stage=Kandinsky5LatentPreparationStage(
scheduler=self.get_module("scheduler"),
transformer=self.get_module("transformer"),
),
)
# Runs AFTER latent preparation so the initial noise is the seeded
# generator's first draw (official kandinsky-5 RNG order); this stage
# then samples the image latent and places it into the latents.
self.add_stage(
stage_name="image_encoding_stage",
stage=Kandinsky5ImageEncodingStage(vae=self.get_module("vae")),
)
self.add_stage(
stage_name="denoising_stage",
stage=Kandinsky5DenoisingStage(
transformer=self.get_module("transformer"),
scheduler=self.get_module("scheduler"),
),
)
self.add_stage(
stage_name="normalization_stage",
stage=Kandinsky5NormalizationStage(),
)
self.add_stage(
stage_name="decoding_stage",
stage=Kandinsky5DecodingStage(vae=self.get_module("vae"), pipeline=self),
)
EntryClass = Kandinsky5I2VPipeline
@@ -1,74 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
from fastvideo.fastvideo_args import FastVideoArgs
from fastvideo.pipelines.composed_pipeline_base import ComposedPipelineBase
from fastvideo.pipelines.stages.input_validation import InputValidationStage
from fastvideo.pipelines.stages.kandinsky5 import (
Kandinsky5DecodingStage,
Kandinsky5DenoisingStage,
Kandinsky5LatentPreparationStage,
)
from fastvideo.pipelines.stages.text_encoding import TextEncodingStage
from fastvideo.pipelines.stages.timestep_preparation import TimestepPreparationStage
class Kandinsky5T2VPipeline(ComposedPipelineBase):
"""Kandinsky-5.0 Lite text-to-video pipeline."""
_required_config_modules = [
"scheduler",
"text_encoder",
"text_encoder_2",
"tokenizer",
"tokenizer_2",
"transformer",
"vae",
]
def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None:
self.add_stage(stage_name="input_validation_stage", stage=InputValidationStage())
self.add_stage(
stage_name="text_encoding_stage",
stage=TextEncodingStage(
text_encoders=[
self.get_module("text_encoder"),
self.get_module("text_encoder_2"),
],
tokenizers=[
self.get_module("tokenizer"),
self.get_module("tokenizer_2"),
],
),
)
self.add_stage(
stage_name="timestep_preparation_stage",
stage=TimestepPreparationStage(scheduler=self.get_module("scheduler")),
)
self.add_stage(
stage_name="latent_preparation_stage",
stage=Kandinsky5LatentPreparationStage(
scheduler=self.get_module("scheduler"),
transformer=self.get_module("transformer"),
),
)
self.add_stage(
stage_name="denoising_stage",
stage=Kandinsky5DenoisingStage(
transformer=self.get_module("transformer"),
scheduler=self.get_module("scheduler"),
),
)
self.add_stage(
stage_name="decoding_stage",
stage=Kandinsky5DecodingStage(vae=self.get_module("vae"), pipeline=self),
)
EntryClass = Kandinsky5T2VPipeline
@@ -1,165 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Kandinsky-5 model family pipeline presets."""
from fastvideo.api.presets import InferencePreset, PresetStageSpec
_NEGATIVE_PROMPT = ("Static, 2D cartoon, cartoon, 2d animation, paintings, images, worst quality, low quality, ugly, "
"deformed, walking backwards")
_DENOISE_STAGE = PresetStageSpec(
name="denoise",
kind="denoising",
description="Main Kandinsky-5 denoising pass",
allowed_overrides=frozenset({
"num_inference_steps",
"guidance_scale",
}),
)
KANDINSKY5_T2V_LITE_5S = InferencePreset(
name="kandinsky5_t2v_lite_5s",
version=1,
model_family="kandinsky5",
description="Kandinsky-5.0 Lite T2V 5s",
workload_type="t2v",
stage_schemas=(_DENOISE_STAGE, ),
defaults={
"height": 512,
"width": 768,
"num_frames": 121,
"fps": 24,
"guidance_scale": 5.0,
"num_inference_steps": 50,
"negative_prompt": _NEGATIVE_PROMPT,
},
)
KANDINSKY5_T2V_LITE_DISTILLED_5S = InferencePreset(
name="kandinsky5_t2v_lite_distilled_5s",
version=1,
model_family="kandinsky5",
description="Kandinsky-5.0 Lite T2V Distilled 5s",
workload_type="t2v",
stage_schemas=(_DENOISE_STAGE, ),
defaults={
"height": 512,
"width": 768,
"num_frames": 121,
"fps": 24,
"guidance_scale": 1.0,
"num_inference_steps": 16,
"negative_prompt": _NEGATIVE_PROMPT,
},
)
KANDINSKY5_T2V_PRO_5S = InferencePreset(
name="kandinsky5_t2v_pro_5s",
version=1,
model_family="kandinsky5",
description="Kandinsky-5.0 Pro T2V 5s",
workload_type="t2v",
stage_schemas=(_DENOISE_STAGE, ),
defaults={
"height": 512,
"width": 768,
"num_frames": 121,
"fps": 24,
"guidance_scale": 5.0,
"num_inference_steps": 50,
"negative_prompt": _NEGATIVE_PROMPT,
},
)
KANDINSKY5_T2V_PRO_DISTILLED_5S = InferencePreset(
name="kandinsky5_t2v_pro_distilled_5s",
version=1,
model_family="kandinsky5",
description="Kandinsky-5.0 Pro T2V Distilled 5s",
workload_type="t2v",
stage_schemas=(_DENOISE_STAGE, ),
defaults={
"height": 512,
"width": 768,
"num_frames": 121,
"fps": 24,
"guidance_scale": 1.0,
"num_inference_steps": 16,
"negative_prompt": _NEGATIVE_PROMPT,
},
)
KANDINSKY5_I2V_LITE_5S = InferencePreset(
name="kandinsky5_i2v_lite_5s",
version=1,
model_family="kandinsky5",
description="Kandinsky-5.0 Lite I2V 5s",
workload_type="i2v",
stage_schemas=(_DENOISE_STAGE, ),
defaults={
"height": 512,
"width": 768,
"num_frames": 121,
"fps": 24,
"guidance_scale": 5.0,
"num_inference_steps": 50,
"negative_prompt": _NEGATIVE_PROMPT,
},
)
KANDINSKY5_I2V_PRO_5S = InferencePreset(
name="kandinsky5_i2v_pro_5s",
version=1,
model_family="kandinsky5",
description="Kandinsky-5.0 Pro I2V 5s",
workload_type="i2v",
stage_schemas=(_DENOISE_STAGE, ),
defaults={
"height": 512,
"width": 768,
"num_frames": 121,
"fps": 24,
"guidance_scale": 5.0,
"num_inference_steps": 50,
"negative_prompt": _NEGATIVE_PROMPT,
},
)
KANDINSKY5_I2V_LITE_DISTILLED_5S = InferencePreset(
name="kandinsky5_i2v_lite_distilled_5s",
version=1,
model_family="kandinsky5",
description="Kandinsky-5.0 Lite I2V Distilled 5s",
workload_type="i2v",
stage_schemas=(_DENOISE_STAGE, ),
defaults={
"height": 512,
"width": 768,
"num_frames": 121,
"fps": 24,
"guidance_scale": 1.0,
"num_inference_steps": 16,
"negative_prompt": _NEGATIVE_PROMPT,
},
)
KANDINSKY5_I2V_PRO_DISTILLED_5S = InferencePreset(
name="kandinsky5_i2v_pro_distilled_5s",
version=1,
model_family="kandinsky5",
description="Kandinsky-5.0 Pro I2V Distilled 5s",
workload_type="i2v",
stage_schemas=(_DENOISE_STAGE, ),
defaults={
"height": 512,
"width": 768,
"num_frames": 121,
"fps": 24,
"guidance_scale": 1.0,
"num_inference_steps": 16,
"negative_prompt": _NEGATIVE_PROMPT,
},
)
ALL_PRESETS = (KANDINSKY5_T2V_LITE_5S, KANDINSKY5_T2V_LITE_DISTILLED_5S, KANDINSKY5_T2V_PRO_5S,
KANDINSKY5_T2V_PRO_DISTILLED_5S, KANDINSKY5_I2V_LITE_5S, KANDINSKY5_I2V_LITE_DISTILLED_5S,
KANDINSKY5_I2V_PRO_5S, KANDINSKY5_I2V_PRO_DISTILLED_5S)
+18 -40
View File
@@ -19,39 +19,6 @@ from fastvideo.pipelines.stages import (DecodingStage, InputValidationStage, LTX
logger = init_logger(__name__)
def _resolve_refine_path(model_path: str, value: str | None) -> str | None:
"""Resolve a refine component path, preferring paths inside the local
model snapshot when the value is relative."""
if value is None:
return None
if os.path.isabs(value):
return value
candidate = os.path.join(model_path, value)
if os.path.exists(candidate):
return candidate
return value
def _resolve_refine_upsampler_path(model_path: str, model_index: dict[str, Any]) -> str | None:
"""Resolve the refine upsampler directory for a local model snapshot.
model_index.json keys ("fastvideo_refine_upsampler_path", then
"spatial_upsampler") are the documented override and win when present.
Otherwise probe the snapshot root: published LTX-2 distilled checkpoints
bundle the upsampler weights on disk (spelled "spatial_upscaler")
without declaring them in model_index.json.
"""
path = _resolve_refine_path(model_path, model_index.get("fastvideo_refine_upsampler_path"))
if path is None and "spatial_upsampler" in model_index:
path = _resolve_refine_path(model_path, "spatial_upsampler")
if path is None:
for dirname in ("spatial_upscaler", "spatial_upsampler"):
candidate = os.path.join(model_path, dirname)
if os.path.isdir(candidate):
return candidate
return path
class LTX2Pipeline(LoRAPipeline):
_required_config_modules = [
@@ -222,16 +189,28 @@ class LTX2Pipeline(LoRAPipeline):
# Apply optional FastVideo-specific refine defaults embedded in
# model_index.json. These are bundled with distilled checkpoints
# so the pipeline can self-configure without explicit user kwargs.
def _resolve_refine_path(value: str | None) -> str | None:
if value is None:
return None
if os.path.isabs(value):
return value
candidate = os.path.join(self.model_path, value)
if os.path.exists(candidate):
return candidate
return value
if (model_index.get("fastvideo_refine_enabled") is True and fastvideo_args.refine_enabled is None):
fastvideo_args.ltx2_refine_enabled = True
if (fastvideo_args.refine_upsampler_path is None and fastvideo_args.ltx2_refine_upsampler_path is None):
fastvideo_args.ltx2_refine_upsampler_path = (_resolve_refine_upsampler_path(self.model_path, model_index))
fastvideo_args.ltx2_refine_upsampler_path = _resolve_refine_path(
model_index.get("fastvideo_refine_upsampler_path"))
if (fastvideo_args.ltx2_refine_upsampler_path is None and "spatial_upsampler" in model_index):
fastvideo_args.ltx2_refine_upsampler_path = (_resolve_refine_path("spatial_upsampler"))
if (fastvideo_args.refine_transformer_path is None and fastvideo_args.ltx2_refine_transformer_path is None):
fastvideo_args.ltx2_refine_transformer_path = (_resolve_refine_path(
self.model_path, model_index.get("fastvideo_refine_transformer_path")))
model_index.get("fastvideo_refine_transformer_path")))
if (fastvideo_args.refine_lora_path is None and fastvideo_args.ltx2_refine_lora_path is None):
fastvideo_args.ltx2_refine_lora_path = _resolve_refine_path(self.model_path,
model_index.get("fastvideo_refine_lora_path"))
fastvideo_args.ltx2_refine_lora_path = _resolve_refine_path(model_index.get("fastvideo_refine_lora_path"))
if (fastvideo_args.refine_num_inference_steps is None
and fastvideo_args.ltx2_refine_num_inference_steps == FastVideoArgs.ltx2_refine_num_inference_steps
and model_index.get("fastvideo_refine_num_inference_steps") is not None):
@@ -247,11 +226,10 @@ class LTX2Pipeline(LoRAPipeline):
if (fastvideo_args.refine_add_noise is None and model_index.get("fastvideo_refine_add_noise") is not None):
fastvideo_args.ltx2_refine_add_noise = bool(model_index["fastvideo_refine_add_noise"])
if (fastvideo_args.refine_noise_path is None and fastvideo_args.ltx2_refine_noise_path is None):
fastvideo_args.ltx2_refine_noise_path = _resolve_refine_path(self.model_path,
model_index.get("fastvideo_refine_noise_path"))
fastvideo_args.ltx2_refine_noise_path = _resolve_refine_path(model_index.get("fastvideo_refine_noise_path"))
if (fastvideo_args.refine_audio_noise_path is None and fastvideo_args.ltx2_refine_audio_noise_path is None):
fastvideo_args.ltx2_refine_audio_noise_path = (_resolve_refine_path(
self.model_path, model_index.get("fastvideo_refine_audio_noise_path")))
model_index.get("fastvideo_refine_audio_noise_path")))
model_index.pop("_class_name")
model_index.pop("_diffusers_version")
-5
View File
@@ -33,8 +33,6 @@ from fastvideo.pipelines.basic.ltx2.stages import ( # noqa: F401
from fastvideo.pipelines.stages.matrixgame2_denoising import MatrixGame2CausalDenoisingStage
from fastvideo.pipelines.stages.matrixgame3_denoising import MatrixGame3DenoisingStage
from fastvideo.pipelines.stages.hyworld_denoising import HYWorldDenoisingStage
from fastvideo.pipelines.stages.kandinsky5 import (Kandinsky5DecodingStage, Kandinsky5DenoisingStage,
Kandinsky5LatentPreparationStage)
from fastvideo.pipelines.stages.gamecraft_denoising import GameCraftDenoisingStage
from fastvideo.pipelines.stages.gen3c_stages import (Gen3CCFGPolicyStage, Gen3CConditioningStage, Gen3CDenoisingStage,
Gen3CLatentPreparationStage)
@@ -67,9 +65,6 @@ __all__ = [
"MatrixGame2CausalDenoisingStage",
"MatrixGame3DenoisingStage",
"HYWorldDenoisingStage",
"Kandinsky5DecodingStage",
"Kandinsky5DenoisingStage",
"Kandinsky5LatentPreparationStage",
"GameCraftDenoisingStage",
"Gen3CCFGPolicyStage",
"Gen3CConditioningStage",
-4
View File
@@ -34,7 +34,6 @@ class PipelineStage(ABC):
composed with other stages to create a complete pipeline. Each stage is responsible
for a specific part of the process, such as prompt encoding, latent preparation, etc.
"""
performance_component_metric: str | None = None
def verify_input(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult:
"""
@@ -156,9 +155,6 @@ class PipelineStage(ABC):
logger.info("[%s] Execution completed in %s ms", stage_name, execution_time * 1000)
batch.logging_info.add_stage_execution_time(stage_key, execution_time)
batch.logging_info.add_stage_metric(stage_key, "stage_class", stage_class_name)
component_metric = self.performance_component_metric
if component_metric is not None:
batch.logging_info.add_stage_metric(stage_key, "component_metric", component_metric)
except Exception as e:
torch.cuda.synchronize()
execution_time = time.perf_counter() - start_time
-1
View File
@@ -28,7 +28,6 @@ class DecodingStage(PipelineStage):
This stage handles the decoding of latent representations into the final
output format (e.g., pixel values).
"""
performance_component_metric = "vae_decode_time_s"
def __init__(self, vae, pipeline=None) -> None:
self.vae: ParallelTiledVAE = vae
-2
View File
@@ -51,7 +51,6 @@ class DenoisingStage(PipelineStage):
This stage handles the iterative denoising process that transforms
the initial noise into the final output.
"""
performance_component_metric = "dit_time_s"
def __init__(self, transformer, scheduler, pipeline=None, transformer_2=None, vae=None) -> None:
super().__init__()
@@ -1207,7 +1206,6 @@ class Cosmos25V2WDenoisingStage(Cosmos25DenoisingStage):
class Cosmos25AutoDenoisingStage(PipelineStage):
"""Route Cosmos 2.5 denoising to T2W vs V2W/I2W."""
performance_component_metric = "dit_time_s"
def __init__(self, transformer, scheduler) -> None:
super().__init__()
-545
View File
@@ -1,545 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
import contextlib
from typing import Any
import PIL
import torch
from diffusers.utils.torch_utils import randn_tensor
from tqdm.auto import tqdm
from fastvideo.attention.backends.nabla import NablaAttentionMetadataBuilder
from fastvideo.distributed import get_local_torch_device
from fastvideo.fastvideo_args import FastVideoArgs
from fastvideo.forward_context import set_forward_context
from fastvideo.logger import init_logger
from fastvideo.models.loader.component_loader import TransformerLoader, VAELoader
from fastvideo.models.vaes.common import ParallelTiledVAE
from fastvideo.models.vision_utils import normalize, numpy_to_pt, pil_to_numpy, resize
from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
from fastvideo.pipelines.stages.base import PipelineStage
from fastvideo.pipelines.stages.decoding import DecodingStage
from fastvideo.pipelines.stages.encoding import EncodingStage
from fastvideo.pipelines.stages.validators import StageValidators as V
from fastvideo.pipelines.stages.validators import VerificationResult
from fastvideo.utils import PRECISION_TO_TYPE
logger = init_logger(__name__)
class Kandinsky5LatentPreparationStage(PipelineStage):
def __init__(self, scheduler, transformer) -> None:
super().__init__()
self.scheduler = scheduler
self.transformer = transformer
def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
if batch.height is None or batch.width is None:
raise ValueError("height and width must be provided for Kandinsky5.")
height = int(batch.height)
width = int(batch.width)
num_frames = int(batch.num_frames)
temporal_ratio = fastvideo_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio
spatial_ratio = fastvideo_args.pipeline_config.vae_config.arch_config.spatial_compression_ratio
patch_size = fastvideo_args.pipeline_config.dit_config.arch_config.patch_size
if num_frames % temporal_ratio != 1:
num_frames = num_frames // temporal_ratio * temporal_ratio + 1
batch.num_frames = num_frames
required_divisor_h = spatial_ratio * patch_size[1]
required_divisor_w = spatial_ratio * patch_size[2]
if height % required_divisor_h != 0 or width % required_divisor_w != 0:
raise ValueError(f"Kandinsky5 height must be divisible by {required_divisor_h} and width by "
f"{required_divisor_w}; "
f"got height={height}, width={width}.")
# NABLA sparse attention (Pro checkpoints) reshapes the post-patch grid
# into 8x8 blocks; validate here instead of crashing mid-denoise after
# all the encoding work is done.
arch_cfg = getattr(self.transformer, "config", None) or fastvideo_args.pipeline_config.dit_config.arch_config
if getattr(arch_cfg, "attention_type", "regular") == "nabla":
nabla_divisor_h = required_divisor_h * 8
nabla_divisor_w = required_divisor_w * 8
if height % nabla_divisor_h != 0 or width % nabla_divisor_w != 0:
raise ValueError(f"Kandinsky5 NABLA checkpoints require height divisible by {nabla_divisor_h} and "
f"width divisible by {nabla_divisor_w}; "
f"got height={height}, width={width}.")
if isinstance(batch.prompt, list):
batch_size = len(batch.prompt)
elif batch.prompt is not None:
batch_size = 1
else:
batch_size = batch.prompt_embeds[0].shape[0]
batch_size *= batch.num_videos_per_prompt
dtype = PRECISION_TO_TYPE[fastvideo_args.pipeline_config.dit_precision]
device = get_local_torch_device()
num_latent_frames = (num_frames - 1) // temporal_ratio + 1
num_channels = getattr(
self.transformer,
"in_visual_dim",
fastvideo_args.pipeline_config.dit_config.arch_config.in_visual_dim,
)
shape = (
batch_size,
num_latent_frames,
height // spatial_ratio,
width // spatial_ratio,
num_channels,
)
if isinstance(batch.generator, list) and len(batch.generator) != batch_size:
raise ValueError(f"generator list length {len(batch.generator)} does not match batch size {batch_size}.")
visual_cond = getattr(self.transformer, "visual_cond", False)
if batch.latents is None:
latents = randn_tensor(shape, generator=batch.generator, device=device, dtype=dtype)
if hasattr(self.scheduler, "init_noise_sigma"):
latents = latents * self.scheduler.init_noise_sigma
else:
valid_shapes = [shape]
if visual_cond:
valid_shapes.append((*shape[:-1], num_channels * 2 + 1))
if tuple(batch.latents.shape) not in valid_shapes:
raise ValueError(f"Provided latents shape {list(batch.latents.shape)} does not match expected "
f"Kandinsky5 latent shape(s): {[list(s) for s in valid_shapes]}.")
latents = batch.latents.to(device=device, dtype=dtype)
if visual_cond and latents.shape[-1] == num_channels:
cond = torch.zeros_like(latents)
cond_mask = torch.zeros(
(*latents.shape[:-1], 1),
device=latents.device,
dtype=latents.dtype,
)
latents = torch.cat([latents, cond, cond_mask], dim=-1)
# I2V image conditioning is placed by Kandinsky5ImageEncodingStage,
# which runs AFTER this stage so the initial noise is the generator's
# first draw (matching the official kandinskylab/kandinsky-5 order).
batch.latents = latents
batch.raw_latent_shape = (
batch_size,
num_channels,
num_latent_frames,
height // spatial_ratio,
width // spatial_ratio,
)
return batch
def verify_input(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult:
result = VerificationResult()
result.add_check("num_frames", batch.num_frames, V.positive_int)
result.add_check("height", batch.height, V.positive_int)
result.add_check("width", batch.width, V.positive_int)
result.add_check("latents", batch.latents, V.none_or_tensor)
return result
def verify_output(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult:
result = VerificationResult()
result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)])
return result
class Kandinsky5DenoisingStage(PipelineStage):
def __init__(self, transformer, scheduler) -> None:
super().__init__()
self.transformer = transformer
self.scheduler = scheduler
@staticmethod
def _scale_factor(height: int, width: int) -> tuple[float, float, float]:
if 480 <= height <= 854 and 480 <= width <= 854:
return (1.0, 2.0, 2.0)
return (1.0, 3.16, 3.16)
@staticmethod
def _text_rope_pos(mask: torch.Tensor, device: torch.device) -> torch.Tensor:
seq_len = int(mask.sum(1).max().item())
return torch.arange(seq_len, device=device)
@staticmethod
def fast_sta_nabla(
T: int,
H: int,
W: int,
wT: int = 3,
wH: int = 3,
wW: int = 3,
device: torch.device | str = "cuda",
) -> torch.Tensor:
"""
Create a sparse temporal attention (STA) mask for efficient video generation.
This method generates a mask that limits attention to nearby frames and spatial positions, reducing
computational complexity for video generation.
Args:
T (int): Number of temporal frames
H (int): Height in latent space
W (int): Width in latent space
wT (int): Temporal attention window size
wH (int): Height attention window size
wW (int): Width attention window size
device (str): Device to create tensor on
Returns:
torch.Tensor: Sparse attention mask of shape (T*H*W, T*H*W)
"""
max_extent = int(torch.tensor([T, H, W], device=device).amax().item())
r = torch.arange(0, max_extent, 1, dtype=torch.int16, device=device)
mat = (r.unsqueeze(1) - r.unsqueeze(0)).abs()
sta_t, sta_h, sta_w = (
mat[:T, :T].flatten(),
mat[:H, :H].flatten(),
mat[:W, :W].flatten(),
)
sta_t = sta_t <= wT // 2
sta_h = sta_h <= wH // 2
sta_w = sta_w <= wW // 2
sta_hw = (sta_h.unsqueeze(1) * sta_w.unsqueeze(0)).reshape(H, H, W, W).transpose(1, 2).flatten()
sta = (sta_t.unsqueeze(1) * sta_hw.unsqueeze(0)).reshape(T, T, H * W, H * W).transpose(1, 2)
return sta.reshape(T * H * W, T * H * W)
def get_sparse_params(self, sample: torch.Tensor, device: torch.device) -> dict[str, Any] | None:
"""
Generate sparse attention parameters for the transformer based on sample dimensions.
This method computes the sparse attention configuration needed for efficient video processing in the
transformer model.
Args:
sample (torch.Tensor): Input sample tensor
device (torch.device): Device to place tensors on
Returns:
Dict: Dictionary containing sparse attention parameters
"""
assert self.transformer.config.patch_size[0] == 1
_, T, H, W, _ = sample.shape
T, H, W = (
T // self.transformer.config.patch_size[0],
H // self.transformer.config.patch_size[1],
W // self.transformer.config.patch_size[2],
)
if self.transformer.config.attention_type == "nabla":
sta_mask = self.fast_sta_nabla(
T,
H // 8,
W // 8,
self.transformer.config.attention_wT,
self.transformer.config.attention_wH,
self.transformer.config.attention_wW,
device=device,
)
sparse_params = {
"sta_mask": sta_mask.unsqueeze_(0).unsqueeze_(0),
"attention_type": self.transformer.config.attention_type,
"to_fractal": True,
"P": self.transformer.config.attention_P,
"wT": self.transformer.config.attention_wT,
"wW": self.transformer.config.attention_wW,
"wH": self.transformer.config.attention_wH,
"add_sta": self.transformer.config.attention_add_sta,
"visual_shape": (T, H, W),
"method": self.transformer.config.attention_method,
}
else:
sparse_params = None
return sparse_params
def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
if batch.timesteps is None:
raise ValueError("timesteps must be prepared before Kandinsky5 denoising.")
if batch.latents is None:
raise ValueError("latents must be prepared before Kandinsky5 denoising.")
if not fastvideo_args.model_loaded["transformer"]:
loader = TransformerLoader()
self.transformer = loader.load(fastvideo_args.model_paths["transformer"], fastvideo_args)
fastvideo_args.model_loaded["transformer"] = True
device = get_local_torch_device()
target_dtype = PRECISION_TO_TYPE[fastvideo_args.pipeline_config.dit_precision]
autocast_enabled = target_dtype != torch.float32 and not fastvideo_args.disable_autocast
latents = batch.latents
num_channels = getattr(
self.transformer,
"in_visual_dim",
fastvideo_args.pipeline_config.dit_config.arch_config.in_visual_dim,
)
prompt_embeds = batch.prompt_embeds[0].to(device=device, dtype=target_dtype)
pooled = batch.prompt_embeds[1].to(device=device, dtype=target_dtype)
if batch.prompt_attention_mask is None or not batch.prompt_attention_mask:
raise ValueError("Kandinsky5 requires Qwen prompt attention masks.")
text_rope_pos = self._text_rope_pos(batch.prompt_attention_mask[0].to(device), device)
neg_prompt_embeds = None
neg_pooled = None
negative_text_rope_pos = None
if batch.do_classifier_free_guidance and batch.negative_prompt_embeds:
neg_prompt_embeds = batch.negative_prompt_embeds[0].to(device=device, dtype=target_dtype)
neg_pooled = batch.negative_prompt_embeds[1].to(device=device, dtype=target_dtype)
if batch.negative_attention_mask is None or not batch.negative_attention_mask:
raise ValueError("Kandinsky5 requires Qwen negative attention masks for CFG.")
negative_text_rope_pos = self._text_rope_pos(batch.negative_attention_mask[0].to(device), device)
height = int(batch.height)
width = int(batch.width)
temporal_ratio = fastvideo_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio
spatial_ratio = fastvideo_args.pipeline_config.vae_config.arch_config.spatial_compression_ratio
num_latent_frames = (int(batch.num_frames) - 1) // temporal_ratio + 1
visual_rope_pos = [
torch.arange(num_latent_frames, device=device),
torch.arange(height // spatial_ratio // 2, device=device),
torch.arange(width // spatial_ratio // 2, device=device),
]
scale_factor = self._scale_factor(height, width)
sparse_params = self.get_sparse_params(latents, device)
# I2V keeps the first (conditioning) frame fixed during denoising.
# Key off the actual image conditioning, not transformer.visual_cond:
# official T2V checkpoints also ship visual_cond=True, and skipping
# frame 0 for them leaves it as undenoised noise.
cond_frames = 1 if batch.image_latent is not None else 0
trajectory_timesteps: list[torch.Tensor] = []
trajectory_latents: list[torch.Tensor] = []
with tqdm(total=batch.num_inference_steps, desc="Kandinsky5 Denoising") as progress_bar:
for i, timestep in enumerate(batch.timesteps):
if hasattr(self, "interrupt") and self.interrupt:
break
t_expand = timestep.unsqueeze(0).repeat(latents.shape[0]).to(device=device, dtype=target_dtype)
attn_metadata = None
if sparse_params is not None:
attn_metadata = NablaAttentionMetadataBuilder().build(
current_timestep=i,
sta_mask=sparse_params["sta_mask"],
P=sparse_params["P"],
visual_shape=sparse_params["visual_shape"],
)
autocast_ctx = (torch.autocast(device_type="cuda", dtype=target_dtype, enabled=autocast_enabled)
if device.type == "cuda" else contextlib.nullcontext())
with set_forward_context(current_timestep=i, attn_metadata=attn_metadata,
forward_batch=batch), autocast_ctx:
pred_velocity = self.transformer(
hidden_states=latents.to(dtype=target_dtype),
encoder_hidden_states=prompt_embeds,
pooled_projections=pooled,
timestep=t_expand,
visual_rope_pos=visual_rope_pos,
text_rope_pos=text_rope_pos,
scale_factor=scale_factor,
sparse_params=sparse_params,
return_dict=True,
).sample
if neg_prompt_embeds is not None and neg_pooled is not None:
uncond_pred_velocity = self.transformer(
hidden_states=latents.to(dtype=target_dtype),
encoder_hidden_states=neg_prompt_embeds,
pooled_projections=neg_pooled,
timestep=t_expand,
visual_rope_pos=visual_rope_pos,
text_rope_pos=negative_text_rope_pos,
scale_factor=scale_factor,
sparse_params=sparse_params,
return_dict=True,
).sample
pred_velocity = uncond_pred_velocity + batch.guidance_scale * (pred_velocity -
uncond_pred_velocity)
latents[:, cond_frames:, :, :, :num_channels] = self.scheduler.step(
pred_velocity[:, cond_frames:],
timestep,
latents[:, cond_frames:, :, :, :num_channels],
return_dict=False,
)[0]
if batch.return_trajectory_latents:
trajectory_timesteps.append(timestep)
# latents is mutated in place, so snapshot a channels-first copy.
trajectory_latents.append(latents[..., :num_channels].permute(0, 4, 1, 2, 3).cpu())
if i == len(batch.timesteps) - 1 or (i + 1) % self.scheduler.order == 0:
progress_bar.update()
if trajectory_latents:
batch.trajectory_latents = torch.stack(trajectory_latents, dim=1)
batch.trajectory_timesteps = torch.stack(trajectory_timesteps, dim=0).cpu()
batch.latents = latents[:, :, :, :, :num_channels]
return batch
def verify_input(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult:
result = VerificationResult()
result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)])
result.add_check("prompt_embeds", batch.prompt_embeds, V.min_list_length(2))
return result
class Kandinsky5DecodingStage(DecodingStage):
def __init__(self, vae: ParallelTiledVAE, pipeline=None) -> None:
super().__init__(vae=vae, pipeline=pipeline)
def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
if batch.latents is None:
raise ValueError("latents must be available before Kandinsky5 decoding.")
# Kandinsky5 latents are channels-last [B, T, H, W, C]; the base stage
# (and the trajectory latents recorded by the denoising stage) work
# channels-first.
batch.latents = batch.latents.permute(0, 4, 1, 2, 3).contiguous()
return super().forward(batch, fastvideo_args)
class Kandinsky5ImageEncodingStage(EncodingStage):
"""Encode the conditioning image into a VAE latent for I2V."""
def __init__(self, vae: ParallelTiledVAE, pipeline=None) -> None:
super().__init__(vae=vae)
@staticmethod
def _preprocess(image, height: int, width: int) -> torch.Tensor:
if isinstance(image, PIL.Image.Image):
image = resize(image, height, width)
image = numpy_to_pt(pil_to_numpy(image)) # always lands in [0, 1]
return normalize(image) # [0, 1] -> [-1, 1]
# Tensor input: no reliable way to tell [0, 1] from an already
# normalized [-1, 1] tensor whose values happen to be non-negative,
# so mirror diffusers' heuristic and say what we assumed.
if image.min() >= 0:
logger.warning("Kandinsky5 conditioning image tensor has no negative values; "
"assuming range [0, 1] and normalizing to [-1, 1]. "
"Pass a [-1, 1] tensor with negative values to skip normalization.")
image = normalize(image) # [0, 1] -> [-1, 1]
if image.ndim == 3:
image = image.unsqueeze(0)
if image.shape[-2:] != (height, width):
image = torch.nn.functional.interpolate(image.float(),
size=(height, width),
mode="bilinear",
antialias=True)
return image
@torch.no_grad()
def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
if batch.pil_image is None:
raise ValueError("Kandinsky5 I2V requires an input image.")
if not fastvideo_args.model_loaded["vae"]:
vae = getattr(self, "vae", None)
if vae is None:
loader = VAELoader()
vae = loader.load(fastvideo_args.model_paths["vae"], fastvideo_args)
self.vae = vae
fastvideo_args.model_loaded["vae"] = True
device = get_local_torch_device()
vae = self.vae.to(device)
self.vae = vae
vae_dtype = PRECISION_TO_TYPE[fastvideo_args.pipeline_config.vae_precision]
vae_autocast_enabled = vae_dtype != torch.float32 and not fastvideo_args.disable_autocast
# [B, C, H, W] -> [B, C, 1, H, W]
image = self._preprocess(batch.pil_image, int(batch.height), int(batch.width))
image = image.to(device=device, dtype=torch.float32).unsqueeze(2)
# Encode the single conditioning frame without tiling (matches diffusers).
# The untested causal-VAE spatial_tiled_encode path corrupts the latent.
prev_use_tiling = vae.use_tiling
vae.use_tiling = False
try:
with torch.autocast(device_type="cuda", dtype=vae_dtype, enabled=vae_autocast_enabled):
if not vae_autocast_enabled:
image = image.to(vae_dtype)
# Sample with the batch generator (diffusers parity); mode()
# would make seed-for-seed reproduction of the reference
# pipeline impossible.
generator = batch.generator
if isinstance(generator, list) and len(generator) != image.shape[0]:
generator = generator[0]
image_latent = vae.encode(image).sample(generator=generator)
finally:
vae.use_tiling = prev_use_tiling
image_latent = image_latent * vae.scaling_factor
# [B, C, 1, H, W] -> [B, 1, H, W, C] to match channels-last latents
batch.image_latent = image_latent.permute(0, 2, 3, 4, 1).contiguous()
# Place the conditioning latent into the prepared latents: frame 0 of
# the main channels, the visual_cond channel block, and the mask.
# NOTE: the official kandinsky-5 repo leaves the visual_cond block
# zeros (generation_utils.py generate()), while the diffusers port
# copies the image latent into it. A same-seed A/B on
# Kandinsky-5.0-I2V-Pro-distilled-5s-Diffusers showed the Diffusers
# export requires the copy: zeroing the block produces smeared faces
# mid-video. Keep the diffusers semantics for Diffusers-format
# checkpoints.
latents = batch.latents
image_latent = batch.image_latent.to(device=latents.device, dtype=latents.dtype)
num_channels = image_latent.shape[-1]
latents[:, 0:1, :, :, :num_channels] = image_latent
if latents.shape[-1] > num_channels:
latents[:, 0:1, :, :, num_channels:2 * num_channels] = image_latent
latents[:, 0:1, :, :, 2 * num_channels:] = 1.0
batch.latents = latents
if fastvideo_args.vae_cpu_offload:
vae.to("cpu")
return batch
def verify_input(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult:
result = VerificationResult()
result.add_check("pil_image", batch.pil_image, V.not_none)
result.add_check("height", batch.height, V.positive_int)
result.add_check("width", batch.width, V.positive_int)
# This stage runs after latent preparation and writes into its output.
result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)])
return result
def verify_output(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult:
result = VerificationResult()
result.add_check("image_latent", batch.image_latent, [V.is_tensor, V.with_dims(5)])
return result
class Kandinsky5NormalizationStage(PipelineStage):
"""Normalize the first latent frames to reduce I2V conditioning artifacts."""
COND_FRAMES = 4
REFERENCE_FRAMES = 5
@staticmethod
def _adaptive_mean_std(source: torch.Tensor, reference: torch.Tensor) -> torch.Tensor:
source_mean = source.mean(dim=(1, 2, 3, 4), keepdim=True)
source_std = source.std(dim=(1, 2, 3, 4), keepdim=True)
# Magic constants limit how far the first frames may drift.
ref_mean = torch.clamp(reference.mean(dim=(1, 2, 3, 4), keepdim=True), source_mean - 0.05, source_mean + 0.1)
ref_std = torch.clamp(reference.std(dim=(1, 2, 3, 4), keepdim=True), source_std - 0.1, source_std + 0.25)
normalized = (source - source_mean) / source_std
return normalized * ref_std + ref_mean
def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
latents = batch.latents
n = self.COND_FRAMES
if latents is None or latents.shape[1] <= n:
return batch
reference = latents[:, n:n + min(self.REFERENCE_FRAMES, latents.shape[1] - 1)]
latents[:, :n] = self._adaptive_mean_std(latents[:, :n].clone(), reference)
batch.latents = latents
return batch
+5 -27
View File
@@ -8,8 +8,6 @@ This module contains implementations of prompt encoding stages for diffusion pip
import torch
from typing import Any
from torch.distributed.tensor import DTensor
from fastvideo.distributed import get_local_torch_device
from fastvideo.fastvideo_args import FastVideoArgs
from fastvideo.forward_context import set_forward_context
@@ -26,7 +24,6 @@ class TextEncodingStage(PipelineStage):
This stage handles the encoding of text prompts into the embedding space
expected by the diffusion model.
"""
performance_component_metric = "text_encoder_time_s"
def __init__(self, text_encoders, tokenizers) -> None:
"""
@@ -205,21 +202,6 @@ class TextEncodingStage(PipelineStage):
encoder_config = encoder_cfgs[i]
preprocess_func = preprocess_funcs[i]
postprocess_func = postprocess_funcs[i]
# cpu_offload semantics: params rest on CPU between calls but the
# forward computes on GPU. FSDP2-wrapped encoders (CPUOffloadPolicy;
# DTensor params) stream themselves per-layer — leave inputs on the
# param device and let FSDP's root pre-forward move them. A plain
# module parked on CPU by text_encoder_cpu_offload is swapped to the
# target device for the forward and back afterwards, mirroring the
# image-encoder/VAE offload pattern.
first_param = next(text_encoder.parameters(), None)
encoder_device = first_param.device if first_param is not None else torch.device(target_device)
moved_for_forward = False
if (first_param is not None and not isinstance(first_param, DTensor)
and encoder_device.type != torch.device(target_device).type):
text_encoder = text_encoder.to(target_device)
encoder_device = torch.device(target_device)
moved_for_forward = True
tok_kwargs = dict(encoder_config.tokenizer_kwargs)
if max_length is not None:
@@ -264,7 +246,7 @@ class TextEncodingStage(PipelineStage):
# pre-format prompts into message lists upstream and rely on
# the inner tokenizer + full tokenizer_kwargs (which include
# add_generation_prompt). Preserve that original path exactly.
text_inputs = tok.apply_chat_template(processed_texts, **tok_kwargs).to(encoder_device)
text_inputs = tok.apply_chat_template(processed_texts, **tok_kwargs).to(target_device)
else:
# Two-step approach matching Diffusers: format with chat
# template first, then tokenize the resulting strings.
@@ -278,9 +260,9 @@ class TextEncodingStage(PipelineStage):
enable_thinking=False,
)
formatted_texts.append(formatted)
text_inputs = tokenizer(formatted_texts, **tok_kwargs).to(encoder_device)
text_inputs = tokenizer(formatted_texts, **tok_kwargs).to(target_device)
else:
text_inputs = tok(processed_texts, **tok_kwargs).to(encoder_device)
text_inputs = tok(processed_texts, **tok_kwargs).to(target_device)
input_ids = text_inputs["input_ids"]
attention_mask = text_inputs["attention_mask"]
@@ -300,18 +282,15 @@ class TextEncodingStage(PipelineStage):
except Exception:
prompt_embeds, attention_mask = postprocess_func(outputs, attention_mask)
if is_ltx2 and getattr(outputs, "hidden_states", None):
audio_embed = outputs.hidden_states[0].to(device=target_device)
audio_embed = outputs.hidden_states[0]
if dtype is not None:
audio_embed = audio_embed.to(dtype=dtype)
audio_embeds_list.append(audio_embed)
prompt_embeds = prompt_embeds.to(device=target_device)
if dtype is not None:
prompt_embeds = prompt_embeds.to(dtype=dtype)
embeds_list.append(prompt_embeds)
if return_attention_mask:
attn_masks_list.append(attention_mask.to(device=target_device))
if moved_for_forward and fastvideo_args.text_encoder_cpu_offload:
text_encoder.to("cpu")
attn_masks_list.append(attention_mask)
self._last_audio_embeds = audio_embeds_list if is_ltx2 else None
return self.return_embeds(embeds_list, attn_masks_list, return_type, return_attention_mask, indices)
@@ -371,7 +350,6 @@ class Cosmos25TextEncodingStage(PipelineStage):
Cosmos 2.5 uses Reason1 (Qwen2.5-VL) and relies on the encoder's
`compute_text_embeddings_online()`.
"""
performance_component_metric = "text_encoder_time_s"
def __init__(self, text_encoder) -> None:
super().__init__()
-10
View File
@@ -219,16 +219,6 @@ class StageValidators:
return validator
@staticmethod
def min_list_length(min_length: int) -> Callable[[Any], bool]:
"""Return a validator that checks if a list has at least min_length items."""
def validator(value: Any) -> bool:
return StageValidators.list_min_length(value, min_length)
validator.__name__ = f"list_min_length_{min_length}"
return validator
@staticmethod
def min_dims(min_dims: int) -> Callable[[Any], bool]:
"""Return a validator that checks if tensor has at least min_dims dimensions and no NaN values."""
-8
View File
@@ -157,14 +157,6 @@ class CudaPlatformBase(Platform):
"ATTN_QAT_TRAIN selected but fastvideo_kernel.triton_kernels.attn_qat_train is not built. "
"Silent fallback would produce a non-QAT training run; refusing to proceed. "
"Install the training kernel or pick a different FASTVIDEO_ATTENTION_BACKEND.")
elif selected_backend == AttentionBackendEnum.NABLA_ATTN:
from fastvideo.attention.backends.nabla import CAN_USE_FLEX_ATTN
if CAN_USE_FLEX_ATTN:
logger.info("Using NABLA block-sparse flex-attention backend.")
return "fastvideo.attention.backends.nabla.NablaAttentionBackend"
raise ImportError("NABLA_ATTN selected but torch.nn.attention.flex_attention is unavailable in this "
"PyTorch build. Silent fallback to dense attention would be orders of magnitude "
"slower and diverge from the reference; upgrade PyTorch or pick a different backend.")
elif selected_backend == AttentionBackendEnum.VIDEO_SPARSE_ATTN:
try:
from fastvideo_kernel import video_sparse_attn # noqa: F401
-1
View File
@@ -22,7 +22,6 @@ class AttentionBackendEnum(enum.Enum):
VMOBA_ATTN = enum.auto()
SLA_ATTN = enum.auto()
SAGE_SLA_ATTN = enum.auto()
NABLA_ATTN = enum.auto()
NO_ATTENTION = enum.auto()
+6 -181
View File
@@ -28,7 +28,6 @@ from fastvideo.configs.pipelines.hunyuan15 import (Hunyuan15T2V480PConfig, Hunyu
Hunyuan15T2V720PConfig, Hunyuan15I2V720PConfig,
Hunyuan15SR1080PConfig)
from fastvideo.configs.pipelines.hyworld import HYWorldConfig
from fastvideo.configs.pipelines.kandinsky5 import Kandinsky5I2VConfig, Kandinsky5T2VConfig
from fastvideo.configs.pipelines.lingbotworld import LingBotWorldI2V480PConfig
from fastvideo.configs.pipelines.longcat import LongCatT2V480PConfig
from fastvideo.pipelines.basic.ltx2.pipeline_configs import LTX2T2VConfig
@@ -119,10 +118,6 @@ class ConfigInfo:
workload_types: tuple[WorkloadType, ...]
model_family: str | None = None
default_preset: str | None = None
# When set, overrides the model_index `_class_name` for pipeline resolution.
# Lets a model family map to a specific pipeline class by path/detector
# (e.g. a T2V and I2V checkpoint that share a `_class_name`).
pipeline_cls_name: str | None = None
# The central registry mapping a model name to its configuration information
@@ -143,7 +138,6 @@ def register_configs(
model_detectors: list[Callable[[str], bool]] | None = None,
model_family: str | None = None,
default_preset: str | None = None,
pipeline_cls_name: str | None = None,
) -> None:
"""Register config classes for a model family.
@@ -158,7 +152,6 @@ def register_configs(
workload_types=workload_types,
model_family=model_family,
default_preset=default_preset,
pipeline_cls_name=pipeline_cls_name,
)
if hf_model_paths:
@@ -498,174 +491,18 @@ def _register_configs() -> None:
default_preset="lingbotworld_i2v",
)
def _kandinsky5_detector(require: tuple[str, ...] = (), exclude: tuple[str, ...] = ()) -> Callable[[str], bool]:
def detect(path: str) -> bool:
path_lower = path.lower()
if "kandinsky5" not in path_lower and "kandinsky-5" not in path_lower:
return False
return (all(token in path_lower for token in require) and not any(token in path_lower for token in exclude))
return detect
# t2v/i2v exclude each other so a checkpoint stored under a directory
# containing the other token (e.g. ~/i2v_experiments/kandinsky5-t2v-ft)
# falls through to the model_index _class_name fallback detectors below
# instead of being misrouted.
_is_kandinsky5_t2v = _kandinsky5_detector(require=("t2v", ), exclude=("i2v", ))
_is_kandinsky5_i2v = _kandinsky5_detector(require=("i2v", ), exclude=("t2v", ))
_is_kandinsky5_t2v_lite = _kandinsky5_detector(require=("t2v", "lite"), exclude=("i2v", "distilled"))
_is_kandinsky5_t2v_pro = _kandinsky5_detector(require=("t2v", "pro"), exclude=("i2v", "distilled"))
_is_kandinsky5_t2v_lite_distilled = _kandinsky5_detector(require=("t2v", "lite", "distilled"), exclude=("i2v", ))
_is_kandinsky5_t2v_pro_distilled = _kandinsky5_detector(require=("t2v", "pro", "distilled"), exclude=("i2v", ))
_is_kandinsky5_i2v_lite = _kandinsky5_detector(require=("i2v", "lite"), exclude=("t2v", "distilled"))
_is_kandinsky5_i2v_pro = _kandinsky5_detector(require=("i2v", "pro"), exclude=("t2v", "distilled"))
_is_kandinsky5_i2v_lite_distilled = _kandinsky5_detector(require=("i2v", "lite", "distilled"), exclude=("t2v", ))
_is_kandinsky5_i2v_pro_distilled = _kandinsky5_detector(require=("i2v", "pro", "distilled"), exclude=("t2v", ))
# Kandinsky5 Lite T2V
register_configs(
sampling_param_cls=None,
pipeline_config_cls=Kandinsky5T2VConfig,
pipeline_config_cls=PipelineConfig,
workload_types=(WorkloadType.T2V, ),
hf_model_paths=[
"kandinskylab/Kandinsky-5.0-T2V-Lite-sft-5s-Diffusers",
],
model_detectors=[
_is_kandinsky5_t2v_lite,
lambda path: any(token in path.lower() for token in ("kandinsky5", "kandinsky-5")),
],
model_family="kandinsky5",
default_preset="kandinsky5_t2v_lite_5s",
pipeline_cls_name="Kandinsky5T2VPipeline",
)
# Kandinsky5 Pro T2V
register_configs(
sampling_param_cls=None,
pipeline_config_cls=Kandinsky5T2VConfig,
workload_types=(WorkloadType.T2V, ),
hf_model_paths=[
"kandinskylab/Kandinsky-5.0-T2V-Pro-sft-5s-Diffusers",
],
model_detectors=[
_is_kandinsky5_t2v_pro,
],
model_family="kandinsky5",
default_preset="kandinsky5_t2v_pro_5s",
)
# Kandinsky5 Lite T2V Distilled
register_configs(
sampling_param_cls=None,
pipeline_config_cls=Kandinsky5T2VConfig,
workload_types=(WorkloadType.T2V, ),
hf_model_paths=[
"kandinskylab/Kandinsky-5.0-T2V-Lite-distilled16steps-5s-Diffusers",
],
model_detectors=[
_is_kandinsky5_t2v_lite_distilled,
],
model_family="kandinsky5",
default_preset="kandinsky5_t2v_lite_distilled_5s",
)
# Kandinsky5 Pro T2V Distilled
register_configs(
sampling_param_cls=None,
pipeline_config_cls=Kandinsky5T2VConfig,
workload_types=(WorkloadType.T2V, ),
hf_model_paths=[
"kandinskylab/Kandinsky-5.0-T2V-Pro-distilled-5s-Diffusers",
],
model_detectors=[
_is_kandinsky5_t2v_pro_distilled,
],
model_family="kandinsky5",
default_preset="kandinsky5_t2v_pro_distilled_5s",
)
# Kandinsky5 Lite I2V
register_configs(
sampling_param_cls=None,
pipeline_config_cls=Kandinsky5I2VConfig,
workload_types=(WorkloadType.I2V, ),
hf_model_paths=["kandinskylab/Kandinsky-5.0-I2V-Lite-5s-Diffusers"],
model_detectors=[
_is_kandinsky5_i2v_lite,
],
model_family="kandinsky5",
default_preset="kandinsky5_i2v_lite_5s",
pipeline_cls_name="Kandinsky5I2VPipeline",
)
# Kandinsky5 Pro I2V
register_configs(
sampling_param_cls=None,
pipeline_config_cls=Kandinsky5I2VConfig,
workload_types=(WorkloadType.I2V, ),
hf_model_paths=["kandinskylab/Kandinsky-5.0-I2V-Pro-sft-5s-Diffusers"],
model_detectors=[
_is_kandinsky5_i2v_pro,
],
model_family="kandinsky5",
default_preset="kandinsky5_i2v_pro_5s",
)
# Kandinsky5 Pro I2V Distilled
register_configs(
sampling_param_cls=None,
pipeline_config_cls=Kandinsky5I2VConfig,
workload_types=(WorkloadType.I2V, ),
hf_model_paths=["kandinskylab/Kandinsky-5.0-I2V-Pro-distilled-5s-Diffusers"],
model_detectors=[
_is_kandinsky5_i2v_pro_distilled,
],
model_family="kandinsky5",
default_preset="kandinsky5_i2v_pro_distilled_5s",
)
# Kandinsky5 Lite I2V Distilled (no official hub repo yet; local
# conversions get distilled sampling defaults instead of the sft ones the
# fallback would apply).
register_configs(
sampling_param_cls=None,
pipeline_config_cls=Kandinsky5I2VConfig,
workload_types=(),
model_detectors=[
_is_kandinsky5_i2v_lite_distilled,
],
model_family="kandinsky5",
default_preset="kandinsky5_i2v_lite_distilled_5s",
pipeline_cls_name="Kandinsky5I2VPipeline",
)
# Kandinsky5 fallbacks — registered AFTER the variant detectors so those
# win first-match. Catch checkpoints the variant detectors cannot resolve:
# token-less local paths matched via the model_index _class_name
# ("kandinsky5t2vpipeline" carries no lite/pro marker), variant combos
# without a dedicated entry (e.g. I2V Lite distilled), and t2v+i2v
# ambiguous paths resolved by the checkpoint's _class_name.
register_configs(
sampling_param_cls=None,
pipeline_config_cls=Kandinsky5T2VConfig,
workload_types=(),
model_detectors=[
_is_kandinsky5_t2v,
],
model_family="kandinsky5",
default_preset="kandinsky5_t2v_lite_5s",
pipeline_cls_name="Kandinsky5T2VPipeline",
)
register_configs(
sampling_param_cls=None,
pipeline_config_cls=Kandinsky5I2VConfig,
workload_types=(),
model_detectors=[
_is_kandinsky5_i2v,
],
model_family="kandinsky5",
default_preset="kandinsky5_i2v_lite_5s",
pipeline_cls_name="Kandinsky5I2VPipeline",
)
# LongCat (T2V, I2V, VC use same config; workload varies by path)
@@ -714,7 +551,7 @@ def _register_configs() -> None:
"FastVideo/Matrix-Game-2.0-Base-Distilled-Diffusers",
"FastVideo/Matrix-Game-2.0-GTA-Distilled-Diffusers",
"FastVideo/Matrix-Game-2.0-TempleRun-Distilled-Diffusers",
# Legacy HF paths (kept for backward compat - pre-rename names):
# Legacy HF paths (kept for backward compat — pre-rename names):
"FastVideo/Matrix-Game-2.0-Base-Diffusers",
"FastVideo/Matrix-Game-2.0-GTA-Diffusers",
"FastVideo/Matrix-Game-2.0-TempleRun-Diffusers",
@@ -943,7 +780,7 @@ def _register_configs() -> None:
pipeline_config_cls=DreamXWorld5BCamPipelineConfig,
workload_types=(WorkloadType.I2V, ),
hf_model_paths=[
"FastVideo/DreamX-World-5B-Cam-Diffusers",
"GD-ML/DreamX-World-5B-Cam",
],
model_detectors=[
# Mutually exclusive with the AR detector below: Cam requires an
@@ -961,7 +798,7 @@ def _register_configs() -> None:
pipeline_config_cls=DreamXWorld5BARPipelineConfig,
workload_types=(WorkloadType.I2V, ),
hf_model_paths=[
"FastVideo/DreamX-World-5B-Diffusers",
"GD-ML/DreamX-World-5B",
],
model_detectors=[
lambda path:
@@ -1100,10 +937,7 @@ def get_model_info(
assert config_info is not None, "config_info must be resolved"
if override_pipeline_cls_name:
# Explicit override: skip config resolution entirely so checkpoints
# without a diffusers model_index.json keep working (and no download
# is triggered just to log the replaced name).
pipeline_name: str | None = override_pipeline_cls_name
pipeline_name = override_pipeline_cls_name
logger.info("Using override pipeline class name %s", pipeline_name)
else:
if os.path.exists(model_path):
@@ -1112,12 +946,6 @@ def get_model_info(
config = maybe_download_model_index(model_path)
pipeline_name = config.get("_class_name")
if config_info.pipeline_cls_name is not None:
# The resolved (path/detector-based) config pins the pipeline class,
# e.g. an I2V checkpoint whose `_class_name` would otherwise resolve
# to the T2V pipeline.
logger.info("Pinning pipeline class name from %s to %s", pipeline_name, config_info.pipeline_cls_name)
pipeline_name = config_info.pipeline_cls_name
if pipeline_name is None:
raise ValueError("Model config does not contain a _class_name attribute. "
@@ -1170,8 +998,6 @@ def _register_presets() -> None:
ALL_PRESETS as HUNYUAN15_PRESETS, )
from fastvideo.pipelines.basic.hyworld.presets import (
ALL_PRESETS as HYWORLD_PRESETS, )
from fastvideo.pipelines.basic.kandinsky5.presets import (
ALL_PRESETS as KANDINSKY5_PRESETS, )
from fastvideo.pipelines.basic.lingbotworld.presets import (
ALL_PRESETS as LINGBOTWORLD_PRESETS, )
from fastvideo.pipelines.basic.longcat.presets import (
@@ -1202,7 +1028,6 @@ def _register_presets() -> None:
HUNYUAN_PRESETS,
HUNYUAN15_PRESETS,
HYWORLD_PRESETS,
KANDINSKY5_PRESETS,
LINGBOTWORLD_PRESETS,
LONGCAT_PRESETS,
LTX2_PRESETS,
@@ -17,6 +17,46 @@ class TestLegacyTorchCompileKwargsTranslation:
first-class :class:`CompileConfig` fields and anything unknown falls
into ``extras``."""
def test_all_typed_keys_promoted(self) -> None:
config = legacy_from_pretrained_to_config(
"/models/ltx2",
{
"enable_torch_compile": True,
"torch_compile_kwargs": {
"backend": "inductor",
"fullgraph": True,
"mode": "max-autotune-no-cudagraphs",
"dynamic": False,
},
},
)
compile_config = config.engine.compile
assert compile_config.enabled is True
assert compile_config.backend == "inductor"
assert compile_config.fullgraph is True
assert compile_config.mode == "max-autotune-no-cudagraphs"
assert compile_config.dynamic is False
assert compile_config.extras == {}
def test_unknown_keys_land_in_extras(self) -> None:
config = legacy_from_pretrained_to_config(
"/models/ltx2",
{
"enable_torch_compile": True,
"torch_compile_kwargs": {
"backend": "inductor",
"options": {"triton.cudagraphs": False},
"disable": False,
},
},
)
compile_config = config.engine.compile
assert compile_config.backend == "inductor"
assert compile_config.extras == {
"options": {"triton.cudagraphs": False},
"disable": False,
}
def test_empty_kwargs_produces_empty_extras(self) -> None:
config = legacy_from_pretrained_to_config(
"/models/ltx2",
@@ -77,6 +117,13 @@ class TestLegacyLtx2VaeTilingTranslation:
``generator.pipeline.vae_tiling``; reverse direction emits the
legacy name back to FastVideoArgs."""
def test_forward_routes_to_pipeline_vae_tiling(self) -> None:
config = legacy_from_pretrained_to_config(
"/models/ltx2",
{"ltx2_vae_tiling": False},
)
assert config.pipeline.vae_tiling is False
def test_true_round_trips(self) -> None:
config = legacy_from_pretrained_to_config(
"/models/ltx2",
@@ -1,43 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""CPU-only regression test for the SDPA metadata mask contract.
SDPAMetadata is cross-backend: HYWorld and HunyuanVideo15 build it while the
layer's attention selector may pick FLASH_ATTN, and the shared convention
for padding masks is the tokenizer-style 2D [batch, key_len]. The builder
must store 2D masks unchanged; only the torch-sdpa impl lifts them to 4D
internally.
Regression guard for the CI break where the builder lifted 2D -> 4D and the
FLASH_ATTN equal-length branch padded ``attn_mask.shape[1]`` (== 1 for 4D)
into a garbage-length mask (CUDA device-side assert, SIGABRT).
"""
from __future__ import annotations
import torch
from fastvideo.attention.backends.sdpa import (SDPAMetadataBuilder, _normalize_attn_mask_for_sdpa)
def test_builder_keeps_2d_padding_mask_2d() -> None:
"""A tokenizer-style 2D int mask must be stored as-is (cross-backend contract)."""
mask = torch.tensor([[1, 1, 0], [1, 0, 0]], dtype=torch.int64)
md = SDPAMetadataBuilder().build(current_timestep=0, attn_mask=mask)
assert md.attn_mask is not None
assert md.attn_mask.dim() == 2
assert md.attn_mask.shape == (2, 3)
assert torch.equal(md.attn_mask, mask)
def test_normalize_lifts_2d_int_mask_to_4d_bool_for_torch_sdpa() -> None:
"""The torch-sdpa consumer coerces int -> bool and lifts 2D -> [B,1,1,K]."""
batch, q_len, k_len, heads, head_dim = 2, 4, 3, 1, 8
query = torch.zeros(batch, heads, q_len, head_dim)
key = torch.zeros(batch, heads, k_len, head_dim)
mask = torch.tensor([[1, 1, 0], [1, 0, 0]], dtype=torch.int64)
out = _normalize_attn_mask_for_sdpa(mask, query, key)
assert out is not None
assert out.dtype == torch.bool
assert out.shape == (batch, 1, 1, k_len)
assert torch.equal(out[:, 0, 0, :], mask != 0)
@@ -1,84 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Guard: every test directory must be collected by some CI lane or be on
the explicit allowlist below.
Three separate incidents on 2026-07-05 found test files that no CI lane
ever collects (fastvideo/tests/stages/, tests/local_tests/ additions in
PR #1509, and this sweep found seven dark directories in total): the tests
pass review, merge, and then silently never run. This test makes going
dark an explicit, reviewed decision instead of an accident: adding a new
test directory fails CI until it is either wired into a lane or
allowlisted here with a reason.
Pure text analysis — no fastvideo imports, no GPU, no torch.
"""
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
TESTS_ROOT = REPO_ROOT / "fastvideo" / "tests"
# Files whose text constitutes "a CI lane references this directory".
CI_SOURCES = [
TESTS_ROOT / "modal" / "pr_test.py",
TESTS_ROOT / "modal" / "ssim_test.py",
*sorted((REPO_ROOT / ".buildkite").rglob("*.yml")),
*sorted((REPO_ROOT / ".buildkite").rglob("*.sh")),
]
# Directories that intentionally have no CI lane today. Every entry needs a
# reason; remove the entry when the directory gets wired into a lane.
# State as found on 2026-07-05 — these SHOULD shrink over time, not grow.
ALLOWLIST = {
"attention": "no lane yet — GPU attention-backend tests, run manually",
"audio": "no lane yet — audio encoder tests, run manually",
"distributed": "no lane yet — multi-GPU torchrun tests, run manually",
"hooks": "no lane yet — run manually",
"layers": "no lane yet — torchrun FSDP dispatch tests, run manually",
"nightly": "by design: nightly cadence, not per-PR",
"modal": "CI infrastructure itself, not a test suite",
}
def _dirs_with_tests() -> list[str]:
dirs = []
for child in sorted(TESTS_ROOT.iterdir()):
if child.is_dir() and any(child.rglob("test_*.py")):
dirs.append(child.name)
return dirs
def _ci_text() -> str:
return "\n".join(
src.read_text(errors="replace") for src in CI_SOURCES if src.exists())
def test_every_test_directory_is_collected_or_allowlisted():
ci_text = _ci_text()
dark = [
name for name in _dirs_with_tests()
if f"tests/{name}" not in ci_text and name not in ALLOWLIST
]
assert not dark, (
f"Test directories not referenced by any CI lane and not "
f"allowlisted: {dark}. Wire them into a lane in "
f"fastvideo/tests/modal/pr_test.py (or a Buildkite step), or add an "
f"allowlist entry with a reason in {__file__}.")
def test_local_tests_stays_out_of_ci():
# tests/local_tests/ (repo root) is developer-local by design (author
# decision, 2026-07-05): parity scaffolds and machine-specific checks
# that must never gate CI. Fail if any CI source starts collecting it.
assert "tests/local_tests" not in _ci_text(), (
"tests/local_tests/ is local-only by design; remove the CI "
"reference or move the tests into a fastvideo/tests/ lane.")
def test_allowlist_entries_are_still_real_directories():
# A stale allowlist hides regressions; entries must track reality.
missing = [
name for name in ALLOWLIST
if name != "modal" and not (TESTS_ROOT / name).is_dir()
]
assert not missing, (
f"Allowlisted directories no longer exist — remove them: {missing}")
@@ -1,58 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Guard the Modal FA4 defaults that keep CI lanes on their intended backend.
Pure text/AST analysis: no fastvideo imports, no torch, no Modal client.
"""
from __future__ import annotations
import ast
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
MODAL_ROOT = REPO_ROOT / "fastvideo" / "tests" / "modal"
PR_TEST = MODAL_ROOT / "pr_test.py"
LAUNCH_L40S_JOB = MODAL_ROOT / "launch_l40s_job.py"
SSIM_TEST = MODAL_ROOT / "ssim_test.py"
def _function_strings(path: Path, function_name: str) -> str:
source = path.read_text(encoding="utf-8")
tree = ast.parse(source)
for node in tree.body:
if isinstance(node, ast.FunctionDef) and node.name == function_name:
return "\n".join(
child.value
for child in ast.walk(node)
if isinstance(child, ast.Constant)
and isinstance(child.value, str)
)
raise AssertionError(f"{function_name} not found in {path}")
def test_generic_l40s_launcher_defaults_fa4_off():
source = LAUNCH_L40S_JOB.read_text(encoding="utf-8")
assert '"FASTVIDEO_FA4": os.environ.get("FASTVIDEO_FA4", "0")' in source
def test_ssim_launcher_keeps_fa4_enabled_by_default():
source = SSIM_TEST.read_text(encoding="utf-8")
assert '"FASTVIDEO_FA4": os.environ.get("FASTVIDEO_FA4", "1")' in source
def test_pr_model_load_and_training_lanes_disable_fa4():
lanes = {
"run_transformer_tests": "pytest ./fastvideo/tests/transformers -vs",
"run_training_tests": "pytest ./fastvideo/tests/training/Vanilla -srP",
"run_training_lora_tests": "pytest ./fastvideo/tests/training/lora/test_lora_training.py -srP",
"run_training_tests_VSA": "pytest ./fastvideo/tests/training/VSA -srP",
"run_distill_dmd_tests": "pytest ./fastvideo/tests/training/distill/test_distill_dmd.py -vs",
"run_self_forcing_tests": "pytest ./fastvideo/tests/training/self-forcing/test_self_forcing.py -vs",
"run_train_framework_tests": "pytest ./fastvideo/tests/train/models ./fastvideo/tests/train/methods -vs",
"seed_grad_norm_references": "pytest ./fastvideo/tests/train/methods -vs -rs",
}
for function_name, pytest_command in lanes.items():
function_strings = _function_strings(PR_TEST, function_name)
assert "FASTVIDEO_FA4=0" in function_strings
assert pytest_command in function_strings
@@ -1,80 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""CPU-only tests for load_image URL retry behavior.
A HF-CDN IncompleteRead mid-body previously killed SSIM runs (build 4202,
test_wan_i2v) because load_image handed PIL a partial ``.raw`` stream with no
retry. These tests mock requests.get: a transient failure must be retried and
a full-body success must yield a usable PIL image.
"""
import io
from unittest import mock
import PIL.Image
import pytest
import requests
from fastvideo.models import vision_utils
URL = "https://example.com/image.png"
def _png_bytes() -> bytes:
buf = io.BytesIO()
PIL.Image.new("RGB", (4, 4), (255, 0, 0)).save(buf, format="PNG")
return buf.getvalue()
def _good_response() -> mock.Mock:
response = mock.Mock()
response.content = _png_bytes()
response.raise_for_status = mock.Mock()
return response
def test_load_image_retries_transient_error_then_succeeds(monkeypatch):
monkeypatch.setattr(vision_utils.time, "sleep", lambda _s: None)
get = mock.Mock(side_effect=[
# The shape that broke build 4202: connection dropped mid-body.
requests.exceptions.ChunkedEncodingError(
"IncompleteRead(1024 bytes read, 2048 more expected)"),
_good_response(),
])
monkeypatch.setattr(vision_utils.requests, "get", get)
image = vision_utils.load_image(URL)
assert get.call_count == 2
assert get.call_args.kwargs["timeout"] == 30.0
assert isinstance(image, PIL.Image.Image)
assert image.size == (4, 4)
assert image.mode == "RGB"
def test_load_image_raises_after_exhausting_retries(monkeypatch):
sleeps: list[float] = []
monkeypatch.setattr(vision_utils.time, "sleep", sleeps.append)
get = mock.Mock(
side_effect=requests.exceptions.ConnectionError("connection reset"))
monkeypatch.setattr(vision_utils.requests, "get", get)
with pytest.raises(requests.exceptions.ConnectionError):
vision_utils.load_image(URL)
assert get.call_count == 3
assert sleeps == [1, 2]
def test_load_image_does_not_retry_on_client_error(monkeypatch):
sleeps: list[float] = []
monkeypatch.setattr(vision_utils.time, "sleep", sleeps.append)
response = mock.Mock(status_code=404)
error = requests.HTTPError("404 Client Error", response=response)
response.raise_for_status.side_effect = error
get = mock.Mock(return_value=response)
monkeypatch.setattr(vision_utils.requests, "get", get)
with pytest.raises(requests.HTTPError):
vision_utils.load_image("https://example.com/missing.png")
assert get.call_count == 1
assert sleeps == []
+3 -6
View File
@@ -12,7 +12,6 @@ from fastvideo.configs.pipelines import CosmosConfig, PipelineConfig, WanT2V480P
from fastvideo.forward_context import set_forward_context
from fastvideo.logger import init_logger
from fastvideo.models.loader.component_loader import TextEncoderLoader
from fastvideo.tests.utils import skip_if_gated_repo_inaccessible
from fastvideo.utils import maybe_download_model, PRECISION_TO_TYPE
from fastvideo.fastvideo_args import FastVideoArgs
from fastvideo.configs.models.encoders import T5Config, T5LargeConfig
@@ -37,11 +36,9 @@ def t5_model_paths_and_config():
@pytest.fixture
def t5_large_model_paths_and_config():
base_model_path = "nvidia/Cosmos-Predict2-2B-Video2World"
local_dir = os.path.join('data', base_model_path)
skip_if_gated_repo_inaccessible(base_model_path,
local_path=local_dir,
test_name="Cosmos T5-large encoder test")
model_path = maybe_download_model(base_model_path, local_dir=local_dir)
model_path = maybe_download_model(base_model_path,
local_dir=os.path.join(
'data', base_model_path))
text_encoder_path = os.path.join(model_path, "text_encoder")
tokenizer_path = os.path.join(model_path, "tokenizer")
return text_encoder_path, tokenizer_path, CosmosConfig()
+3 -4
View File
@@ -97,10 +97,9 @@ image = (
"TOKENIZERS_PARALLELISM": "false",
**({"UV_TORCH_BACKEND": uv_torch_backend_override} if uv_torch_backend_override else {}),
"FASTVIDEO_ATTENTION_BACKEND": os.environ.get("FASTVIDEO_ATTENTION_BACKEND", "FLASH_ATTN"),
# FA4 is opt-in (FASTVIDEO_FA4). Generic ad hoc jobs should follow the
# product default unless a caller opts in through the local env or
# --env-vars.
"FASTVIDEO_FA4": os.environ.get("FASTVIDEO_FA4", "0"),
# FA4 is opt-in (FASTVIDEO_FA4); keep CI parity with the seeded
# references. Caller override wins.
"FASTVIDEO_FA4": os.environ.get("FASTVIDEO_FA4", "1"),
})
)
+98 -93
View File
@@ -30,15 +30,6 @@ print(f"Using image: {image_ref}")
# the baked setting, and a caller override always wins.
uv_torch_backend_override = resolve_uv_torch_backend(image_tag)
# INVARIANT: this image definition must be byte-identical for every CI job at
# a given base image digest -- one build, shared cache across all concurrent
# lanes. Never put a per-job/per-commit value (BUILDKITE_*, TEST_SCOPE,
# env-derived overrides) into the image via .env()/run_commands: it becomes an
# image layer, so whenever the base digest changes every concurrent job
# rebuilds its own image variant (~15-20 min each), blowing the Buildkite job
# budget. `image_ref` is the only env-derived input allowed here, because it
# *selects* the base digest. Per-job values arrive at runtime via
# `ci_env_secret` below.
image = (modal.Image.from_registry(
image_ref, add_python="3.12"
).run_commands("rm -rf /FastVideo").apt_install(
@@ -46,45 +37,41 @@ image = (modal.Image.from_registry(
).run_commands(
"curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable"
).run_commands("echo 'source ~/.cargo/env' >> ~/.bashrc").env({
"PATH": "/root/.cargo/bin:$PATH",
"HF_REPO_ID": "FastVideo/performance-tracking",
"PATH":
"/root/.cargo/bin:$PATH",
"BUILDKITE_REPO":
os.environ.get("BUILDKITE_REPO", ""),
"BUILDKITE_COMMIT":
os.environ.get("BUILDKITE_COMMIT", ""),
"BUILDKITE_PULL_REQUEST":
os.environ.get("BUILDKITE_PULL_REQUEST", ""),
"BUILDKITE_BRANCH":
os.environ.get("BUILDKITE_BRANCH", ""),
"BUILDKITE_BUILD_URL":
os.environ.get("BUILDKITE_BUILD_URL", ""),
"BUILDKITE_BUILD_ID":
os.environ.get("BUILDKITE_BUILD_ID", ""),
"BUILDKITE_JOB_ID":
os.environ.get("BUILDKITE_JOB_ID", ""),
"TEST_SCOPE":
os.environ.get("TEST_SCOPE", ""),
"IMAGE_VERSION":
image_version,
**({
"UV_TORCH_BACKEND": uv_torch_backend_override
} if uv_torch_backend_override else {}),
# FA4 is opt-in (FASTVIDEO_FA4); CI lanes keep it enabled to match the
# SSIM/perf baselines. Caller override wins.
"FASTVIDEO_FA4":
os.environ.get("FASTVIDEO_FA4", "1"),
"HF_REPO_ID":
"FastVideo/performance-tracking",
}))
dreamverse_image = (image.run_commands(
"curl -fsSL https://deb.nodesource.com/setup_22.x | bash -"
).apt_install("nodejs").run_commands("node --version && npm --version"))
# Per-job/per-invocation values are injected into the container environment at
# RUNTIME via this secret (attached to every function below), so the image
# stays identical across jobs. Consumers (run_test_command's checkout,
# fastvideo/tests/performance/{compare_baseline,identity}.py, the FA4
# resolver, `uv pip install`) all read os.environ at runtime, so nothing else
# changes.
ci_env_secret = modal.Secret.from_dict({
"BUILDKITE_REPO": os.environ.get("BUILDKITE_REPO", ""),
"BUILDKITE_COMMIT": os.environ.get("BUILDKITE_COMMIT", ""),
"BUILDKITE_PULL_REQUEST": os.environ.get("BUILDKITE_PULL_REQUEST", ""),
"BUILDKITE_BRANCH": os.environ.get("BUILDKITE_BRANCH", ""),
"BUILDKITE_BUILD_URL": os.environ.get("BUILDKITE_BUILD_URL", ""),
"BUILDKITE_BUILD_ID": os.environ.get("BUILDKITE_BUILD_ID", ""),
"BUILDKITE_JOB_ID": os.environ.get("BUILDKITE_JOB_ID", ""),
"TEST_SCOPE": os.environ.get("TEST_SCOPE", ""),
"IMAGE_VERSION": image_version,
**({
"UV_TORCH_BACKEND": uv_torch_backend_override
} if uv_torch_backend_override else {}),
# FA4 is opt-in (FASTVIDEO_FA4). Keep the default enabled for
# inference/perf parity; model-load and training lanes that do not exercise
# FA4 explicitly set FASTVIDEO_FA4=0 in their command strings below.
# Caller override wins.
"FASTVIDEO_FA4": os.environ.get("FASTVIDEO_FA4", "1"),
})
hf_secret = modal.Secret.from_dict(
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
wandb_secret = modal.Secret.from_dict(
{"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})
def run_test(pytest_command: str):
"""Helper function to run a test suite with custom pytest command"""
@@ -160,7 +147,10 @@ def run_test_command(test_command: str,
@app.function(gpu="H100:1",
image=image,
timeout=1200,
secrets=[hf_secret, ci_env_secret],
secrets=[
modal.Secret.from_dict(
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
],
volumes={"/root/data": model_vol})
def run_encoder_tests():
run_test(
@@ -171,7 +161,10 @@ def run_encoder_tests():
@app.function(gpu="L40S:1",
image=image,
timeout=1200,
secrets=[hf_secret, ci_env_secret],
secrets=[
modal.Secret.from_dict(
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
],
volumes={"/root/data": model_vol})
def run_vae_tests():
run_test(
@@ -182,109 +175,113 @@ def run_vae_tests():
@app.function(gpu="L40S:1",
image=image,
timeout=900,
secrets=[hf_secret, ci_env_secret],
secrets=[
modal.Secret.from_dict(
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
],
volumes={"/root/data": model_vol})
def run_transformer_tests():
run_test(
"export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && "
"FASTVIDEO_FA4=0 pytest ./fastvideo/tests/transformers -vs"
"export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/transformers -vs"
)
@app.function(gpu="L40S:4",
cpu=8.0,
memory=32768,
image=image,
timeout=900,
secrets=[wandb_secret, ci_env_secret],
secrets=[
modal.Secret.from_dict(
{"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})
],
volumes={"/root/data": model_vol})
def run_training_tests():
run_test(
"export HF_HOME='/root/data/.cache' && wandb login $WANDB_API_KEY && "
"FASTVIDEO_FA4=0 pytest ./fastvideo/tests/training/Vanilla -srP"
"export HF_HOME='/root/data/.cache' && wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/Vanilla -srP"
)
@app.function(gpu="L40S:2",
cpu=8.0,
memory=32768,
image=image,
timeout=900,
secrets=[wandb_secret, ci_env_secret],
secrets=[
modal.Secret.from_dict(
{"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})
],
volumes={"/root/data": model_vol})
def run_training_lora_tests():
run_test(
"export HF_HOME='/root/data/.cache' && wandb login $WANDB_API_KEY && "
"FASTVIDEO_FA4=0 pytest ./fastvideo/tests/training/lora/test_lora_training.py -srP"
"export HF_HOME='/root/data/.cache' && wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/lora/test_lora_training.py -srP"
)
@app.function(gpu="H100:2",
image=image,
timeout=900,
secrets=[wandb_secret, ci_env_secret])
secrets=[
modal.Secret.from_dict(
{"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})
])
def run_training_tests_VSA():
run_test(
"wandb login $WANDB_API_KEY && FASTVIDEO_FA4=0 pytest ./fastvideo/tests/training/VSA -srP"
"wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/VSA -srP"
)
@app.function(gpu="H100:1", image=image, timeout=900, secrets=[ci_env_secret])
@app.function(gpu="H100:1", image=image, timeout=900)
def run_kernel_tests():
run_test("pytest fastvideo-kernel/tests/ -vs")
# @app.function(gpu="H100:1", image=image, timeout=900, secrets=[ci_env_secret])
# @app.function(gpu="H100:1", image=image, timeout=900)
# def run_precision_tests_VSA():
# # VSA correctness is covered by the same file now
# run_test("pytest fastvideo-kernel/tests/test_correctness.py")
# @app.function(gpu="L40S:1", image=image, timeout=900, secrets=[ci_env_secret])
# @app.function(gpu="L40S:1", image=image, timeout=900)
# def run_precision_tests_vmoba():
# run_test("pytest fastvideo-kernel/tests/test_vmoba_correctness.py")
@app.function(gpu="L40S:1", image=image, timeout=900, secrets=[ci_env_secret])
@app.function(gpu="L40S:1", image=image, timeout=900)
def run_inference_tests_vmoba():
run_test('python fastvideo/tests/inference/vmoba/test_vmoba_inference.py')
@app.function(gpu="L40S:1", image=image, timeout=1200, secrets=[ci_env_secret])
@app.function(gpu="L40S:1", image=image, timeout=1200)
def run_inference_lora_tests():
run_test(
"pytest ./fastvideo/tests/inference/lora/test_lora_inference_similarity.py -vs"
)
@app.function(gpu="L40S:2", image=image, timeout=900, secrets=[ci_env_secret])
@app.function(gpu="L40S:2", image=image, timeout=900)
def run_distill_dmd_tests():
run_test(
"FASTVIDEO_FA4=0 pytest ./fastvideo/tests/training/distill/test_distill_dmd.py -vs")
"pytest ./fastvideo/tests/training/distill/test_distill_dmd.py -vs")
@app.function(gpu="L40S:2",
image=image,
timeout=900,
secrets=[wandb_secret, ci_env_secret])
secrets=[
modal.Secret.from_dict(
{"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})
])
def run_self_forcing_tests():
run_test(
"wandb login $WANDB_API_KEY && "
"FASTVIDEO_FA4=0 pytest ./fastvideo/tests/training/self-forcing/test_self_forcing.py -vs"
"wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/self-forcing/test_self_forcing.py -vs"
)
@app.function(gpu="L40S:1", image=image, timeout=900, secrets=[ci_env_secret])
@app.function(gpu="L40S:1", image=image, timeout=900)
def run_unit_test():
run_test(
"pytest ./fastvideo/tests/api/ ./fastvideo/tests/contract/ ./fastvideo/tests/dataset/ ./fastvideo/tests/workflow/ ./fastvideo/tests/entrypoints/ ./fastvideo/tests/train/ ./fastvideo/tests/stages/ ./fastvideo/tests/ops/ ./fastvideo/tests/training/test_trackers.py ./fastvideo/tests/attention/test_sdpa_metadata_mask_contract.py --ignore=./fastvideo/tests/entrypoints/test_openai_api_integration.py --ignore=./fastvideo/tests/train/models --ignore=./fastvideo/tests/train/methods -vs"
"pytest ./fastvideo/tests/api/ ./fastvideo/tests/contract/ ./fastvideo/tests/dataset/ ./fastvideo/tests/workflow/ ./fastvideo/tests/entrypoints/ ./fastvideo/tests/train/ --ignore=./fastvideo/tests/entrypoints/test_openai_api_integration.py --ignore=./fastvideo/tests/train/models --ignore=./fastvideo/tests/train/methods -vs"
)
# TODO: David: GPU only used to resolve import time requirement (not needed for this test). Maybe make those imports lazy?
@app.function(gpu="L40S:1",
image=dreamverse_image,
timeout=1800,
secrets=[ci_env_secret])
@app.function(gpu="L40S:1", image=dreamverse_image, timeout=1800)
def run_dreamverse_app_tests():
run_test_command(
install_command="",
@@ -321,23 +318,26 @@ def run_dreamverse_app_tests():
@app.function(gpu="L40S:1",
cpu=8.0,
memory=32768,
image=image,
timeout=1800,
secrets=[hf_secret, ci_env_secret],
secrets=[
modal.Secret.from_dict(
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
],
volumes={"/root/data": model_vol})
def run_train_framework_tests():
run_test(
"export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && "
"FASTVIDEO_FA4=0 pytest ./fastvideo/tests/train/models ./fastvideo/tests/train/methods -vs"
"export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/train/models ./fastvideo/tests/train/methods -vs"
)
@app.function(gpu="L40S:1",
image=image,
timeout=1800,
secrets=[hf_secret, ci_env_secret],
secrets=[
modal.Secret.from_dict(
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
],
volumes={"/root/data": model_vol})
def seed_grad_norm_references():
"""Record the per-method grad-norm reference for the **CI GPU (L40S only)**.
@@ -356,15 +356,17 @@ def seed_grad_norm_references():
the local command and the ``_DEVICE_MAPPINGS`` table.
"""
run_test(
"export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && "
"FASTVIDEO_FA4=0 FASTVIDEO_GRADNORM_UPDATE=1 pytest ./fastvideo/tests/train/methods -vs -rs"
"export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && FASTVIDEO_GRADNORM_UPDATE=1 pytest ./fastvideo/tests/train/methods -vs -rs"
)
@app.function(gpu="L40S:1",
image=image,
timeout=3600,
secrets=[hf_secret, ci_env_secret],
secrets=[
modal.Secret.from_dict(
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
],
volumes={"/root/data": model_vol})
def run_eval_tests():
# Eval metric regression: drives the high-level fastvideo.eval API on a
@@ -387,7 +389,10 @@ def run_eval_tests():
@app.function(gpu="L40S:1",
image=image,
timeout=3600,
secrets=[hf_secret, ci_env_secret])
secrets=[
modal.Secret.from_dict(
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
])
def run_lora_extraction_tests():
run_test(
"hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/lora_extraction/test_lora_extraction.py"
@@ -395,11 +400,12 @@ def run_lora_extraction_tests():
@app.function(gpu="L40S:2",
cpu=8.0,
memory=32768,
image=image,
timeout=1800,
secrets=[hf_secret, ci_env_secret],
secrets=[
modal.Secret.from_dict(
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
],
volumes={"/root/data": model_vol})
def run_performance_tests():
# PR/direct records are uploaded only on pass; scheduled main uploads pass
@@ -421,8 +427,6 @@ def run_performance_tests():
"export PERF_RUN_SOURCE='unknown'; "
"export PERF_UPLOAD_POLICY='never'; "
"fi; "
"(nvidia-smi --query-gpu=index,timestamp,clocks.sm,clocks.max.sm,power.draw,power.limit,temperature.gpu "
"--format=csv -l 10 > /tmp/gpu_telemetry.csv 2>/dev/null &); "
"pytest ./fastvideo/tests/performance -vs; "
"PYTEST_RC=$?; "
"PERF_RC=0; "
@@ -431,8 +435,6 @@ def run_performance_tests():
"PERF_RC=$?; "
"fi; "
"python ./fastvideo/tests/performance/dashboard.py || true; "
"echo '--- GPU telemetry (clocks.sm vs clocks.max.sm reveals capped hosts) ---'; "
"cat /tmp/gpu_telemetry.csv || true; "
"FINAL_RC=$PYTEST_RC; "
"if [ $FINAL_RC -eq 0 ]; then FINAL_RC=$PERF_RC; fi; "
"exit $FINAL_RC")
@@ -441,7 +443,10 @@ def run_performance_tests():
@app.function(gpu="L40S:1",
image=image,
timeout=1800,
secrets=[hf_secret, ci_env_secret],
secrets=[
modal.Secret.from_dict(
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
],
volumes={"/root/data": model_vol})
def run_api_server_tests():
run_test(
+3 -5
View File
@@ -57,12 +57,10 @@ image = (
.run_commands("echo 'source ~/.cargo/env' >> ~/.bashrc")
.env(
{
# INVARIANT: keep this image identical for every CI job at a given
# base digest -- per-job/per-commit values (BUILDKITE_*) must never
# be baked here as image layers or every job rebuilds its own image
# variant. repo/commit/PR already reach the container as
# run_ssim_partition() arguments.
"PATH": "/root/.cargo/bin:$PATH",
"BUILDKITE_REPO": os.environ.get("BUILDKITE_REPO", ""),
"BUILDKITE_COMMIT": os.environ.get("BUILDKITE_COMMIT", ""),
"BUILDKITE_PULL_REQUEST": os.environ.get("BUILDKITE_PULL_REQUEST", ""),
"IMAGE_VERSION": image_version,
**({"UV_TORCH_BACKEND": uv_torch_backend_override} if uv_torch_backend_override else {}),
# FA4 is opt-in (FASTVIDEO_FA4); the SSIM references were seeded
@@ -34,9 +34,6 @@ class TestAbsMaxFP8LinearMethod(unittest.TestCase):
method._convert_scale(scale)
def test_create_weights_rejects_invalid_dtype(self):
# float64 is outside the supported set (bf16/fp16/fp32). The test
# originally passed float32, which create_weights has accepted since
# PR #981 — it never ran in CI, so the mismatch went unnoticed.
method = AbsMaxFP8LinearMethod()
layer = nn.Module()
with self.assertRaisesRegex(AssertionError, "only supports"):
@@ -46,7 +43,7 @@ class TestAbsMaxFP8LinearMethod(unittest.TestCase):
output_partition_sizes=[3],
input_size=2,
output_size=3,
params_dtype=torch.float64,
params_dtype=torch.float32,
)
def test_absmax_fp8_parameter_weight_loader(self):
+95 -214
View File
@@ -5,12 +5,11 @@ This script:
1) reads current benchmark results from fastvideo/tests/performance/results,
2) syncs the canonical baseline from the configured HF dataset repo,
3) compares each current record against the median of up to 5 prior
baseline-eligible successful records in the same workload/variant/version,
GPU, recipe, hardware, and software cohort,
baseline-eligible successful records (filtered by gpu_type),
4) writes normalized records back to the HF dataset repo according to
PERF_UPLOAD_POLICY,
5) exits non-zero if any gated metric exceeds both its percent and absolute
regression floors.
5) exits non-zero if any metric regresses by more than PERF_MAX_REGRESSION
(default 5%).
"""
import glob
@@ -22,36 +21,21 @@ from datetime import datetime, timezone
from typing import Any
try:
from fastvideo.performance.hf_store import (
from .hf_store import (
load_records_for_model,
safe_float,
sanitize,
sync_from_hf,
upload_record,
)
from fastvideo.performance.metric_policy import (
MetricPolicy,
regression_delta,
resolve_metric_policies,
serialize_metric_thresholds,
)
except ImportError:
repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))
if repo_root not in sys.path:
sys.path.insert(0, repo_root)
from fastvideo.performance.hf_store import (
from hf_store import (
load_records_for_model,
safe_float,
sanitize,
sync_from_hf,
upload_record,
)
from fastvideo.performance.metric_policy import (
MetricPolicy,
regression_delta,
resolve_metric_policies,
serialize_metric_thresholds,
)
RESULTS_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
@@ -62,33 +46,25 @@ TRACKING_ROOT = os.environ.get(
"/tmp/perf-tracking",
)
PERF_REPORTS_DIR = os.environ.get("PERF_REPORTS_DIR", "/root/data/perf_reports")
MAX_REGRESSION = float(os.environ.get("PERF_MAX_REGRESSION", "0.05"))
UPLOAD_POLICY = os.environ.get("PERF_UPLOAD_POLICY", "never").strip().lower()
VALID_UPLOAD_POLICIES = {"never", "pass", "always"}
VALID_RUN_SOURCES = {"pr", "local", "scheduled_main", "unknown"}
IDENTITY_KEYS = (
"result_schema_version",
"workload_id",
"variant_id",
"benchmark_version",
"recipe",
"recipe_fingerprint",
"hardware_profile",
"hardware_profile_id",
"software_profile",
"software_profile_id",
"environment_metadata",
"environment_fingerprint",
"quality_metadata",
"variant_metadata",
)
COMPARISON_IDENTITY_KEYS = (
"workload_id",
"variant_id",
"benchmark_version",
"recipe_fingerprint",
"hardware_profile_id",
"software_profile_id",
METRICS = (
("latency", "Latency", 3),
("throughput", "Throughput", 3),
("memory", "Memory", 1),
("text_encoder_time_s", "Text Enc", 3),
("dit_time_s", "DiT", 3),
("vae_decode_time_s", "VAE Decode", 3),
)
LOWER_IS_BETTER_METRICS = {
"latency",
"memory",
"text_encoder_time_s",
"dit_time_s",
"vae_decode_time_s",
}
def _should_persist_tracking() -> bool:
@@ -146,72 +122,21 @@ def _result_failed_static_thresholds() -> bool:
def _record_metadata(run_source: str, result: dict[str, Any]) -> dict[str, Any]:
raw_run_source = str(result.get("run_source") or "").strip().lower()
if raw_run_source in VALID_RUN_SOURCES:
run_source = raw_run_source
pr_number = result.get("pr_number") or os.environ.get("BUILDKITE_PULL_REQUEST", "")
if not _truthy_pr_number(str(pr_number)):
pr_number = ""
return {
"run_source": run_source,
"baseline_eligible": False,
"branch": result.get("branch") or os.environ.get("BUILDKITE_BRANCH", ""),
"branch": os.environ.get("BUILDKITE_BRANCH", ""),
"pr_number": pr_number,
"test_scope": result.get("test_scope") or os.environ.get("TEST_SCOPE", ""),
"build_url": result.get("build_url") or os.environ.get("BUILDKITE_BUILD_URL", ""),
"build_id": result.get("build_id") or os.environ.get("BUILDKITE_BUILD_ID", ""),
"job_id": result.get("job_id") or os.environ.get("BUILDKITE_JOB_ID", ""),
"test_scope": os.environ.get("TEST_SCOPE", ""),
"build_url": os.environ.get("BUILDKITE_BUILD_URL", ""),
"build_id": os.environ.get("BUILDKITE_BUILD_ID", ""),
"job_id": os.environ.get("BUILDKITE_JOB_ID", ""),
}
def _identity_metadata(result: dict[str, Any]) -> dict[str, Any]:
metadata = {
key: result[key]
for key in IDENTITY_KEYS
if key in result and result[key] is not None
}
recipe = result.get("recipe")
benchmark = recipe.get("benchmark") if isinstance(recipe, dict) else None
if isinstance(benchmark, dict):
for key in ("workload_id", "variant_id", "benchmark_version"):
if key not in metadata and benchmark.get(key) is not None:
metadata[key] = benchmark[key]
return metadata
def _comparison_identity_filters(record: dict[str, Any]) -> dict[str, str]:
missing = [
key for key in COMPARISON_IDENTITY_KEYS
if key not in record or record[key] is None or (isinstance(record[key], str) and not record[key].strip())
]
if missing:
raise ValueError("Performance record missing required comparison identity fields: " + ", ".join(missing))
return {
key: str(record[key])
for key in COMPARISON_IDENTITY_KEYS
}
def _comparison_identity_filters_or_none(record: dict[str, Any]) -> dict[str, str] | None:
try:
return _comparison_identity_filters(record)
except ValueError as exc:
print("Skipping rolling baseline comparison for "
f"{record.get('model_id', 'unknown')} on "
f"{record.get('gpu_type', 'unknown')}: {exc}. "
"The normalized artifact will still be written, but this record "
"will not be baseline eligible.")
return None
def _format_identity_filters(filters: dict[str, str]) -> str:
if not filters:
return ""
compact = ", ".join(f"{key}={value}" for key, value in filters.items())
return f" ({compact})"
def _load_current_results() -> list[dict[str, Any]]:
pattern = os.path.join(RESULTS_DIR, "perf_*.json")
records: list[dict[str, Any]] = []
@@ -243,9 +168,8 @@ def normalize_performance_result(result: dict[str, Any]) -> dict[str, Any]:
text_encoder_time = safe_float(result.get("text_encoder_time_s"))
dit_time = safe_float(result.get("dit_time_s"))
vae_decode_time = safe_float(result.get("vae_decode_time_s"))
metric_policies = resolve_metric_policies(result.get("regression_thresholds"))
record = {
return {
"model_id": model_id,
"timestamp": timestamp,
"commit_sha": commit_sha,
@@ -256,12 +180,9 @@ def normalize_performance_result(result: dict[str, Any]) -> dict[str, Any]:
"text_encoder_time_s": text_encoder_time,
"dit_time_s": dit_time,
"vae_decode_time_s": vae_decode_time,
"regression_thresholds": serialize_metric_thresholds(metric_policies),
"success": True,
**_record_metadata(_detect_run_source(), result),
}
record.update(_identity_metadata(result))
return record
def _normalize_record(result: dict[str, Any]) -> dict[str, Any]:
@@ -308,41 +229,55 @@ def _baseline_metric(records: list[dict[str, Any]], key: str) -> float | None:
return statistics.median(values)
def _metric_policy_summary(policy: MetricPolicy) -> str:
gated = "gated" if policy.gated else "info"
return (
f"{gated}, >{policy.threshold_percent * 100:.1f}% "
f"and >{policy.threshold_absolute:.{policy.precision}f}"
)
def _check_regressions(
current: dict[str, Any],
baseline_records: list[dict[str, Any]],
metric_policies: tuple[MetricPolicy, ...],
max_regression: float,
) -> list[str]:
failures: list[str] = []
for policy in metric_policies:
baseline = _baseline_metric(baseline_records, policy.key)
curr = safe_float(current.get(policy.key))
if baseline is None or curr is None:
for metric, _label, _precision in METRICS:
if metric not in LOWER_IS_BETTER_METRICS:
continue
delta = regression_delta(policy, curr, baseline)
if delta is None or not delta.regressed:
baseline = _baseline_metric(baseline_records, metric)
curr = safe_float(current.get(metric))
if baseline is None or curr is None or baseline <= 0:
continue
failures.append(
f"{current['model_id']} {policy.key} regressed by "
f"{delta.percent * 100:.1f}% and "
f"{delta.absolute:.{policy.precision}f} "
f"(current={curr:.{policy.precision}f}, "
f"baseline_median={baseline:.{policy.precision}f}, "
f"threshold={_metric_policy_summary(policy)})"
)
regression = (curr - baseline) / baseline
if regression > max_regression:
failures.append(f"{current['model_id']} {metric} regressed by "
f"{regression * 100:.1f}% "
f"(current={curr:.3f}, baseline_median={baseline:.3f})")
baseline_tp = _baseline_metric(baseline_records, "throughput")
curr_tp = safe_float(current.get("throughput"))
if baseline_tp is not None and curr_tp is not None and baseline_tp > 0:
regression = (baseline_tp - curr_tp) / baseline_tp
if regression > max_regression:
failures.append(f"{current['model_id']} throughput regressed by "
f"{regression * 100:.1f}% "
f"(current={curr_tp:.3f}, baseline_median={baseline_tp:.3f})")
return failures
def _metric_delta_percent(
metric: str,
current: dict[str, Any],
baseline_records: list[dict[str, Any]],
) -> float | None:
curr = safe_float(current.get(metric))
baseline = _baseline_metric(baseline_records, metric)
if curr is None or baseline is None or baseline <= 0:
return None
if metric in LOWER_IS_BETTER_METRICS:
return (curr - baseline) / baseline * 100.0
if metric == "throughput":
return (baseline - curr) / baseline * 100.0
return None
def _compact_value(value: float | None, precision: int = 3) -> str:
if value is None:
return "n/a"
@@ -352,42 +287,23 @@ def _compact_value(value: float | None, precision: int = 3) -> str:
def _build_summary_row(
record: dict[str, Any],
baseline_records: list[dict[str, Any]],
metric_policies: tuple[MetricPolicy, ...],
has_failed: bool,
) -> dict[str, Any]:
"""Format a single benchmark result as a row for the Markdown table."""
metric_values: dict[str, dict[str, Any]] = {}
metric_values: dict[str, dict[str, float | None]] = {}
regressions: list[float] = []
failing_metrics: list[str] = []
threshold_exceeded_metrics: list[str] = []
for policy in metric_policies:
curr = safe_float(record.get(policy.key))
baseline = _baseline_metric(baseline_records, policy.key)
delta = (
regression_delta(policy, curr, baseline)
if curr is not None and baseline is not None
else None
)
regression = None if delta is None else delta.percent * 100.0
absolute_delta = None if delta is None else delta.absolute
metric_values[policy.key] = {
for metric, _label, _precision in METRICS:
curr = safe_float(record.get(metric))
baseline = _baseline_metric(baseline_records, metric)
regression = _metric_delta_percent(metric, record, baseline_records)
metric_values[metric] = {
"curr": curr,
"base": baseline,
"regression_pct": regression,
"absolute_delta": absolute_delta,
"threshold_percent": policy.threshold_percent * 100.0,
"threshold_absolute": policy.threshold_absolute,
"gated": policy.gated,
"threshold_exceeded": False if delta is None else delta.threshold_exceeded,
"regressed": False if delta is None else delta.regressed,
}
if regression is not None:
regressions.append(regression)
if delta is not None and delta.threshold_exceeded:
threshold_exceeded_metrics.append(policy.key)
if delta is not None and delta.regressed:
failing_metrics.append(policy.key)
worst_regression_pct = max(regressions) if regressions else None
@@ -397,50 +313,40 @@ def _build_summary_row(
"baseline_n": len(baseline_records),
"metrics": metric_values,
"worst_regression_pct": worst_regression_pct,
"threshold_exceeded_metrics": threshold_exceeded_metrics,
"failing_metrics": failing_metrics,
"failed": has_failed,
}
def _build_markdown_summary(
summary_rows: list[dict[str, Any]],
metric_policies: tuple[MetricPolicy, ...],
max_regression: float,
) -> str:
lines = [
"## Performance Baseline Comparison",
"",
"Threshold: gated metrics fail only when both percent and absolute "
"regression floors are exceeded.",
f"Threshold: regressions greater than {max_regression * 100:.1f}% fail",
"",
("| Model | GPU | Baseline N | Latency (curr/base) | "
"Throughput (curr/base) | Memory (curr/base) | "
"Text Enc (curr/base) | DiT (curr/base) | "
"VAE Decode (curr/base) | Worst Regression | Exceeded Metrics | "
"Failing Metrics | Status |"),
"|---|---|---:|---|---|---|---|---|---|---:|---|---|---|",
"VAE Decode (curr/base) | Worst Regression | Status |"),
"|---|---|---:|---|---|---|---|---|---|---:|---|",
]
for row in summary_rows:
metric_cells = []
for policy in metric_policies:
values = row["metrics"][policy.key]
metric_cells.append(f"{_compact_value(values['curr'], policy.precision)} / "
f"{_compact_value(values['base'], policy.precision)}")
for metric, _label, precision in METRICS:
values = row["metrics"][metric]
metric_cells.append(f"{_compact_value(values['curr'], precision)} / "
f"{_compact_value(values['base'], precision)}")
worst_reg = ("n/a" if row["worst_regression_pct"] is None else f"{row['worst_regression_pct']:.1f}%")
exceeded_metrics = (
", ".join(row["threshold_exceeded_metrics"])
if row["threshold_exceeded_metrics"]
else "none"
)
failing_metrics = ", ".join(row["failing_metrics"]) if row["failing_metrics"] else "none"
status = "FAIL" if row["failed"] else "PASS"
lines.append(f"| {row['model_id']} | {row['gpu_type']} | "
f"{row['baseline_n']} | "
f"{' | '.join(metric_cells)} | "
f"{worst_reg} | {exceeded_metrics} | {failing_metrics} | {status} |")
f"{worst_reg} | {status} |")
return "\n".join(lines) + "\n"
@@ -494,54 +400,29 @@ def main() -> int:
for raw in current_results:
record = _normalize_record(raw)
metric_policies = resolve_metric_policies(record.get("regression_thresholds"))
identity_filters = _comparison_identity_filters_or_none(record)
baseline_records: list[dict[str, Any]] = []
if identity_filters is None:
# Records without the full v2 identity block skip rolling-baseline
# comparison entirely: only the static thresholds gate them and
# they never become baseline eligible.
record["baseline_status"] = "skipped_missing_identity"
baseline_records = load_records_for_model(
TRACKING_ROOT,
record["model_id"],
record["gpu_type"],
last_n=5,
successful_only=True,
baseline_eligible_only=True,
)
if not baseline_records:
print(f"No baseline for {record['model_id']} on "
f"{record['gpu_type']}. Initializing...")
failures: list[str] = []
record["success"] = True
else:
baseline_records = load_records_for_model(
TRACKING_ROOT,
record["model_id"],
record["gpu_type"],
**identity_filters,
last_n=5,
successful_only=True,
baseline_eligible_only=True,
)
if not baseline_records:
# A brand-new cohort has NO regression gating until history
# accumulates — make that loud and machine-readable instead of
# an indistinguishable pass, so a cohort shift (intended or
# accidental, e.g. an identity-field change) never silently
# blinds the comparison.
record["baseline_status"] = "initialized_new_cohort"
print("=" * 72)
print(f"WARNING: NO BASELINE — initializing a NEW cohort for "
f"{record['model_id']} on {record['gpu_type']}"
f"{_format_identity_filters(identity_filters)}")
print("Regression gating is INACTIVE for this cohort until "
"baseline history accumulates. If this cohort shift is "
"unexpected, check the identity fields above.")
print("=" * 72)
failures = []
else:
record["baseline_status"] = "compared"
failures = _check_regressions(record, baseline_records, metric_policies)
failures = _check_regressions(record, baseline_records, MAX_REGRESSION)
if static_threshold_failed:
failures.append(f"{record['model_id']} fixed-threshold phase failed "
f"(PERF_PYTEST_RC={os.environ.get('PERF_PYTEST_RC')})")
record["success"] = not failures
record["baseline_eligible"] = (
identity_filters is not None and _is_baseline_eligible(record["run_source"], record["success"])
)
record["baseline_eligible"] = _is_baseline_eligible(record["run_source"], record["success"])
all_failures.extend(failures)
_write_normalized_artifact(record)
@@ -553,10 +434,10 @@ def main() -> int:
print("Tracking upload skipped for "
f"{record['model_id']} ({record['run_source']}, success={record['success']})")
summary_rows.append(_build_summary_row(record, baseline_records, metric_policies, bool(failures)))
summary_rows.append(_build_summary_row(record, baseline_records, bool(failures)))
commit_sha = os.environ.get("BUILDKITE_COMMIT", "unknown")[:7]
markdown = _build_markdown_summary(summary_rows, resolve_metric_policies(None))
markdown = _build_markdown_summary(summary_rows, MAX_REGRESSION)
_emit_markdown_summary(markdown, commit_sha)
if all_failures:
+13 -89
View File
@@ -1,19 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
import os
import sys
from html import escape
from datetime import datetime
import plotly.express as px
import pandas as pd
try:
from fastvideo.performance.hf_store import load_as_dataframe, sync_from_hf
except ImportError:
repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))
if repo_root not in sys.path:
sys.path.insert(0, repo_root)
from fastvideo.performance.hf_store import load_as_dataframe, sync_from_hf
from hf_store import sync_from_hf, load_as_dataframe
TRACKING_ROOT = os.environ.get(
"PERFORMANCE_TRACKING_ROOT",
@@ -29,76 +22,15 @@ METRICS = (
"dit_time_s",
"vae_decode_time_s",
)
COMPARISON_COHORT_KEYS = (
"workload_id",
"variant_id",
"benchmark_version",
"recipe_fingerprint",
"hardware_profile_id",
"software_profile_id",
)
GROUP_KEYS = ("model_id", "gpu_type", *COMPARISON_COHORT_KEYS)
def _cohort_value(value: object) -> str:
if value is None:
return ""
try:
if pd.isna(value):
return ""
except (TypeError, ValueError):
pass
return str(value)
def _display_value(value: object) -> str:
return _cohort_value(value) or "legacy"
def _short_value(value: object) -> str:
text = _display_value(value)
if text == "legacy" or len(text) <= 14:
return text
return text[:12]
def _cohort_title(group_key: tuple[object, ...]) -> str:
workload = _display_value(group_key[2])
variant = _display_value(group_key[3])
version = _display_value(group_key[4])
version_label = version if version == "legacy" else f"v{version}"
return f"{workload} / {variant} / {version_label}"
def _cohort_detail(group_key: tuple[object, ...]) -> str:
return " | ".join((
f"recipe {_short_value(group_key[5])}",
_short_value(group_key[6]),
_short_value(group_key[7]),
))
def _group_record(group_key: tuple[object, ...]) -> dict[str, str]:
return {
key: _cohort_value(value)
for key, value in zip(GROUP_KEYS, group_key)
}
def _dashboard_frame(df: pd.DataFrame) -> pd.DataFrame:
dashboard_df = df.copy()
for key in GROUP_KEYS:
if key not in dashboard_df.columns:
dashboard_df[key] = ""
dashboard_df[key] = dashboard_df[key].map(_cohort_value)
return dashboard_df
# -----------------------------
# 1. Grouping
# -----------------------------
def group_data(df: pd.DataFrame):
# Group by the same comparison cohort used by baseline gating.
return _dashboard_frame(df).groupby(list(GROUP_KEYS), dropna=False)
# Group only by model+GPU so each group produces a time-series line.
# config_id (commit SHA) is carried as a column for hover/color use.
keys = ["model_id", "gpu_type"]
return df.groupby(keys, dropna=False)
# -----------------------------
# 2. Plot builder
@@ -107,20 +39,15 @@ def build_plots(df: pd.DataFrame) -> tuple[list, list[dict[str, object]]]:
figs = []
skipped_metrics: list[dict[str, object]] = []
for group_key, g in group_data(df):
model_id, gpu_type = group_key[:2]
group_record = _group_record(group_key)
cohort_title = _cohort_title(group_key)
cohort_detail = _cohort_detail(group_key)
for (model_id, gpu_type), g in group_data(df):
g = g.sort_values("timestamp")
# One chart per metric so the y-axes aren't on wildly different scales
for metric in METRICS:
if metric not in g.columns:
skipped_metrics.append({
**group_record,
"cohort": cohort_title,
"cohort_detail": cohort_detail,
"model_id": model_id,
"gpu_type": gpu_type,
"metric": metric,
"reason": "column missing from loaded records",
"records": len(g),
@@ -131,9 +58,8 @@ def build_plots(df: pd.DataFrame) -> tuple[list, list[dict[str, object]]]:
non_null = int(g[metric].notna().sum())
if non_null == 0:
skipped_metrics.append({
**group_record,
"cohort": cohort_title,
"cohort_detail": cohort_detail,
"model_id": model_id,
"gpu_type": gpu_type,
"metric": metric,
"reason": "no non-null values in loaded records",
"records": len(g),
@@ -146,8 +72,8 @@ def build_plots(df: pd.DataFrame) -> tuple[list, list[dict[str, object]]]:
x="timestamp",
y=metric,
markers=True,
hover_data=["config_id", "commit_sha", *COMPARISON_COHORT_KEYS],
title=f"{model_id} | {gpu_type} | {cohort_title} | {cohort_detail} | {metric}",
hover_data=["config_id", "commit_sha"],
title=f"{model_id} | {gpu_type} | {metric}",
labels={"timestamp": "Time", metric: metric},
)
figs.append(fig)
@@ -162,7 +88,7 @@ def render_skipped_metrics(skipped_metrics: list[dict[str, object]]) -> str:
rows = [
"<h3>Skipped Metric Plots</h3>",
"<table>",
("<thead><tr><th>Model</th><th>GPU</th><th>Cohort</th><th>Metric</th>"
("<thead><tr><th>Model</th><th>GPU</th><th>Metric</th>"
"<th>Records</th><th>Non-null</th><th>Reason</th></tr></thead>"),
"<tbody>",
]
@@ -171,7 +97,6 @@ def render_skipped_metrics(skipped_metrics: list[dict[str, object]]) -> str:
"<tr>"
f"<td>{escape(str(item['model_id']))}</td>"
f"<td>{escape(str(item['gpu_type']))}</td>"
f"<td>{escape(str(item['cohort']))}<br><code>{escape(str(item['cohort_detail']))}</code></td>"
f"<td>{escape(str(item['metric']))}</td>"
f"<td>{item['records']}</td>"
f"<td>{item['non_null']}</td>"
@@ -233,7 +158,6 @@ def main() -> None:
print("Skipped metric plots:")
for item in skipped_metrics:
print(f" - {item['model_id']} | {item['gpu_type']} | "
f"{item['cohort']} | {item['cohort_detail']} | "
f"{item['metric']}: {item['reason']} "
f"({item['non_null']}/{item['records']} non-null)")
print(f"Generated {len(figs)} metric plot(s)")
@@ -57,7 +57,9 @@ def is_baseline_eligible_record(record: dict[str, Any]) -> bool:
"""
if record.get("baseline_eligible") is True:
return True
return "baseline_eligible" not in record and "run_source" not in record
if "baseline_eligible" not in record and "run_source" not in record:
return True
return False
def resolve_hf_token() -> str | None:
@@ -268,31 +270,16 @@ def load_records_for_model(
model_id: str,
gpu_type: str | None = None,
*,
workload_id: str | None = None,
variant_id: str | None = None,
benchmark_version: str | None = None,
recipe_fingerprint: str | None = None,
hardware_profile_id: str | None = None,
software_profile_id: str | None = None,
last_n: int | None = None,
successful_only: bool = True,
baseline_eligible_only: bool = False,
) -> list[dict[str, Any]]:
"""Return records for a specific *model_id*, optionally filtered by cohort.
"""Return records for a specific *model_id*, optionally filtered by GPU.
Args:
local_dir: Root directory previously populated by :func:`sync_from_hf`.
model_id: Matches the ``model_id`` field inside each JSON record.
gpu_type: When set, only records whose ``gpu_type`` matches are returned.
workload_id: When set, only records from the same workload are returned.
variant_id: When set, only records from the same workload variant are returned.
benchmark_version: When set, only records from the same benchmark version are returned.
recipe_fingerprint: When set, only records from the same benchmark
recipe are returned.
hardware_profile_id: When set, only records from the same hardware
cohort are returned.
software_profile_id: When set, only records from the same software
cohort are returned.
last_n: When set, return only the most recent *n* records (after all
other filters). Useful for sliding-window baseline calculations.
successful_only: Passed through to :func:`load_records`.
@@ -314,18 +301,6 @@ def load_records_for_model(
if gpu_type is not None:
records = [r for r in records if r.get("gpu_type") == gpu_type]
identity_filters = {
"workload_id": workload_id,
"variant_id": variant_id,
"benchmark_version": benchmark_version,
"recipe_fingerprint": recipe_fingerprint,
"hardware_profile_id": hardware_profile_id,
"software_profile_id": software_profile_id,
}
for key, expected in identity_filters.items():
if expected is not None:
records = [r for r in records if str(r.get(key)) == str(expected)]
if last_n is not None:
records = records[-last_n:]
@@ -337,12 +312,8 @@ def load_records_for_model(
# ---------------------------------------------------------------------------
_NUMERIC_COLS = (
"latency",
"throughput",
"memory",
"text_encoder_time_s",
"dit_time_s",
"vae_decode_time_s",
"latency", "throughput", "memory",
"text_encoder_time_s", "dit_time_s", "vae_decode_time_s",
)
-435
View File
@@ -1,435 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Comparable identity helpers for performance benchmark records."""
from __future__ import annotations
import dataclasses
import enum
import hashlib
import importlib.metadata
import json
import os
import platform
import re
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Any
import torch
RECIPE_SCHEMA_VERSION = 1
PROFILE_ID_LENGTH = 16
_PATH_EXCLUDED_GENERATION_KEYS = {"output_path", "output_video_name"}
_PROMPT_KEYS = {"prompt", "negative_prompt", "neg_prompt"}
REQUIRED_BENCHMARK_IDENTITY_KEYS = (
"benchmark_id",
"workload_id",
"variant_id",
"benchmark_version",
)
_PROFILE_ENV_VARS = (
"CUDA_VISIBLE_DEVICES",
"FASTVIDEO_ATTENTION_BACKEND",
"IMAGE_VERSION",
"UV_TORCH_BACKEND",
)
_PACKAGE_DISTRIBUTIONS = {
"fastvideo_kernel": ("fastvideo-kernel", "fastvideo_kernel"),
"flash_attn": ("flash-attn", "flash_attn"),
"flash_attn_4": ("flash-attn-4",),
"flash_attention_fp4": ("flash-attention-fp4",),
"flashinfer": ("flashinfer-python", "flashinfer"),
"nvidia_cutlass_dsl": ("nvidia-cutlass-dsl", "cutlass-dsl"),
"sageattention": ("sageattention",),
"sageattn3": ("sageattn3",),
"triton": ("triton",),
"xformers": ("xformers",),
}
def canonical_json(payload: Any) -> str:
"""Return deterministic, compact JSON for a recipe/profile payload."""
return json.dumps(
_canonicalize(payload),
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
)
def sha256_hexdigest(payload: str | bytes) -> str:
if isinstance(payload, str):
payload = payload.encode("utf-8")
return hashlib.sha256(payload).hexdigest()
def recipe_fingerprint(recipe: Mapping[str, Any]) -> str:
# Runtime-resolved values are audit metadata, not identity: hashing the
# resolved HF snapshot revision would churn the cohort (and via the
# no-baseline init path, silently reset the rolling baseline) on ANY
# upstream repo commit, including model-card edits with unchanged
# weights. The fingerprint is pinned to declared inputs only; the
# resolved values stay in the stored recipe for auditability.
pruned = {
key: (dict(value) if isinstance(value, Mapping) else value)
for key, value in recipe.items()
}
model = pruned.get("model")
if isinstance(model, dict):
model.pop("resolved_revision", None)
attention = pruned.get("attention")
if isinstance(attention, dict):
# Same reasoning as resolved_revision: with requested_backend="auto",
# a silent runtime fallback (e.g. a broken flash-attn wheel) must NOT
# open a fresh ungated cohort and seed degraded numbers as the new
# baseline — it should fail the regression gate in the old cohort.
# Intentional backend changes are declared via requested_backend,
# which stays in the hash.
attention.pop("resolved_backend", None)
return sha256_hexdigest(canonical_json(pruned))
def profile_id(prefix: str, profile: Mapping[str, Any]) -> str:
return f"{prefix}-{sha256_hexdigest(canonical_json(profile))[:PROFILE_ID_LENGTH]}"
def hardware_profile_id(profile: Mapping[str, Any]) -> str:
return profile_id("hw", profile)
def software_profile_id(profile: Mapping[str, Any]) -> str:
return profile_id("sw", profile)
def environment_fingerprint(metadata: Mapping[str, Any]) -> str:
return profile_id("env", metadata)
def benchmark_identity_from_config(cfg: Mapping[str, Any]) -> dict[str, Any]:
missing = [
key for key in REQUIRED_BENCHMARK_IDENTITY_KEYS
if _none_if_empty(cfg.get(key)) is None
]
if missing:
raise ValueError("Benchmark config missing required identity fields: " + ", ".join(missing))
return {
key: cfg[key]
for key in REQUIRED_BENCHMARK_IDENTITY_KEYS
}
def build_recipe_from_benchmark_config(
cfg: Mapping[str, Any],
*,
attention_backend: str | None = None,
resolved_attention_backend: Any | None = None,
resolved_model_revision: Any | None = None,
measured_prompts: Sequence[Any] | None = None,
) -> dict[str, Any]:
"""Build a deterministic recipe document from a benchmark config.
The recipe captures fields that describe what benchmark workload was run. It
intentionally excludes timings, timestamps, commit metadata, and output paths.
"""
model = dict(cfg.get("model") or {})
init_kwargs = dict(cfg.get("init_kwargs") or {})
generation_kwargs = dict(cfg.get("generation_kwargs") or {})
benchmark_identity = benchmark_identity_from_config(cfg)
prompts = list(measured_prompts) if measured_prompts is not None else list(
cfg.get("test_prompts") or ["A cinematic video."])
if attention_backend is None:
attention_backend = os.environ.get("FASTVIDEO_ATTENTION_BACKEND")
negative_prompt = generation_kwargs.pop("negative_prompt", generation_kwargs.pop("neg_prompt", None))
generation_recipe = {
key: value
for key, value in generation_kwargs.items()
if key not in _PATH_EXCLUDED_GENERATION_KEYS and key not in _PROMPT_KEYS
}
return {
"recipe_schema_version": RECIPE_SCHEMA_VERSION,
"benchmark": benchmark_identity,
"model": {
"model_path": normalize_model_path(model.get("model_path")),
"revision": _none_if_empty(model.get("revision") or cfg.get("revision")),
"resolved_revision": _none_if_empty(resolved_model_revision),
},
"init_kwargs": init_kwargs,
"generation_kwargs": generation_recipe,
"inputs": {
"prompt_count": len(prompts),
"prompt_sha256": [_optional_digest(prompt) for prompt in prompts],
"negative_prompt_sha256": _optional_digest(negative_prompt),
},
"attention": {
"requested_backend": _none_if_empty(attention_backend) or "auto",
"resolved_backend": _none_if_empty(resolved_attention_backend),
},
}
def normalize_model_path(value: Any) -> str | None:
if value is None:
return None
text = os.fspath(value) if isinstance(value, os.PathLike) else str(value)
text = text.strip()
if not text:
return None
if _looks_like_local_path(text):
return Path(text).expanduser().as_posix()
return re.sub(r"/+", "/", text)
def resolved_revision_from_model_path(value: Any) -> str | None:
normalized = normalize_model_path(value)
if normalized is None:
return None
parts = Path(normalized).parts
for index, part in enumerate(parts[:-1]):
if part == "snapshots":
revision = parts[index + 1]
if re.fullmatch(r"[0-9a-f]{40}", revision):
return revision
return None
def hardware_profile(
*,
num_gpus: int | None = None,
gpu_devices: Sequence[Mapping[str, Any]] | None = None,
device_type: str | None = None,
interconnect: str | None = None,
) -> dict[str, Any]:
"""Return normalized hardware cohort metadata.
``gpu_devices`` is an injection point for tests and non-CUDA callers.
"""
if gpu_devices is not None:
devices = [_normalize_gpu_device(device) for device in gpu_devices]
return {
"device_type": device_type or "cuda",
"gpu_count": num_gpus if num_gpus is not None else len(devices),
"gpus": devices,
"interconnect": interconnect or "unknown",
}
if not torch.cuda.is_available():
return {
"device_type": device_type or "cpu",
"gpu_count": 0,
"gpus": [],
"interconnect": interconnect or "none",
}
gpu_count = int(num_gpus if num_gpus is not None else torch.cuda.device_count())
devices = []
cuda_device_count = torch.cuda.device_count()
for device_id in range(gpu_count):
if device_id >= cuda_device_count:
devices.append(_normalize_gpu_device({}))
continue
props = torch.cuda.get_device_properties(device_id)
capability = torch.cuda.get_device_capability(device_id)
devices.append(
_normalize_gpu_device({
"name": props.name,
"memory_bytes": props.total_memory,
"compute_capability": f"{capability[0]}.{capability[1]}",
}))
return {
"device_type": device_type or "cuda",
"gpu_count": gpu_count,
"gpus": devices,
"interconnect": interconnect or _detect_cuda_interconnect(gpu_count),
}
def software_profile(
*,
python_version: str | None = None,
torch_version: str | None = None,
cuda_version: str | None = None,
package_versions: Mapping[str, str | None] | None = None,
) -> dict[str, Any]:
"""Return normalized software cohort metadata.
Python, PyTorch, and CUDA use major/minor cohorts. Attention and kernel
packages keep exact versions because patch releases can change performance.
"""
versions = dict(package_versions) if package_versions is not None else _installed_package_versions()
return {
"python": _major_minor(python_version or platform.python_version()),
"pytorch": _major_minor(torch_version or torch.__version__),
"cuda": _major_minor(cuda_version or torch.version.cuda),
"packages": {
name: str(version)
for name, version in sorted(versions.items())
if version
},
}
def environment_metadata(
*,
env: Mapping[str, str] | None = None,
package_versions: Mapping[str, str | None] | None = None,
hardware: Mapping[str, Any] | None = None,
software: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
"""Return audit metadata kept separate from comparable identity keys."""
source_env = env if env is not None else os.environ
full_package_versions = dict(package_versions) if package_versions is not None else _installed_package_versions()
return {
"python": {
"version": platform.python_version(),
"implementation": platform.python_implementation(),
},
"platform": {
"system": platform.system(),
"machine": platform.machine(),
"release": platform.release(),
},
"torch": {
"version": torch.__version__,
"cuda": torch.version.cuda,
"cudnn": torch.backends.cudnn.version(),
},
"env": {
key: source_env.get(key)
for key in _PROFILE_ENV_VARS
if source_env.get(key) is not None
},
"packages": {
key: value
for key, value in sorted(full_package_versions.items())
if value
},
"hardware_profile": dict(hardware) if hardware is not None else hardware_profile(),
"software_profile": dict(software) if software is not None else software_profile(
package_versions=full_package_versions),
}
def _canonicalize(value: Any) -> Any:
if dataclasses.is_dataclass(value) and not isinstance(value, type):
return _canonicalize(dataclasses.asdict(value))
if isinstance(value, Mapping):
return {
str(key): _canonicalize(value[key])
for key in sorted(value, key=lambda item: str(item))
}
if isinstance(value, (set, frozenset)):
return [_canonicalize(item) for item in sorted(value, key=lambda item: str(item))]
if isinstance(value, tuple):
return [_canonicalize(item) for item in value]
if isinstance(value, list):
return [_canonicalize(item) for item in value]
if isinstance(value, enum.Enum):
return value.name
if isinstance(value, Path):
return value.expanduser().as_posix()
if isinstance(value, os.PathLike):
return os.fspath(value)
if isinstance(value, type):
return f"{value.__module__}.{value.__qualname__}"
if isinstance(value, torch.dtype):
return str(value).removeprefix("torch.")
return value
def _none_if_empty(value: Any) -> Any | None:
if value is None:
return None
if isinstance(value, str) and not value.strip():
return None
return value
def _optional_digest(value: Any) -> str | None:
if value is None:
return None
return sha256_hexdigest(str(value))
def _looks_like_local_path(value: str) -> bool:
if value.startswith(("/", "./", "../", "~")):
return True
looks_like_hf_repo = re.match(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$", value)
has_separator = "/" in value or "\\" in value or os.sep in value
return has_separator and looks_like_hf_repo is None
def _normalize_gpu_device(device: Mapping[str, Any]) -> dict[str, Any]:
memory_gb = device.get("memory_gb")
if memory_gb is None and device.get("memory_bytes") is not None:
memory_gb = round(float(device["memory_bytes"]) / (1024**3))
return {
"name": _none_if_empty(device.get("name")) or "unknown",
"memory_gb": memory_gb,
"compute_capability": _none_if_empty(device.get("compute_capability")),
}
def _detect_cuda_interconnect(gpu_count: int) -> str:
if gpu_count <= 1:
return "single_gpu"
try:
from fastvideo.platforms import current_platform
if current_platform.is_cuda() and current_platform.is_full_nvlink(list(range(gpu_count))):
return "full_nvlink"
return "none_or_partial"
except Exception:
return "unknown"
def _installed_package_versions() -> dict[str, str | None]:
return {
name: _distribution_version(*distribution_names)
for name, distribution_names in _PACKAGE_DISTRIBUTIONS.items()
}
def _distribution_version(*names: str) -> str | None:
for name in names:
try:
return importlib.metadata.version(name)
except importlib.metadata.PackageNotFoundError:
continue
return None
def _major_minor(version: str | None) -> str | None:
if not version:
return None
match = re.search(r"(\d+)\.(\d+)", version)
if match is None:
return version
return f"{match.group(1)}.{match.group(2)}"
__all__ = [
"RECIPE_SCHEMA_VERSION",
"REQUIRED_BENCHMARK_IDENTITY_KEYS",
"benchmark_identity_from_config",
"canonical_json",
"environment_fingerprint",
"environment_metadata",
"hardware_profile",
"hardware_profile_id",
"normalize_model_path",
"profile_id",
"recipe_fingerprint",
"resolved_revision_from_model_path",
"build_recipe_from_benchmark_config",
"sha256_hexdigest",
"software_profile",
"software_profile_id",
]
@@ -14,9 +14,9 @@ def _v2_config():
return {
"benchmark_id": "wan-t2v-1.3b-2gpu",
"config_schema_version": 2,
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"workload_id": "wan-t2v-1.3b",
"variant_id": "canonical",
"benchmark_version": 1,
}
@@ -41,9 +41,9 @@ def test_v2_benchmark_config_identity_validates_and_is_preserved():
assert _is_v2_config(cfg) is True
assert _config_identity_metadata(cfg) == {
"config_schema_version": 2,
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"workload_id": "wan-t2v-1.3b",
"variant_id": "canonical",
"benchmark_version": 1,
"quality_metadata": {"some": "data"},
}
@@ -91,9 +91,9 @@ def test_v2_benchmark_config_rejects_invalid_benchmark_version_values(value):
def test_partial_v2_identity_requires_schema_version():
cfg = {
"benchmark_id": "wan-t2v-1.3b-2gpu",
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"workload_id": "wan-t2v-1.3b",
"variant_id": "canonical",
"benchmark_version": 1,
}
expected = "wan.json: v2 benchmark identity fields require config_schema_version=2"
@@ -105,9 +105,9 @@ def test_optional_v2_metadata_fields_must_be_objects():
cfg = {
"benchmark_id": "wan-t2v-1.3b-2gpu",
"config_schema_version": 2,
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"workload_id": "wan-t2v-1.3b",
"variant_id": "canonical",
"benchmark_version": 1,
"quality_metadata": ["not", "an", "object"],
}
@@ -1,11 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
import json
import pytest
from fastvideo.tests.performance import compare_baseline
from fastvideo.performance.metric_policy import resolve_metric_policies
def _raw_result():
@@ -77,347 +72,9 @@ def test_normalized_record_includes_source_metadata(monkeypatch):
assert record["job_id"] == "job-1"
def test_normalized_record_includes_effective_regression_thresholds(monkeypatch):
monkeypatch.setenv("PERF_RUN_SOURCE", "pr")
raw = _raw_result()
raw["regression_thresholds"] = {
"latency": {
"threshold_percent": 0.09,
"threshold_absolute": 0.75,
"gated": True,
},
"throughput": {
"gated": False,
},
}
record = compare_baseline.normalize_performance_result(raw)
assert record["regression_thresholds"]["latency"] == {
"threshold_percent": 0.09,
"threshold_absolute": 0.75,
"gated": True,
}
assert record["regression_thresholds"]["throughput"]["gated"] is False
def test_invalid_regression_threshold_container_uses_defaults():
policies = resolve_metric_policies(["not", "a", "mapping"])
latency = next(policy for policy in policies if policy.key == "latency")
assert latency.threshold_percent == 0.08
assert latency.threshold_absolute == 0.5
assert latency.gated is True
def test_boolean_regression_threshold_values_are_ignored():
policies = resolve_metric_policies({
"latency": {
"threshold_percent": True,
"threshold_absolute": False,
"gated": "false",
}
})
latency = next(policy for policy in policies if policy.key == "latency")
assert latency.threshold_percent == 0.08
assert latency.threshold_absolute == 0.5
assert latency.gated is False
def test_normalized_record_preserves_identity_metadata(monkeypatch):
monkeypatch.setenv("PERF_RUN_SOURCE", "pr")
raw = _raw_result()
raw.update({
"result_schema_version": 2,
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"recipe": {
"recipe_schema_version": 1,
},
"recipe_fingerprint": "recipe-1",
"hardware_profile": {
"gpu_count": 1,
},
"hardware_profile_id": "hw-1",
"software_profile": {
"python": "3.12",
},
"software_profile_id": "sw-1",
"environment_metadata": {
"env": {
"IMAGE_VERSION": "latest",
},
},
"environment_fingerprint": "env-1",
"quality_metadata": {
"quality_status": "canonical",
},
})
record = compare_baseline.normalize_performance_result(raw)
assert record["result_schema_version"] == 2
assert record["workload_id"] == "wan-t2v"
assert record["variant_id"] == "1.3b-sp2"
assert record["benchmark_version"] == 2
assert record["recipe"] == {"recipe_schema_version": 1}
assert record["recipe_fingerprint"] == "recipe-1"
assert record["hardware_profile"] == {"gpu_count": 1}
assert record["hardware_profile_id"] == "hw-1"
assert record["software_profile"] == {"python": "3.12"}
assert record["software_profile_id"] == "sw-1"
assert record["environment_metadata"] == {"env": {"IMAGE_VERSION": "latest"}}
assert record["environment_fingerprint"] == "env-1"
assert record["quality_metadata"] == {"quality_status": "canonical"}
def test_normalized_record_prefers_raw_v2_provenance(monkeypatch):
monkeypatch.setenv("PERF_RUN_SOURCE", "pr")
monkeypatch.setenv("BUILDKITE_BRANCH", "feature/from-env")
monkeypatch.setenv("TEST_SCOPE", "direct")
monkeypatch.setenv("BUILDKITE_BUILD_URL", "https://buildkite.example/env")
monkeypatch.setenv("BUILDKITE_BUILD_ID", "env-build")
monkeypatch.setenv("BUILDKITE_JOB_ID", "env-job")
raw = _raw_result()
raw.update({
"result_schema_version": 2,
"run_source": "scheduled_main",
"branch": "main",
"test_scope": "full",
"build_url": "https://buildkite.example/raw",
"build_id": "raw-build",
"job_id": "raw-job",
"pr_number": "false",
})
record = compare_baseline.normalize_performance_result(raw)
assert record["result_schema_version"] == 2
assert record["run_source"] == "scheduled_main"
assert record["branch"] == "main"
assert record["test_scope"] == "full"
assert record["build_url"] == "https://buildkite.example/raw"
assert record["build_id"] == "raw-build"
assert record["job_id"] == "raw-job"
assert record["pr_number"] == ""
def test_v1_normalized_record_has_no_result_schema_version(monkeypatch):
monkeypatch.setenv("PERF_RUN_SOURCE", "pr")
record = compare_baseline.normalize_performance_result(_raw_result())
assert "result_schema_version" not in record
def test_main_writes_v1_current_artifact_without_comparison_identity(monkeypatch, tmp_path, capsys):
results_dir = tmp_path / "results"
reports_dir = tmp_path / "reports"
tracking_root = tmp_path / "tracking"
results_dir.mkdir()
(results_dir / "perf_legacy.json").write_text(json.dumps(_raw_result()), encoding="utf-8")
uploaded_records = []
monkeypatch.setenv("PERF_RUN_SOURCE", "scheduled_main")
monkeypatch.delenv("PERF_PYTEST_RC", raising=False)
monkeypatch.setattr(compare_baseline, "RESULTS_DIR", str(results_dir))
monkeypatch.setattr(compare_baseline, "PERF_REPORTS_DIR", str(reports_dir))
monkeypatch.setattr(compare_baseline, "TRACKING_ROOT", str(tracking_root))
monkeypatch.setattr(compare_baseline, "UPLOAD_POLICY", "always")
monkeypatch.setattr(compare_baseline, "sync_from_hf", lambda local_dir, strict=False: local_dir)
def fail_load_records_for_model(*_args, **_kwargs):
raise AssertionError("legacy current records should skip v2 baseline lookup")
def fake_upload_record(_path, record, *, strict=False):
uploaded_records.append(record.copy())
monkeypatch.setattr(compare_baseline, "load_records_for_model", fail_load_records_for_model)
monkeypatch.setattr(compare_baseline, "upload_record", fake_upload_record)
assert compare_baseline.main() == 0
output = capsys.readouterr().out
normalized_files = list((reports_dir / "results").glob("normalized_perf_*.json"))
assert "Skipping rolling baseline comparison" in output
assert "workload_id" in output
assert len(normalized_files) == 1
normalized = json.loads(normalized_files[0].read_text(encoding="utf-8"))
assert "result_schema_version" not in normalized
assert normalized["success"] is True
assert normalized["baseline_eligible"] is False
assert len(uploaded_records) == 1
assert uploaded_records[0]["baseline_eligible"] is False
def test_normalized_record_reads_identity_labels_from_recipe(monkeypatch):
monkeypatch.setenv("PERF_RUN_SOURCE", "pr")
raw = _raw_result()
raw["recipe"] = {
"benchmark": {
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
},
}
record = compare_baseline.normalize_performance_result(raw)
assert record["workload_id"] == "wan-t2v"
assert record["variant_id"] == "1.3b-sp2"
assert record["benchmark_version"] == 2
def test_comparison_identity_filters_use_full_issue_key():
record = {
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"recipe_fingerprint": "recipe-1",
"hardware_profile_id": "hw-1",
"software_profile_id": "sw-1",
}
assert compare_baseline._comparison_identity_filters(record) == {
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": "2",
"recipe_fingerprint": "recipe-1",
"hardware_profile_id": "hw-1",
"software_profile_id": "sw-1",
}
def test_comparison_identity_filters_require_full_issue_key():
record = {
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 0,
"recipe_fingerprint": "recipe-1",
"hardware_profile_id": "hw-1",
}
with pytest.raises(ValueError, match="software_profile_id"):
compare_baseline._comparison_identity_filters(record)
def test_comparison_identity_filters_keep_zero_version():
record = {
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 0,
"recipe_fingerprint": "recipe-1",
"hardware_profile_id": "hw-1",
"software_profile_id": "sw-1",
}
assert compare_baseline._comparison_identity_filters(record)["benchmark_version"] == "0"
def test_baseline_eligibility_only_for_successful_scheduled_main():
assert compare_baseline._is_baseline_eligible("scheduled_main", True) is True
assert compare_baseline._is_baseline_eligible("scheduled_main", False) is False
assert compare_baseline._is_baseline_eligible("pr", True) is False
assert compare_baseline._is_baseline_eligible("local", True) is False
def test_latency_regression_requires_percent_and_absolute_floors():
baseline = [{"latency": 10.0}]
current = {"model_id": "wan", "latency": 10.6}
percent_only = resolve_metric_policies({
"latency": {
"threshold_percent": 0.05,
"threshold_absolute": 0.75,
}
})
absolute_only = resolve_metric_policies({
"latency": {
"threshold_percent": 0.10,
"threshold_absolute": 0.5,
}
})
both = resolve_metric_policies({
"latency": {
"threshold_percent": 0.05,
"threshold_absolute": 0.5,
}
})
assert compare_baseline._check_regressions(current, baseline, percent_only) == []
assert compare_baseline._check_regressions(current, baseline, absolute_only) == []
failures = compare_baseline._check_regressions(current, baseline, both)
assert len(failures) == 1
assert "latency regressed by 6.0% and 0.600" in failures[0]
def test_throughput_regression_uses_higher_is_better_direction():
baseline = [{"throughput": 10.0}]
current = {"model_id": "wan", "throughput": 9.0}
policies = resolve_metric_policies({
"throughput": {
"threshold_percent": 0.05,
"threshold_absolute": 0.5,
}
})
failures = compare_baseline._check_regressions(current, baseline, policies)
assert len(failures) == 1
assert "throughput regressed by 10.0% and 1.000" in failures[0]
def test_memory_regression_uses_metric_specific_absolute_floor():
baseline = [{"memory": 10000.0}]
current = {"model_id": "wan", "memory": 10600.0}
policies = resolve_metric_policies({
"memory": {
"threshold_percent": 0.05,
"threshold_absolute": 256.0,
}
})
failures = compare_baseline._check_regressions(current, baseline, policies)
assert len(failures) == 1
assert "memory regressed by 6.0% and 600.0" in failures[0]
def test_component_metric_can_gate_independently():
baseline = [{"dit_time_s": 8.0}]
current = {"model_id": "wan", "dit_time_s": 8.6}
policies = resolve_metric_policies({
"dit_time_s": {
"threshold_percent": 0.05,
"threshold_absolute": 0.25,
}
})
failures = compare_baseline._check_regressions(current, baseline, policies)
assert len(failures) == 1
assert "dit_time_s regressed by 7.5% and 0.600" in failures[0]
def test_informational_metric_remains_visible_without_failing():
baseline = [{"throughput": 10.0}]
current = {"model_id": "wan", "gpu_type": "NVIDIA L40S", "throughput": 8.0}
policies = resolve_metric_policies({
"throughput": {
"threshold_percent": 0.01,
"threshold_absolute": 0.01,
"gated": False,
}
})
row = compare_baseline._build_summary_row(current, baseline, policies, False)
assert compare_baseline._check_regressions(current, baseline, policies) == []
assert row["metrics"]["throughput"]["regression_pct"] == 20.0
assert row["metrics"]["throughput"]["gated"] is False
assert row["metrics"]["throughput"]["threshold_exceeded"] is True
assert row["metrics"]["throughput"]["regressed"] is False
assert row["threshold_exceeded_metrics"] == ["throughput"]
assert row["failing_metrics"] == []
@@ -56,34 +56,8 @@ def _record(model_id, gpu_type, ts, commit, latency, throughput, success=True, *
def test_summary_endpoint_returns_latest_group_status():
app = create_app(FakeStore([
_record(
"wan",
"NVIDIA L40S",
"2026-01-01T00:00:00+00:00",
"a" * 40,
10.0,
10.0,
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=2,
recipe_fingerprint="recipe-a",
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
),
_record(
"wan",
"NVIDIA L40S",
"2026-01-02T00:00:00+00:00",
"b" * 40,
11.0,
9.0,
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=2,
recipe_fingerprint="recipe-a",
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
),
_record("wan", "NVIDIA L40S", "2026-01-01T00:00:00+00:00", "a" * 40, 10.0, 10.0),
_record("wan", "NVIDIA L40S", "2026-01-02T00:00:00+00:00", "b" * 40, 11.0, 9.0),
]))
client = TestClient(app)
@@ -94,13 +68,7 @@ def test_summary_endpoint_returns_latest_group_status():
assert body["count"] == 1
assert body["status_counts"] == {"pass": 1, "fail": 0}
assert body["rows"][0]["metrics"]["latency"]["baseline"] == 10.0
assert body["rows"][0]["metrics"]["latency"]["threshold_exceeded"] is True
assert body["rows"][0]["threshold_exceeded_metrics"] == ["latency", "throughput"]
assert body["rows"][0]["computed_regression_status"] == "fail"
assert body["rows"][0]["workload_id"] == "wan-t2v"
assert body["rows"][0]["variant_id"] == "1.3b-sp2"
assert body["rows"][0]["benchmark_version"] == 2
assert body["rows"][0]["recipe_fingerprint"] == "recipe-a"
def test_summary_status_is_independent_of_days_window():
@@ -154,8 +122,8 @@ def test_dashboard_endpoints_filter_and_return_run_source_metadata():
assert summary["count"] == 1
assert summary["rows"][0]["run_source"] == "pr"
assert summary["rows"][0]["pr_number"] == "123"
assert summary["rows"][0]["baseline_n"] == 0
assert summary["rows"][0]["metrics"]["latency"]["baseline"] is None
assert summary["rows"][0]["baseline_n"] == 1
assert summary["rows"][0]["metrics"]["latency"]["baseline"] == 11.0
assert summary["filters"]["run_source"] == "pr"
assert trends["count"] == 1
assert trends["groups"][0]["points"][0]["run_source"] == "pr"
@@ -1,98 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
import pandas as pd
from fastvideo.tests.performance import dashboard
def _record(recipe_fingerprint="recipe-a", software_profile_id="sw-a", **overrides):
record = {
"model_id": "wan-t2v-1.3b-2gpu",
"gpu_type": "NVIDIA L40S",
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"recipe_fingerprint": recipe_fingerprint,
"hardware_profile_id": "hw-l40s",
"software_profile_id": software_profile_id,
"timestamp": "2026-01-01T00:00:00+00:00",
"commit_sha": "a" * 40,
"config_id": "aaaaaaa",
"latency": 10.0,
"throughput": 4.5,
"memory": 10000.0,
"text_encoder_time_s": 2.0,
"dit_time_s": 8.0,
"vae_decode_time_s": 3.0,
}
record.update(overrides)
return record
def test_group_data_uses_full_comparison_cohort():
df = pd.DataFrame([
_record(recipe_fingerprint="recipe-a", software_profile_id="sw-a"),
_record(recipe_fingerprint="recipe-b", software_profile_id="sw-b"),
])
groups = list(dashboard.group_data(df))
assert len(groups) == 2
assert {key[5] for key, _group in groups} == {"recipe-a", "recipe-b"}
assert {key[7] for key, _group in groups} == {"sw-a", "sw-b"}
def test_group_data_fills_legacy_cohort_columns():
df = pd.DataFrame([{
"model_id": "legacy-wan",
"gpu_type": "NVIDIA L40S",
"timestamp": "2026-01-01T00:00:00+00:00",
"commit_sha": "a" * 40,
"config_id": "aaaaaaa",
"latency": 10.0,
}])
groups = list(dashboard.group_data(df))
assert len(groups) == 1
assert groups[0][0][2:] == ("", "", "", "", "", "")
def test_build_plots_labels_distinct_cohorts():
df = pd.DataFrame([
_record(recipe_fingerprint="recipe-a", software_profile_id="sw-a"),
_record(recipe_fingerprint="recipe-b", software_profile_id="sw-b"),
])
figs, skipped_metrics = dashboard.build_plots(df)
titles = [fig.layout.title.text for fig in figs]
assert len(figs) == len(dashboard.METRICS) * 2
assert skipped_metrics == []
assert any("recipe recipe-a | hw-l40s | sw-a" in title for title in titles)
assert any("recipe recipe-b | hw-l40s | sw-b" in title for title in titles)
def test_skipped_metric_table_includes_cohort_identity():
df = pd.DataFrame([{
"model_id": "wan-t2v-1.3b-2gpu",
"gpu_type": "NVIDIA L40S",
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"recipe_fingerprint": "recipe-a",
"hardware_profile_id": "hw-l40s",
"software_profile_id": "sw-a",
"timestamp": "2026-01-01T00:00:00+00:00",
"commit_sha": "a" * 40,
"config_id": "aaaaaaa",
"latency": 10.0,
}])
_figs, skipped_metrics = dashboard.build_plots(df)
html = dashboard.render_skipped_metrics(skipped_metrics[:1])
assert skipped_metrics[0]["cohort"] == "wan-t2v / 1.3b-sp2 / v2"
assert skipped_metrics[0]["cohort_detail"] == "recipe recipe-a | hw-l40s | sw-a"
assert "wan-t2v / 1.3b-sp2 / v2" in html
assert "recipe recipe-a | hw-l40s | sw-a" in html
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
from fastvideo.performance import hf_store
from fastvideo.performance_dashboard.service import build_latest_summary, build_trends, filter_records
from fastvideo.tests.performance import hf_store
def _record(ts, commit, latency, throughput, success=True, **metadata):
@@ -28,23 +28,16 @@ def test_build_latest_summary_uses_previous_successful_records_for_baseline():
_record("2026-01-03T00:00:00+00:00", "c" * 40, 11.0, 9.0),
]
rows = build_latest_summary(records)
rows = build_latest_summary(records, max_regression=0.05)
assert len(rows) == 1
row = rows[0]
assert row["baseline_n"] == 1
assert row["metrics"]["latency"]["baseline"] == 10.0
assert row["metrics"]["latency"]["regression_pct"] == 10.0
assert row["metrics"]["latency"]["absolute_delta"] == 1.0
assert row["metrics"]["latency"]["threshold_percent"] == 8.0
assert row["metrics"]["latency"]["threshold_absolute"] == 0.5
assert row["metrics"]["latency"]["threshold_exceeded"] is True
assert row["metrics"]["latency"]["regressed"] is True
assert row["metrics"]["throughput"]["regression_pct"] == 10.0
assert row["status"] == "pass"
assert row["computed_regression_status"] == "fail"
assert row["threshold_exceeded_metrics"] == ["latency", "throughput"]
assert row["failing_metrics"] == ["latency", "throughput"]
def test_build_latest_summary_status_uses_latest_record_success_field():
@@ -80,7 +73,7 @@ def test_build_latest_summary_run_source_filter_keeps_canonical_baseline():
),
]
rows = build_latest_summary(records, run_source="pr")
rows = build_latest_summary(records, max_regression=0.05, run_source="pr")
assert len(rows) == 1
assert rows[0]["run_source"] == "pr"
@@ -90,258 +83,6 @@ def test_build_latest_summary_run_source_filter_keeps_canonical_baseline():
assert rows[0]["computed_regression_status"] == "fail"
def test_build_latest_summary_requires_absolute_floor_for_computed_regression():
records = [
_record("2026-01-01T00:00:00+00:00", "a" * 40, 10.0, 10.0),
_record(
"2026-01-02T00:00:00+00:00",
"b" * 40,
10.6,
10.0,
regression_thresholds={
"latency": {
"threshold_percent": 0.05,
"threshold_absolute": 0.75,
"gated": True,
}
},
),
]
rows = build_latest_summary(records)
assert round(rows[0]["metrics"]["latency"]["regression_pct"], 1) == 6.0
assert round(rows[0]["metrics"]["latency"]["absolute_delta"], 3) == 0.6
assert rows[0]["metrics"]["latency"]["threshold_exceeded"] is False
assert rows[0]["metrics"]["latency"]["regressed"] is False
assert rows[0]["computed_regression_status"] == "pass"
def test_build_latest_summary_separates_informational_threshold_crossing():
records = [
_record("2026-01-01T00:00:00+00:00", "a" * 40, 10.0, 10.0),
_record(
"2026-01-02T00:00:00+00:00",
"b" * 40,
10.6,
10.0,
regression_thresholds={
"latency": {
"threshold_percent": 0.05,
"threshold_absolute": 0.5,
"gated": False,
}
},
),
]
rows = build_latest_summary(records)
assert rows[0]["metrics"]["latency"]["threshold_exceeded"] is True
assert rows[0]["metrics"]["latency"]["regressed"] is False
assert rows[0]["threshold_exceeded_metrics"] == ["latency"]
assert rows[0]["failing_metrics"] == []
assert rows[0]["computed_regression_status"] == "pass"
def test_build_latest_summary_run_source_filter_excludes_future_baselines():
records = [
_record(
"2026-01-01T00:00:00+00:00",
"a" * 40,
10.0,
10.0,
run_source="scheduled_main",
baseline_eligible=True,
),
_record(
"2026-01-02T00:00:00+00:00",
"b" * 40,
11.0,
9.0,
run_source="pr",
baseline_eligible=False,
pr_number="123",
),
_record(
"2026-01-03T00:00:00+00:00",
"c" * 40,
30.0,
3.0,
run_source="scheduled_main",
baseline_eligible=True,
),
]
rows = build_latest_summary(records, run_source="pr")
assert len(rows) == 1
assert rows[0]["run_source"] == "pr"
assert rows[0]["baseline_n"] == 1
assert rows[0]["metrics"]["latency"]["baseline"] == 10.0
assert rows[0]["metrics"]["throughput"]["baseline"] == 10.0
def test_build_latest_summary_keeps_identity_cohorts_separate():
records = [
_record(
"2026-01-01T00:00:00+00:00",
"a" * 40,
10.0,
10.0,
recipe_fingerprint="recipe-a",
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=2,
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
run_source="scheduled_main",
baseline_eligible=True,
),
_record(
"2026-01-02T00:00:00+00:00",
"b" * 40,
20.0,
5.0,
recipe_fingerprint="recipe-b",
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=2,
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
run_source="scheduled_main",
baseline_eligible=True,
),
_record(
"2026-01-03T00:00:00+00:00",
"c" * 40,
22.0,
4.5,
recipe_fingerprint="recipe-b",
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=2,
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
),
]
rows = build_latest_summary(records)
recipe_b_row = next(row for row in rows if row["recipe_fingerprint"] == "recipe-b")
assert len(rows) == 2
assert recipe_b_row["baseline_n"] == 1
assert recipe_b_row["metrics"]["latency"]["baseline"] == 20.0
assert recipe_b_row["workload_id"] == "wan-t2v"
assert recipe_b_row["variant_id"] == "1.3b-sp2"
assert recipe_b_row["benchmark_version"] == 2
def test_build_latest_summary_keeps_variant_versions_separate():
records = [
_record(
"2026-01-01T00:00:00+00:00",
"a" * 40,
10.0,
10.0,
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=1,
recipe_fingerprint="recipe-a",
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
run_source="scheduled_main",
baseline_eligible=True,
),
_record(
"2026-01-02T00:00:00+00:00",
"b" * 40,
20.0,
5.0,
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=2,
recipe_fingerprint="recipe-a",
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
run_source="scheduled_main",
baseline_eligible=True,
),
_record(
"2026-01-03T00:00:00+00:00",
"c" * 40,
22.0,
4.5,
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=2,
recipe_fingerprint="recipe-a",
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
),
]
rows = build_latest_summary(records)
version_2_row = next(row for row in rows if row["benchmark_version"] == 2)
assert len(rows) == 2
assert version_2_row["baseline_n"] == 1
assert version_2_row["metrics"]["latency"]["baseline"] == 20.0
def test_dashboard_identity_preserves_zero_benchmark_version():
records = [
_record(
"2026-01-01T00:00:00+00:00",
"a" * 40,
10.0,
10.0,
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=0,
recipe_fingerprint="recipe-a",
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
run_source="scheduled_main",
baseline_eligible=True,
),
_record(
"2026-01-02T00:00:00+00:00",
"b" * 40,
11.0,
9.0,
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=0,
recipe_fingerprint="recipe-a",
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
),
_record(
"2026-01-03T00:00:00+00:00",
"c" * 40,
20.0,
5.0,
),
]
rows = build_latest_summary(records)
version_zero_row = next(row for row in rows if row["benchmark_version"] == 0)
legacy_row = next(row for row in rows if row["benchmark_version"] == "")
trends = build_trends(records)
version_zero_trend = next(trend for trend in trends if trend["benchmark_version"] == 0)
legacy_trend = next(trend for trend in trends if trend["benchmark_version"] == "")
assert len(rows) == 2
assert version_zero_row["baseline_n"] == 1
assert version_zero_row["metrics"]["latency"]["baseline"] == 10.0
assert legacy_row["baseline_n"] == 0
assert len(trends) == 2
assert version_zero_trend["points"][0]["benchmark_version"] == 0
assert legacy_trend["points"][0]["benchmark_version"] == ""
def test_filter_records_and_trends_preserve_metric_points():
records = [
_record("2026-01-01T00:00:00+00:00", "a" * 40, 10.0, 10.0),
@@ -416,78 +157,3 @@ def test_load_records_can_filter_baseline_eligible_records(tmp_path):
"2026-01-02T00:00:00+00:00",
"2026-01-03T00:00:00+00:00",
}
def test_load_records_for_model_filters_identity_cohort(tmp_path):
model_dir = tmp_path / "wan"
model_dir.mkdir()
(model_dir / "matching.json").write_text(
"""
{
"model_id": "wan",
"gpu_type": "NVIDIA L40S",
"timestamp": "2026-01-01T00:00:00+00:00",
"success": true,
"baseline_eligible": true,
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"recipe_fingerprint": "recipe-a",
"hardware_profile_id": "hw-l40s",
"software_profile_id": "sw-cu130"
}
""",
encoding="utf-8",
)
(model_dir / "other_recipe.json").write_text(
"""
{
"model_id": "wan",
"gpu_type": "NVIDIA L40S",
"timestamp": "2026-01-02T00:00:00+00:00",
"success": true,
"baseline_eligible": true,
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"recipe_fingerprint": "recipe-b",
"hardware_profile_id": "hw-l40s",
"software_profile_id": "sw-cu130"
}
""",
encoding="utf-8",
)
(model_dir / "other_version.json").write_text(
"""
{
"model_id": "wan",
"gpu_type": "NVIDIA L40S",
"timestamp": "2026-01-03T00:00:00+00:00",
"success": true,
"baseline_eligible": true,
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 3,
"recipe_fingerprint": "recipe-a",
"hardware_profile_id": "hw-l40s",
"software_profile_id": "sw-cu130"
}
""",
encoding="utf-8",
)
records = hf_store.load_records_for_model(
str(tmp_path),
"wan",
"NVIDIA L40S",
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version="2",
recipe_fingerprint="recipe-a",
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
baseline_eligible_only=True,
)
assert len(records) == 1
assert records[0]["recipe_fingerprint"] == "recipe-a"
@@ -1,422 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
from copy import deepcopy
import pytest
import torch
from fastvideo.tests.performance import identity as identity_module
from fastvideo.tests.performance.identity import (
build_recipe_from_benchmark_config,
canonical_json,
environment_fingerprint,
environment_metadata,
hardware_profile,
hardware_profile_id,
recipe_fingerprint,
resolved_revision_from_model_path,
software_profile,
software_profile_id,
)
def _benchmark_config():
return {
"benchmark_id": "wan-t2v-1.3b-2gpu",
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"model": {
"model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
"model_short_name": "Wan2.1-T2V-1.3B",
},
"init_kwargs": {
"num_gpus": 2,
"sp_size": 2,
"tp_size": 1,
"vae_sp": True,
"vae_tiling": True,
"text_encoder_precisions": ["fp32"],
},
"generation_kwargs": {
"height": 480,
"width": 832,
"num_frames": 45,
"num_inference_steps": 4,
"guidance_scale": 3,
"embedded_cfg_scale": 6,
"seed": 1024,
"fps": 24,
"neg_prompt": "low quality",
},
"test_prompts": ["A cinematic video."],
}
def _fingerprint(cfg, attention_backend="FLASH_ATTN"):
recipe = build_recipe_from_benchmark_config(cfg, attention_backend=attention_backend)
return recipe_fingerprint(recipe)
def test_canonical_json_is_stable_for_mapping_order():
left = {"b": [2, 1], "a": {"z": "last", "m": "middle"}}
right = {"a": {"m": "middle", "z": "last"}, "b": [2, 1]}
assert canonical_json(left) == canonical_json(right)
def test_canonical_json_handles_sets_deterministically():
assert canonical_json({"values": {"b", "a"}}) == '{"values":["a","b"]}'
assert canonical_json({"values": frozenset(("b", "a"))}) == '{"values":["a","b"]}'
def test_same_config_produces_same_recipe_fingerprint():
cfg = _benchmark_config()
assert _fingerprint(cfg) == _fingerprint(deepcopy(cfg))
def test_recipe_includes_first_class_benchmark_identity():
recipe = build_recipe_from_benchmark_config(_benchmark_config())
assert recipe["benchmark"] == {
"benchmark_id": "wan-t2v-1.3b-2gpu",
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
}
def test_recipe_requires_first_class_benchmark_identity():
cfg = _benchmark_config()
del cfg["workload_id"]
with pytest.raises(ValueError, match="workload_id"):
build_recipe_from_benchmark_config(cfg)
def test_semantically_equivalent_sequence_forms_match():
cfg = _benchmark_config()
equivalent = deepcopy(cfg)
equivalent["init_kwargs"]["text_encoder_precisions"] = tuple(
equivalent["init_kwargs"]["text_encoder_precisions"])
assert _fingerprint(cfg) == _fingerprint(equivalent)
def test_output_path_does_not_change_recipe_fingerprint():
cfg = _benchmark_config()
with_output_path = deepcopy(cfg)
with_output_path["generation_kwargs"]["output_path"] = "/tmp/generated"
assert _fingerprint(cfg) == _fingerprint(with_output_path)
def test_runtime_resolved_values_do_not_change_recipe_fingerprint():
# Runtime-resolved values are audit metadata: an upstream repo commit
# (even a README-only edit) or a silent attention-backend fallback must
# not churn the cohort — that would open a fresh ungated cohort and let
# degraded numbers seed a new baseline. Declared inputs (including
# requested_backend) stay in the hash.
cfg = _benchmark_config()
recipe = build_recipe_from_benchmark_config(
cfg, attention_backend="FLASH_ATTN",
resolved_attention_backend="FLASH_ATTN",
resolved_model_revision="aaaa1111")
changed = build_recipe_from_benchmark_config(
cfg, attention_backend="FLASH_ATTN",
resolved_attention_backend="TORCH_SDPA",
resolved_model_revision="bbbb2222")
assert recipe["model"]["resolved_revision"] == "aaaa1111" # kept for audit
assert recipe["attention"]["resolved_backend"] == "FLASH_ATTN" # kept for audit
assert recipe_fingerprint(recipe) == recipe_fingerprint(changed)
def test_measured_prompt_override_ignores_unused_configured_prompts():
cfg = _benchmark_config()
changed = deepcopy(cfg)
changed["test_prompts"] = [
"Unused prompt that should not affect this measured workload.",
cfg["test_prompts"][0],
]
recipe = build_recipe_from_benchmark_config(cfg, measured_prompts=[cfg["test_prompts"][0]])
changed_recipe = build_recipe_from_benchmark_config(changed, measured_prompts=[cfg["test_prompts"][0]])
assert recipe_fingerprint(recipe) == recipe_fingerprint(changed_recipe)
def test_num_inference_steps_changes_recipe_fingerprint():
cfg = _benchmark_config()
changed = deepcopy(cfg)
changed["generation_kwargs"]["num_inference_steps"] = 8
assert _fingerprint(cfg) != _fingerprint(changed)
def test_benchmark_version_changes_recipe_fingerprint():
cfg = _benchmark_config()
changed = deepcopy(cfg)
changed["benchmark_version"] = 3
assert _fingerprint(cfg) != _fingerprint(changed)
def test_attention_backend_changes_recipe_fingerprint():
cfg = _benchmark_config()
assert _fingerprint(cfg, attention_backend="FLASH_ATTN") != _fingerprint(
cfg, attention_backend="TORCH_SDPA")
def test_precision_changes_recipe_fingerprint():
cfg = _benchmark_config()
changed = deepcopy(cfg)
changed["init_kwargs"]["text_encoder_precisions"] = ["bf16"]
assert _fingerprint(cfg) != _fingerprint(changed)
def test_dimensions_and_seed_change_recipe_fingerprint():
cfg = _benchmark_config()
different_dimensions = deepcopy(cfg)
different_dimensions["generation_kwargs"]["width"] = 1024
different_seed = deepcopy(cfg)
different_seed["generation_kwargs"]["seed"] = 2048
assert _fingerprint(cfg) != _fingerprint(different_dimensions)
assert _fingerprint(cfg) != _fingerprint(different_seed)
def test_distributed_layout_changes_recipe_fingerprint():
cfg = _benchmark_config()
changed = deepcopy(cfg)
changed["init_kwargs"]["sp_size"] = 1
changed["init_kwargs"]["tp_size"] = 2
assert _fingerprint(cfg) != _fingerprint(changed)
def test_software_profile_uses_exact_attention_kernel_package_versions_for_id():
profile_a = software_profile(
python_version="3.12.4",
torch_version="2.12.0+cu130",
cuda_version="13.0",
package_versions={
"triton": "3.4.1",
"fastvideo_kernel": "0.3.2",
"flash_attn_4": "4.0.0.dev0",
"flashinfer": "0.2.11",
"nvidia_cutlass_dsl": "4.5.0",
},
)
profile_b = software_profile(
python_version="3.12.5",
torch_version="2.12.1+cu130",
cuda_version="13.0",
package_versions={
"triton": "3.4.9",
"fastvideo_kernel": "0.3.7",
"flash_attn_4": "4.0.0.dev1",
"flashinfer": "0.2.12",
"nvidia_cutlass_dsl": "4.5.1",
},
)
assert profile_a == {
"python": "3.12",
"pytorch": "2.12",
"cuda": "13.0",
"packages": {
"fastvideo_kernel": "0.3.2",
"flash_attn_4": "4.0.0.dev0",
"flashinfer": "0.2.11",
"nvidia_cutlass_dsl": "4.5.0",
"triton": "3.4.1",
},
}
assert profile_b["python"] == "3.12"
assert profile_b["pytorch"] == "2.12"
assert profile_b["cuda"] == "13.0"
assert software_profile_id(profile_a) != software_profile_id(profile_b)
def test_installed_package_versions_checks_attention_kernel_distributions(monkeypatch):
versions = {
"fastvideo-kernel": "0.3.2",
"flash-attn": "2.8.1",
"flash-attn-4": "4.0.0.dev0",
"flash-attention-fp4": "0.1.0",
"flashinfer-python": "0.2.11",
"nvidia-cutlass-dsl": "4.5.0",
}
def fake_version(name):
if name not in versions:
raise identity_module.importlib.metadata.PackageNotFoundError(name)
return versions[name]
monkeypatch.setattr(identity_module.importlib.metadata, "version", fake_version)
installed = identity_module._installed_package_versions()
assert installed["fastvideo_kernel"] == "0.3.2"
assert installed["flash_attn"] == "2.8.1"
assert installed["flash_attn_4"] == "4.0.0.dev0"
assert installed["flash_attention_fp4"] == "0.1.0"
assert installed["flashinfer"] == "0.2.11"
assert installed["nvidia_cutlass_dsl"] == "4.5.0"
def test_hardware_profile_id_uses_normalized_gpu_cohort():
profile = hardware_profile(
num_gpus=2,
gpu_devices=[
{
"name": "NVIDIA L40S",
"memory_bytes": 48 * 1024**3,
"compute_capability": "8.9",
},
{
"name": "NVIDIA L40S",
"memory_gb": 48,
"compute_capability": "8.9",
},
],
interconnect="none_or_partial",
)
assert profile == {
"device_type": "cuda",
"gpu_count": 2,
"gpus": [
{
"name": "NVIDIA L40S",
"memory_gb": 48,
"compute_capability": "8.9",
},
{
"name": "NVIDIA L40S",
"memory_gb": 48,
"compute_capability": "8.9",
},
],
"interconnect": "none_or_partial",
}
assert hardware_profile_id(profile).startswith("hw-")
def test_hardware_profile_pads_missing_requested_cuda_devices(monkeypatch):
class Props:
name = "NVIDIA L40S"
total_memory = 48 * 1024**3
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
monkeypatch.setattr(torch.cuda, "get_device_properties", lambda device_id: Props())
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device_id: (8, 9))
profile = hardware_profile(num_gpus=2, interconnect="unknown")
assert profile["gpu_count"] == 2
assert profile["gpus"] == [
{
"name": "NVIDIA L40S",
"memory_gb": 48,
"compute_capability": "8.9",
},
{
"name": "unknown",
"memory_gb": None,
"compute_capability": None,
},
]
def test_local_path_detection_checks_both_path_separators(monkeypatch):
monkeypatch.setattr(identity_module.os, "sep", "\\")
assert identity_module._looks_like_local_path("models/local/checkpoint") is True
assert identity_module._looks_like_local_path(r"C:\models\checkpoint") is True
assert identity_module._looks_like_local_path("Wan-AI/Wan2.1-T2V-1.3B-Diffusers") is False
def test_environment_metadata_is_separate_audit_fingerprint():
cfg = _benchmark_config()
recipe_hash = _fingerprint(cfg)
audit_a = environment_metadata(
env={"IMAGE_VERSION": "py3.12-cuda13.0.0"},
package_versions={"triton": "3.4.1"},
hardware={"device_type": "cuda", "gpu_count": 2},
software={"python": "3.12", "pytorch": "2.12", "cuda": "13.0"},
)
audit_b = environment_metadata(
env={"IMAGE_VERSION": "py3.12-cuda13.0.1"},
package_versions={"triton": "3.4.1"},
hardware={"device_type": "cuda", "gpu_count": 2},
software={"python": "3.12", "pytorch": "2.12", "cuda": "13.0"},
)
assert recipe_hash == _fingerprint(cfg)
assert environment_fingerprint(audit_a) != environment_fingerprint(audit_b)
def test_resolved_revision_from_hf_snapshot_path():
revision = "a" * 40
assert resolved_revision_from_model_path(
f"/root/.cache/huggingface/hub/models--org--repo/snapshots/{revision}") == revision
assert resolved_revision_from_model_path("/models/local-repo") is None
def test_runtime_identity_from_generator_summarizes_worker_records():
from fastvideo.tests.performance.test_inference_performance import _runtime_identity_from_generator
revision = "b" * 40
class FakeExecutor:
def collective_rpc(self, _method):
return [
{
"resolved_attention_backends": ["FLASH_ATTN"],
"resolved_model_path": f"/root/.cache/huggingface/hub/models--org--repo/snapshots/{revision}",
},
{
"resolved_attention_backends": ["FLASH_ATTN"],
"resolved_model_path": f"/root/.cache/huggingface/hub/models--org--repo/snapshots/{revision}",
},
]
class FakeGenerator:
executor = FakeExecutor()
assert _runtime_identity_from_generator(FakeGenerator()) == {
"resolved_attention_backend": "FLASH_ATTN",
"resolved_model_revision": revision,
}
def test_collect_worker_identity_handles_torch_module_pipeline():
from fastvideo.tests.performance.test_inference_performance import _collect_worker_identity
class Backend:
name = "FLASH_ATTN"
class Leaf(torch.nn.Module):
backend = Backend()
class Pipeline(torch.nn.Module):
model_path = "/models/local"
def __init__(self):
super().__init__()
self.leaf = Leaf()
class Worker:
pipeline = Pipeline()
assert _collect_worker_identity(Worker()) == {
"resolved_attention_backends": ["FLASH_ATTN"],
"resolved_model_path": "/models/local",
}
@@ -12,25 +12,12 @@ import os
import time
from collections.abc import Mapping
from datetime import datetime, timezone
from typing import Any
import torch
import pytest
from fastvideo import VideoGenerator
from fastvideo.logger import init_logger
from fastvideo.tests.performance.identity import (
benchmark_identity_from_config,
build_recipe_from_benchmark_config,
environment_fingerprint,
environment_metadata,
hardware_profile,
hardware_profile_id,
recipe_fingerprint,
resolved_revision_from_model_path,
software_profile,
software_profile_id,
)
from fastvideo.worker.multiproc_executor import MultiprocExecutor
logger = init_logger(__name__)
@@ -47,17 +34,11 @@ V2_REQUIRED_IDENTITY_FIELDS = (
"variant_id",
"benchmark_version",
)
# "recipe" is no longer config-declarable: the fingerprint follow-up landed,
# and the generated recipe document owns that key in emitted records. A config
# declaring it now fails validation loudly instead of being silently
# overwritten by the generated one.
V2_OPTIONAL_METADATA_FIELDS = (
"recipe",
"metric_threshold_policy",
"quality_metadata",
)
RESULT_SCHEMA_VERSION = 2
VALID_RUN_SOURCES = {"pr", "local", "scheduled_main", "unknown"}
OPTIONAL_RESULT_METADATA_FIELDS = ("quality_metadata", "variant_metadata")
# -- Config discovery -------------------------------------------------------
@@ -198,11 +179,7 @@ def _extract_component_times(result: dict) -> dict[str, float | None]:
logger.debug("Skipping malformed stage '%s' data: %r", stage_name, stage_data)
continue
stage_class = stage_data.get("stage_class", stage_name)
component_metric = stage_data.get("component_metric")
if isinstance(component_metric, str) and component_metric in component_times:
metric_key = component_metric
else:
metric_key = STAGE_METRIC_MAP.get(stage_class)
metric_key = STAGE_METRIC_MAP.get(stage_class)
if metric_key is None:
logger.debug("Unmapped stage '%s' class '%s' (%.3fs)",
stage_name,
@@ -234,17 +211,6 @@ def _run_generation(generator, prompt, generation_kwargs):
component_times = _extract_component_times(result)
return elapsed, peak_memory_mb, component_times
def _validate_run_counts(run_config: Mapping[str, Any], benchmark_id: str) -> tuple[int, int]:
num_warmup = run_config.get("num_warmup_runs", 1)
num_measure = run_config.get("num_measurement_runs", 3)
if isinstance(num_warmup, bool) or not isinstance(num_warmup, int) or num_warmup < 0:
raise ValueError(f"{benchmark_id}: run_config.num_warmup_runs must be a non-negative integer")
if isinstance(num_measure, bool) or not isinstance(num_measure, int) or num_measure < 1:
raise ValueError(f"{benchmark_id}: run_config.num_measurement_runs must be a positive integer")
return num_warmup, num_measure
def _write_results(results):
"""Write JSON results to the results directory."""
script_dir = os.path.dirname(os.path.abspath(__file__))
@@ -261,205 +227,6 @@ def _write_results(results):
logger.info("Performance results written to %s", filepath)
def _backend_name(value) -> str:
if hasattr(value, "name"):
return str(value.name)
return str(value)
def _collect_worker_identity(worker) -> dict[str, Any]:
pipeline = getattr(worker, "pipeline", None)
model_path = getattr(pipeline, "model_path", None)
backends: set[str] = set()
modules = getattr(pipeline, "modules", None)
if isinstance(modules, Mapping):
module_iter = modules.values()
elif isinstance(pipeline, torch.nn.Module):
module_iter = (pipeline,)
else:
module_iter = ()
for module in module_iter:
if isinstance(module, torch.nn.Module):
for submodule in module.modules():
backend = getattr(submodule, "backend", None)
if backend is not None:
backends.add(_backend_name(backend))
else:
backend = getattr(module, "backend", None)
if backend is not None:
backends.add(_backend_name(backend))
return {
"resolved_attention_backends": sorted(backends),
"resolved_model_path": model_path,
}
def _single_or_list(values: set[str]) -> str | list[str] | None:
ordered = sorted(values)
if not ordered:
return None
if len(ordered) == 1:
return ordered[0]
return ordered
def _runtime_identity_from_generator(generator) -> dict[str, Any]:
worker_records = generator.executor.collective_rpc(_collect_worker_identity)
resolved_backends: set[str] = set()
resolved_revisions: set[str] = set()
for record in worker_records:
resolved_backends.update(record.get("resolved_attention_backends") or [])
revision = resolved_revision_from_model_path(record.get("resolved_model_path"))
if revision is not None:
resolved_revisions.add(revision)
return {
"resolved_attention_backend": _single_or_list(resolved_backends),
"resolved_model_revision": _single_or_list(resolved_revisions),
}
def _benchmark_identity_fields(cfg):
identity = benchmark_identity_from_config(cfg)
return {
key: identity[key]
for key in ("workload_id", "variant_id", "benchmark_version")
}
def _build_identity_fields(cfg, init_kwargs, prompt, runtime_identity):
# Legacy v1 configs (no config_schema_version) stay on the legacy
# (model_id, gpu_type) cohort: they lack the required identity fields,
# and building a recipe for them would raise AFTER the GPU measurement
# completed. The validator already enforces that v2 configs carry the
# identity fields, so v2 records always get the full identity block.
if cfg.get("config_schema_version") is None:
return {}
recipe = build_recipe_from_benchmark_config(
cfg,
resolved_attention_backend=runtime_identity.get("resolved_attention_backend"),
resolved_model_revision=runtime_identity.get("resolved_model_revision"),
measured_prompts=[prompt],
)
num_gpus = init_kwargs.get("num_gpus", cfg.get("run_config", {}).get("required_gpus", 1))
hw_profile = hardware_profile(num_gpus=num_gpus)
sw_profile = software_profile()
env_metadata = environment_metadata(
hardware=hw_profile,
software=sw_profile,
)
return {
"recipe": recipe,
"recipe_fingerprint": recipe_fingerprint(recipe),
"hardware_profile": hw_profile,
"hardware_profile_id": hardware_profile_id(hw_profile),
"software_profile": sw_profile,
"software_profile_id": software_profile_id(sw_profile),
"environment_metadata": env_metadata,
"environment_fingerprint": environment_fingerprint(env_metadata),
**_benchmark_identity_fields(cfg),
}
def _truthy_pr_number(value: str | None) -> bool:
return bool(value and value not in {"false", "0", "None", "none"})
def _detect_run_source() -> str:
explicit = os.environ.get("PERF_RUN_SOURCE", "").strip().lower()
if explicit in VALID_RUN_SOURCES:
return explicit
if _truthy_pr_number(os.environ.get("BUILDKITE_PULL_REQUEST")):
return "pr"
if os.environ.get("BUILDKITE_BRANCH") == "main" and os.environ.get("TEST_SCOPE") == "full":
return "scheduled_main"
if not os.environ.get("BUILDKITE_COMMIT"):
return "local"
return "unknown"
def _ci_provenance_fields() -> dict[str, str]:
pr_number = os.environ.get("BUILDKITE_PULL_REQUEST", "")
if not _truthy_pr_number(pr_number):
pr_number = ""
return {
"run_source": _detect_run_source(),
"branch": os.environ.get("BUILDKITE_BRANCH", ""),
"pr_number": pr_number,
"test_scope": os.environ.get("TEST_SCOPE", ""),
"build_url": os.environ.get("BUILDKITE_BUILD_URL", ""),
"build_id": os.environ.get("BUILDKITE_BUILD_ID", ""),
"job_id": os.environ.get("BUILDKITE_JOB_ID", ""),
}
def _configured_result_metadata(cfg: Mapping[str, Any]) -> dict[str, Any]:
return {
field: cfg[field]
for field in OPTIONAL_RESULT_METADATA_FIELDS
if field in cfg and cfg[field] is not None
}
def _build_result_record(
*,
cfg: Mapping[str, Any],
model_info: Mapping[str, Any],
init_kwargs: Mapping[str, Any],
gen_kwargs: Mapping[str, Any],
num_warmup: int,
num_measure: int,
thresholds: Mapping[str, Any],
times: list[float],
peak_memories: list[float],
all_component_times: list[dict],
prompt: str,
runtime_identity: Mapping[str, Any],
device_name: str,
timestamp: str | None = None,
) -> dict[str, Any]:
if not times or not peak_memories:
raise ValueError("Cannot build a performance result record without measurement runs")
avg_time = sum(times) / len(times)
max_peak_memory = max(peak_memories)
num_frames = gen_kwargs.get("num_frames")
throughput_fps = (1.0 / avg_time) if avg_time > 0 else None
if isinstance(num_frames, (int, float)) and avg_time > 0:
throughput_fps = num_frames / avg_time
return {
"benchmark_id": cfg["benchmark_id"],
"result_schema_version": RESULT_SCHEMA_VERSION,
"model_short_name": model_info.get("model_short_name", ""),
"device": device_name,
"num_gpus": init_kwargs.get("num_gpus", 1),
"num_warmup_runs": num_warmup,
"num_measurement_runs": num_measure,
"avg_generation_time_s": round(avg_time, 3),
"individual_times_s": [round(t, 3) for t in times],
"throughput_fps": round(throughput_fps, 3)
if throughput_fps is not None else None,
"max_peak_memory_mb": round(max_peak_memory, 1),
"individual_peak_memories_mb": [round(m, 1) for m in peak_memories],
"thresholds": dict(thresholds),
"regression_thresholds": cfg.get("regression_thresholds", {}),
"commit": os.environ.get("BUILDKITE_COMMIT", ""),
**_ci_provenance_fields(),
"timestamp": timestamp or datetime.now(timezone.utc).isoformat(),
**_configured_result_metadata(cfg),
"text_encoder_time_s": _avg_component(all_component_times,
"text_encoder_time_s"),
"dit_time_s": _avg_component(all_component_times, "dit_time_s"),
"vae_decode_time_s": _avg_component(all_component_times,
"vae_decode_time_s"),
**_build_identity_fields(cfg, init_kwargs, prompt, runtime_identity),
}
# -- Test -------------------------------------------------------------------
def _run_benchmark(cfg):
@@ -475,7 +242,8 @@ def _run_benchmark(cfg):
prompts = cfg.get("test_prompts", ["A cinematic video."])
prompt = prompts[0]
num_warmup, num_measure = _validate_run_counts(run_config, cfg["benchmark_id"])
num_warmup = run_config.get("num_warmup_runs", 1)
num_measure = run_config.get("num_measurement_runs", 3)
thresholds = _get_thresholds(cfg)
# Remap JSON keys to VideoGenerator kwargs
@@ -496,7 +264,6 @@ def _run_benchmark(cfg):
model_path=model_info["model_path"],
**init_kwargs,
)
runtime_identity = _runtime_identity_from_generator(generator)
for i in range(num_warmup):
logger.info("Warmup run %d/%d", i + 1, num_warmup)
@@ -522,21 +289,35 @@ def _run_benchmark(cfg):
avg_time = sum(times) / len(times)
max_peak_memory = max(peak_memories)
device_name = torch.cuda.get_device_name()
results = _build_result_record(
cfg=cfg,
model_info=model_info,
init_kwargs=init_kwargs,
gen_kwargs=gen_kwargs,
num_warmup=num_warmup,
num_measure=num_measure,
thresholds=thresholds,
times=times,
peak_memories=peak_memories,
all_component_times=all_component_times,
prompt=prompt,
runtime_identity=runtime_identity,
device_name=device_name,
)
num_frames = gen_kwargs.get("num_frames")
throughput_fps = (1.0 / avg_time) if avg_time > 0 else None
if isinstance(num_frames, (int, float)) and avg_time > 0:
throughput_fps = num_frames / avg_time
results = {
"benchmark_id": cfg["benchmark_id"],
**_config_identity_metadata(cfg),
"model_short_name": model_info.get("model_short_name", ""),
"device": device_name,
"num_gpus": init_kwargs.get("num_gpus", 1),
"num_warmup_runs": num_warmup,
"num_measurement_runs": num_measure,
"avg_generation_time_s": round(avg_time, 3),
"individual_times_s": [round(t, 3) for t in times],
"throughput_fps": round(throughput_fps, 3)
if throughput_fps is not None else None,
"max_peak_memory_mb": round(max_peak_memory, 1),
"individual_peak_memories_mb": [round(m, 1) for m in peak_memories],
"thresholds": thresholds,
"commit": os.environ.get("BUILDKITE_COMMIT", ""),
"pr_number": os.environ.get("BUILDKITE_PULL_REQUEST", ""),
"timestamp": datetime.now(timezone.utc).isoformat(),
"text_encoder_time_s": _avg_component(all_component_times,
"text_encoder_time_s"),
"dit_time_s": _avg_component(all_component_times, "dit_time_s"),
"vae_decode_time_s": _avg_component(all_component_times,
"vae_decode_time_s"),
}
logger.info(
"Performance results: avg_time=%.2fs, "
@@ -1,20 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
from fastvideo.pipelines.pipeline_batch_info import PipelineLoggingInfo
from fastvideo.pipelines.stages.denoising import Cosmos25AutoDenoisingStage, DenoisingStage
from fastvideo.pipelines.stages.text_encoding import Cosmos25TextEncodingStage
from fastvideo.tests.performance.test_inference_performance import _extract_component_times
class SubclassStyleDenoisingStage(DenoisingStage):
pass
def test_extract_component_times_handles_pipeline_logging_info_object():
logging_info = PipelineLoggingInfo()
logging_info.add_stage_execution_time("prompt_encoding_stage", 1.25)
logging_info.add_stage_metric("prompt_encoding_stage", "stage_class", "TextEncodingStage")
logging_info.add_stage_metric("prompt_encoding_stage", "component_metric", "text_encoder_time_s")
assert _extract_component_times({"logging_info": logging_info}) == {
"text_encoder_time_s": 1.25,
@@ -52,35 +45,6 @@ def test_extract_component_times_uses_stage_class_for_pipeline_stage_keys():
}
def test_denoising_stage_subclasses_inherit_component_metric():
assert SubclassStyleDenoisingStage.performance_component_metric == "dit_time_s"
def test_cosmos25_direct_pipeline_stages_define_component_metrics():
assert Cosmos25TextEncodingStage.performance_component_metric == "text_encoder_time_s"
assert Cosmos25AutoDenoisingStage.performance_component_metric == "dit_time_s"
def test_extract_component_times_uses_component_metric_for_stage_subclasses():
result = {
"logging_info": {
"stages": {
"denoising_stage": {
"execution_time": 4.2,
"stage_class": "CosmosDenoisingStage",
"component_metric": "dit_time_s",
},
},
},
}
assert _extract_component_times(result) == {
"text_encoder_time_s": None,
"dit_time_s": 4.2,
"vae_decode_time_s": None,
}
def test_extract_component_times_keeps_legacy_class_name_keys():
# Backward-compatibility check for logs produced before pipeline-unique
# stage keys carried a separate stage_class field.
@@ -1,176 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
import pytest
from fastvideo.tests.performance import test_inference_performance as perf_test
def _benchmark_config():
return {
"benchmark_id": "wan-t2v-1.3b-2gpu",
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"quality_metadata": {
"quality_status": "canonical",
},
"model": {
"model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
"model_short_name": "Wan2.1-T2V-1.3B",
},
"generation_kwargs": {
"num_frames": 45,
},
}
def _identity_fields():
return {
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"recipe": {
"recipe_schema_version": 1,
"benchmark": {
"benchmark_id": "wan-t2v-1.3b-2gpu",
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
},
},
"recipe_fingerprint": "recipe-1",
"hardware_profile": {
"device_type": "cuda",
"gpu_count": 2,
},
"hardware_profile_id": "hw-l40s",
"software_profile": {
"python": "3.12",
"pytorch": "2.12",
"cuda": "13.0",
},
"software_profile_id": "sw-cu130",
"environment_metadata": {
"env": {
"IMAGE_VERSION": "py3.12-cuda13.0.0",
},
},
"environment_fingerprint": "env-ci",
}
def test_build_result_record_emits_v2_wan_shape(monkeypatch):
monkeypatch.setenv("PERF_RUN_SOURCE", "scheduled_main")
monkeypatch.setenv("BUILDKITE_COMMIT", "a" * 40)
monkeypatch.setenv("BUILDKITE_PULL_REQUEST", "false")
monkeypatch.setenv("BUILDKITE_BRANCH", "main")
monkeypatch.setenv("TEST_SCOPE", "full")
monkeypatch.setenv("BUILDKITE_BUILD_URL", "https://buildkite.example/build")
monkeypatch.setenv("BUILDKITE_BUILD_ID", "build-1")
monkeypatch.setenv("BUILDKITE_JOB_ID", "job-1")
monkeypatch.setattr(perf_test, "_build_identity_fields", lambda *_args: _identity_fields())
record = perf_test._build_result_record(
cfg=_benchmark_config(),
model_info={"model_short_name": "Wan2.1-T2V-1.3B"},
init_kwargs={"num_gpus": 2},
gen_kwargs={"num_frames": 45},
num_warmup=2,
num_measure=3,
thresholds={
"max_generation_time_s": 34.0,
"max_peak_memory_mb": 11000.0,
},
times=[30.0, 31.0, 32.0],
peak_memories=[10000.0, 10100.0, 10050.0],
all_component_times=[
{
"text_encoder_time_s": 1.0,
"dit_time_s": 8.0,
"vae_decode_time_s": 3.0,
},
{
"text_encoder_time_s": 1.2,
"dit_time_s": 8.2,
"vae_decode_time_s": 3.2,
},
{
"text_encoder_time_s": None,
"dit_time_s": 8.4,
"vae_decode_time_s": 3.4,
},
],
prompt="A cinematic video.",
runtime_identity={
"resolved_attention_backend": "FLASH_ATTN",
},
device_name="NVIDIA L40S",
timestamp="2026-07-05T00:00:00+00:00",
)
assert record["result_schema_version"] == perf_test.RESULT_SCHEMA_VERSION
assert record["benchmark_id"] == "wan-t2v-1.3b-2gpu"
assert record["workload_id"] == "wan-t2v"
assert record["variant_id"] == "1.3b-sp2"
assert record["benchmark_version"] == 2
assert record["recipe_fingerprint"] == "recipe-1"
assert record["hardware_profile_id"] == "hw-l40s"
assert record["software_profile_id"] == "sw-cu130"
assert record["environment_fingerprint"] == "env-ci"
assert record["environment_metadata"]["env"]["IMAGE_VERSION"] == "py3.12-cuda13.0.0"
assert record["quality_metadata"] == {"quality_status": "canonical"}
assert record["run_source"] == "scheduled_main"
assert record["branch"] == "main"
assert record["test_scope"] == "full"
assert record["build_url"] == "https://buildkite.example/build"
assert record["build_id"] == "build-1"
assert record["job_id"] == "job-1"
assert record["pr_number"] == ""
assert record["commit"] == "a" * 40
assert record["avg_generation_time_s"] == 31.0
assert record["throughput_fps"] == 1.452
assert record["max_peak_memory_mb"] == 10100.0
assert record["text_encoder_time_s"] == 1.1
assert record["dit_time_s"] == 8.2
assert record["vae_decode_time_s"] == 3.2
def test_validate_run_counts_rejects_zero_measurement_runs():
with pytest.raises(ValueError, match="num_measurement_runs"):
perf_test._validate_run_counts({
"num_warmup_runs": 1,
"num_measurement_runs": 0,
}, "wan-t2v-1.3b-2gpu")
def test_validate_run_counts_rejects_negative_warmup_runs():
with pytest.raises(ValueError, match="num_warmup_runs"):
perf_test._validate_run_counts({
"num_warmup_runs": -1,
"num_measurement_runs": 3,
}, "wan-t2v-1.3b-2gpu")
def test_validate_run_counts_returns_defaults():
assert perf_test._validate_run_counts({}, "wan-t2v-1.3b-2gpu") == (1, 3)
def test_build_result_record_rejects_empty_measurements(monkeypatch):
monkeypatch.setattr(perf_test, "_build_identity_fields", lambda *_args: _identity_fields())
with pytest.raises(ValueError, match="measurement runs"):
perf_test._build_result_record(
cfg=_benchmark_config(),
model_info={},
init_kwargs={},
gen_kwargs={},
num_warmup=0,
num_measure=0,
thresholds={},
times=[],
peak_memories=[],
all_component_times=[],
prompt="A cinematic video.",
runtime_identity={},
device_name="NVIDIA L40S",
)
@@ -20,13 +20,22 @@ REQUIRED_GPUS = 1
device_reference_folder = resolve_inference_device_reference_folder(logger)
_LOCAL_CONVERTED_MODEL = Path("converted_weights/dreamx_world")
_MODEL_PATH = os.getenv(
"DREAMX_WORLD_SSIM_MODEL_PATH",
"FastVideo/DreamX-World-5B-Cam-Diffusers",
str(_LOCAL_CONVERTED_MODEL),
)
_LOCAL_AR_CANDIDATES = (
Path("/tmp/converted_dreamx_world_ar"),
Path("/root/data/dreamx_world_ar_converted"),
)
_DEFAULT_AR_MODEL_PATH = next(
(str(path) for path in _LOCAL_AR_CANDIDATES if path.exists()),
str(_LOCAL_AR_CANDIDATES[0]),
)
_AR_MODEL_PATH = os.getenv(
"DREAMX_WORLD_AR_SSIM_MODEL_PATH",
"FastVideo/DreamX-World-5B-Diffusers",
_DEFAULT_AR_MODEL_PATH,
)
DREAMX_WORLD_PARAMS = {
@@ -128,6 +137,13 @@ def test_dreamx_world_inference_similarity(
model_id: str,
tmp_path: Path,
) -> None:
model_path = Path(str(DREAMX_WORLD_MODEL_TO_PARAMS[model_id]["model_path"]))
if not model_path.exists():
pytest.skip(
f"DreamX-World converted model path is missing: {model_path}. "
"Set DREAMX_WORLD_SSIM_MODEL_PATH to a FastVideo-loadable converted root."
)
image_path = tmp_path / "dreamx_world_ssim_input.png"
_write_deterministic_reference_image(image_path)
@@ -166,6 +182,13 @@ def test_dreamx_world_ar_inference_similarity(
attention_backend_name: str,
model_id: str,
) -> None:
model_path = Path(str(DREAMX_WORLD_AR_MODEL_TO_PARAMS[model_id]["model_path"]))
if not model_path.exists():
pytest.skip(
f"DreamX-World AR converted model path is missing: {model_path}. "
"Set DREAMX_WORLD_AR_SSIM_MODEL_PATH to a FastVideo-loadable converted root."
)
run_text_to_video_similarity_test(
logger=logger,
script_dir=os.path.dirname(os.path.abspath(__file__)),
@@ -1,109 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""SSIM regression test for Kandinsky-5.0 Lite text-to-video.
Runs a reduced-size generation (256x384, 21 frames) with a fixed seed and
compares against a device-specific reference video via MS-SSIM. The preset
default is 512x768 x 121 frames; the reduced size keeps CI cheap while the
patch (1, 2, 2) and VAE temporal-4 / spatial-8 constraints stay satisfied
(dims divisible by 16, num_frames % 4 == 1).
"""
import os
import pytest
from fastvideo.logger import init_logger
from fastvideo.pipelines.basic.kandinsky5.presets import KANDINSKY5_T2V_LITE_5S
from fastvideo.tests.ssim.inference_similarity_utils import (
resolve_inference_device_reference_folder,
run_text_to_video_similarity_test,
)
logger = init_logger(__name__)
REQUIRED_GPUS = 1
device_reference_folder = resolve_inference_device_reference_folder(logger)
_PRESET_DEFAULTS = KANDINSKY5_T2V_LITE_5S.defaults
KANDINSKY5_T2V_PARAMS = {
"num_gpus": 1,
"model_path": "kandinskylab/Kandinsky-5.0-T2V-Lite-sft-5s-Diffusers",
"height": 256,
"width": 384,
"num_frames": 21,
"num_inference_steps": _PRESET_DEFAULTS["num_inference_steps"],
"guidance_scale": _PRESET_DEFAULTS["guidance_scale"],
"seed": 1024,
"sp_size": 1,
"tp_size": 1,
"fps": _PRESET_DEFAULTS["fps"],
"neg_prompt": _PRESET_DEFAULTS["negative_prompt"],
}
KANDINSKY5_T2V_FULL_QUALITY_PARAMS = {
**KANDINSKY5_T2V_PARAMS,
"height": _PRESET_DEFAULTS["height"],
"width": _PRESET_DEFAULTS["width"],
# num_frames stays at the inherited reduced 21 even at full quality
# (preset default: 121).
}
KANDINSKY5_T2V_MODEL_TO_PARAMS = {
"Kandinsky-5.0-T2V-Lite-sft-5s-Diffusers": KANDINSKY5_T2V_PARAMS,
}
FULL_QUALITY_KANDINSKY5_T2V_MODEL_TO_PARAMS = {
"Kandinsky-5.0-T2V-Lite-sft-5s-Diffusers": KANDINSKY5_T2V_FULL_QUALITY_PARAMS,
}
KANDINSKY5_T2V_TEST_PROMPTS = [
"A curious raccoon peers through a vibrant field of yellow sunflowers, its "
"eyes wide with interest. The playful yet serene atmosphere is complemented "
"by soft natural light filtering through the petals. Mid-shot, warm and "
"cheerful tones.",
]
@pytest.mark.parametrize("prompt", KANDINSKY5_T2V_TEST_PROMPTS)
@pytest.mark.parametrize("attention_backend_name", ["FLASH_ATTN"])
@pytest.mark.parametrize("model_id", list(KANDINSKY5_T2V_MODEL_TO_PARAMS.keys()))
def test_kandinsky5_t2v_inference_similarity(
prompt: str,
attention_backend_name: str,
model_id: str,
) -> None:
# The SSIM lane image bakes FASTVIDEO_FA4=1, but on L40S (sm89) the CLIP
# encoder's dense MHA inference call routes into the FA4 CuTeDSL JIT,
# which fails to compile there (nvvm.fmax signature mismatch vs the
# image's nvidia_cutlass_dsl). Pin FA4 off so reference seeding and CI
# runs both use the FA2 path with identical numerics. Scoped to this test
# and restored afterwards: the other SSIM references are seeded with FA4
# on, so a module-level pin would corrupt every test collected in the
# same pytest process.
saved_fa4 = os.environ.get("FASTVIDEO_FA4")
os.environ["FASTVIDEO_FA4"] = "0"
try:
run_text_to_video_similarity_test(
logger=logger,
script_dir=os.path.dirname(os.path.abspath(__file__)),
device_reference_folder=device_reference_folder,
prompt=prompt,
attention_backend_name=attention_backend_name,
model_id=model_id,
default_params_map=KANDINSKY5_T2V_MODEL_TO_PARAMS,
full_quality_params_map=FULL_QUALITY_KANDINSKY5_T2V_MODEL_TO_PARAMS,
min_acceptable_ssim=0.93,
# Match examples/inference/basic/basic_kandinsky5_t2v.py: FSDP
# inference is not used for Kandinsky-5, and the Qwen2.5-VL text
# encoder stays on CPU between encodes.
init_kwargs_override={
"use_fsdp_inference": False,
"text_encoder_cpu_offload": True,
"pin_cpu_memory": True,
},
)
finally:
if saved_fa4 is None:
os.environ.pop("FASTVIDEO_FA4", None)
else:
os.environ["FASTVIDEO_FA4"] = saved_fa4
@@ -1,36 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Unit tests for LTX-2 refine upsampler path resolution.
Pure tmp_path tests: no GPU, no model downloads.
"""
from fastvideo.pipelines.basic.ltx2.ltx2_pipeline import _resolve_refine_upsampler_path
def test_model_index_key_wins_over_directory_probe(tmp_path):
(tmp_path / "custom_upsampler").mkdir()
(tmp_path / "spatial_upscaler").mkdir()
model_index = {"fastvideo_refine_upsampler_path": "custom_upsampler"}
assert _resolve_refine_upsampler_path(str(tmp_path), model_index) == str(tmp_path / "custom_upsampler")
def test_model_index_spatial_upsampler_module_entry(tmp_path):
(tmp_path / "spatial_upsampler").mkdir()
model_index = {"spatial_upsampler": ["diffusers", "LTX2SpatialUpsampler"]}
assert _resolve_refine_upsampler_path(str(tmp_path), model_index) == str(tmp_path / "spatial_upsampler")
def test_directory_fallback_finds_spatial_upscaler(tmp_path):
# The published LTX-2 distilled checkpoints ship the weights on disk as
# "spatial_upscaler" without any model_index.json declaration.
(tmp_path / "spatial_upscaler").mkdir()
assert _resolve_refine_upsampler_path(str(tmp_path), {}) == str(tmp_path / "spatial_upscaler")
def test_directory_fallback_finds_spatial_upsampler(tmp_path):
(tmp_path / "spatial_upsampler").mkdir()
assert _resolve_refine_upsampler_path(str(tmp_path), {}) == str(tmp_path / "spatial_upsampler")
def test_returns_none_when_nothing_resolves(tmp_path):
assert _resolve_refine_upsampler_path(str(tmp_path), {}) is None
@@ -21,26 +21,6 @@ class FakeTokenizer:
"attention_mask": torch.ones(B, seq_len, dtype=torch.long),
})
class FakeChatTokenizer:
def __init__(self):
self.last_messages = None
self.last_kwargs = None
def apply_chat_template(self, messages, **kwargs):
self.last_messages = messages
self.last_kwargs = kwargs
assert isinstance(messages[0], list)
assert messages[0][0]["role"] == "system"
assert messages[0][1]["role"] == "user"
B = len(messages)
seq_len = int(kwargs.get("max_length", 4))
return TensorDict({
"input_ids": torch.arange(B * seq_len).view(B, seq_len),
"attention_mask": torch.ones(B, seq_len, dtype=torch.long),
})
class FakeTextEncoder(torch.nn.Module):
def __init__(self, hidden_size=8):
super().__init__()
@@ -58,14 +38,6 @@ class FakeTextEncoder(torch.nn.Module):
def id_preprocess(x: str) -> str:
return x
def chat_list_preprocess(x: str):
return [
{"role": "system", "content": "Describe the video."},
{"role": "user", "content": x if x else " "},
]
def take_mean_postprocess(outputs: BaseEncoderOutput) -> torch.Tensor:
# [B, T, H] -> [B, H]
return outputs.last_hidden_state.mean(dim=1)
@@ -184,32 +156,3 @@ def test_encode_text_does_not_force_hidden_states_for_ltx2_prefix():
stage.encode_text("a", fastvideo_args, encoder_index=[0])
assert stage.text_encoders[0].last_output_hidden_states is False
def test_chat_list_preprocess_output_is_not_stripped():
fastvideo_args, hidden = make_args(num_encoders=1, text_len=5, hidden_size=8)
encoder_config = fastvideo_args.pipeline_config.text_encoder_configs[0]
encoder_config.is_chat_model = True
encoder_config.treat_empty_as_dot = True
fastvideo_args.pipeline_config.preprocess_text_funcs = (chat_list_preprocess, )
tokenizer = FakeChatTokenizer()
stage = TextEncodingStage(
text_encoders=[FakeTextEncoder(hidden_size=hidden)],
tokenizers=[tokenizer],
)
embeds, masks = stage.encode_text(
"a robotic arm welding a metal structure",
fastvideo_args,
encoder_index=[0],
return_attention_mask=True,
)
assert embeds[0].shape == (1, hidden)
assert masks[0].shape == (1, 5)
assert tokenizer.last_messages == [[
{"role": "system", "content": "Describe the video."},
{"role": "user", "content": "a robotic arm welding a metal structure"},
]]
assert tokenizer.last_kwargs["return_tensors"] == "pt"
@@ -1,56 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# Minimum config to run a single training step of
# CausalConsistencyDistillationMethod on WanCausalModel for the
# per-method smoke test. Uses the real Wan 2.1 1.3B checkpoint for both
# the trainable student and the frozen teacher (AR Euler-step target),
# with tiny synthetic latents.
models:
student:
_target_: fastvideo.train.models.wan.WanCausalModel
init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers
trainable: true
teacher:
_target_: fastvideo.train.models.wan.WanCausalModel
init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers
trainable: false
ema:
_target_: fastvideo.train.models.wan.WanCausalModel
init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers
trainable: false
method:
_target_: fastvideo.train.methods.consistency_model.causal_cd.CausalConsistencyDistillationMethod
discrete_cd_N: 12
guidance_scale: 3.0
ema_decay: 0.95
training:
dit_precision: bf16
distributed:
num_gpus: 1
sp_size: 1
tp_size: 1
data:
seed: 42
train_batch_size: 1
training_cfg_rate: 0.0
num_latent_t: 6
num_height: 64
num_width: 64
num_frames: 21
optimizer:
learning_rate: 2.0e-6
betas: [0.9, 0.999]
weight_decay: 0.01
lr_scheduler: constant
lr_warmup_steps: 0
loop:
max_train_steps: 1
gradient_accumulation_steps: 1
pipeline: {}

Some files were not shown because too many files have changed in this diff Show More