Compare commits

..
Author SHA1 Message Date
Jinzhe Pan 3a7925c674 [ci] Add /test pre-commit slash command
- /test pre-commit triggers pre-commit via workflow_call
- Posts commit status to PR head SHA so check appears on PR page
- Mergify check-success~=pre-commit condition can now be satisfied
2026-03-31 08:18:06 -04:00
510 changed files with 2767 additions and 57508 deletions
+1 -1
View File
@@ -119,7 +119,7 @@ FastVideo-WorldModel/
## Build & Test Commands
```bash
uv pip install -e ".[dev]" # Editable install
uv pip install -e .[dev] # Editable install
pre-commit run --all-files # Lint/format/spell
pytest tests/ # Top-level tests
pytest fastvideo/tests/ -v # Package tests
-96
View File
@@ -1,96 +0,0 @@
#!/usr/bin/env bash
# Sync .agents/skills/ into .claude/skills/ via per-skill symlinks.
#
# Why: Claude Code only scans .claude/skills/ and ~/.claude/skills/ for
# user-invocable skills (no skillsPath config exists — see
# https://code.claude.com/docs/en/skills.md). This repo's skills live
# in .agents/skills/ so they travel with the repo and stay under git.
# Run this once after cloning (or after adding/removing a skill) to
# expose them to Claude Code without maintaining a parallel tree.
#
# Usage:
# .agents/scripts/sync-skills.sh
#
# Idempotent and safe to re-run. Prunes stale symlinks whose source
# has been removed from .agents/skills/. Leaves hand-written
# .claude/skills/<name>/ directories untouched (only symlinks are
# managed).
set -euo pipefail
REPO_ROOT="$(git -C "$(dirname "$0")" rev-parse --show-toplevel)"
SRC_DIR="$REPO_ROOT/.agents/skills"
DST_DIR="$REPO_ROOT/.claude/skills"
if [[ ! -d "$SRC_DIR" ]]; then
echo "Error: $SRC_DIR does not exist." >&2
exit 1
fi
mkdir -p "$DST_DIR"
linked=0
unchanged=0
skipped=0
pruned=0
link_skill() {
local name="$1"
local src="$SRC_DIR/$name"
local dst="$DST_DIR/$name"
# Relative target keeps symlinks portable across clones.
local rel="../../.agents/skills/$name"
if [[ -L "$dst" ]]; then
if [[ "$(readlink "$dst")" == "$rel" ]]; then
unchanged=$((unchanged + 1))
return
fi
rm "$dst"
elif [[ -e "$dst" ]]; then
echo "Skipped (not a symlink): .claude/skills/$name" >&2
skipped=$((skipped + 1))
return
fi
ln -s "$rel" "$dst"
echo "Linked: .claude/skills/$name -> $rel"
linked=$((linked + 1))
}
prune_stale() {
local link="$1"
local target
target="$(readlink "$link")"
case "$target" in
../../.agents/skills/*) ;;
*) return ;;
esac
local name="${target##*/}"
if [[ ! -d "$SRC_DIR/$name" ]]; then
rm "$link"
echo "Pruned stale: .claude/skills/$(basename "$link")"
pruned=$((pruned + 1))
fi
}
for src in "$SRC_DIR"/*/; do
[[ -d "$src" ]] || continue
name="$(basename "$src")"
# Only treat directories that actually contain a SKILL.md as skills.
[[ -f "$src/SKILL.md" ]] || continue
link_skill "$name"
done
shopt -s nullglob
for link in "$DST_DIR"/*; do
[[ -L "$link" ]] || continue
prune_stale "$link"
done
shopt -u nullglob
printf "\nSummary: %d linked, %d unchanged, %d pruned" "$linked" "$unchanged" "$pruned"
if [[ "$skipped" -gt 0 ]]; then
printf ", %d skipped (non-symlink collision)" "$skipped"
fi
printf "\n"
-3
View File
@@ -5,6 +5,3 @@
{"name": "evaluate-video-quality", "description": "Evaluate generated video quality using available metrics (SSIM, loss trajectory, caption consistency)", "path": "evaluate-video-quality/SKILL.md", "status": "draft", "trust": "low"}
{"name": "index-related-work", "description": "Ingest a paper or repository into the related work index", "path": "index-related-work/SKILL.md", "status": "draft", "trust": "low"}
{"name": "search-related-work", "description": "Query the related work index for relevant papers, repos, or comparisons", "path": "search-related-work/SKILL.md", "status": "draft", "trust": "low"}
{"name": "seed-ssim-references", "description": "Run a new or updated fastvideo/tests/ssim/ test on Modal, pull generated videos, and upload them to FastVideo/ssim-reference-videos so the test has a regression baseline", "path": "seed-ssim-references/SKILL.md", "status": "draft", "trust": "low"}
{"name": "reseed-ssim-references", "description": "Re-seed (overwrite) HF reference videos for an existing fastvideo/tests/ssim/ test and a single model id on Modal L40S. Always backs up current refs first, regenerates on Modal, pauses for the user to eyeball before-vs-after, then uploads with --force scoped to --model-id. Sister skill to seed-ssim-references; use when intentional code change has invalidated existing refs", "path": "reseed-ssim-references/SKILL.md", "status": "draft", "trust": "low"}
{"name": "add-model", "description": "Add a new model (or variant) to FastVideo: DiT + configs + pipeline + presets + registry + tests. Walks through FastVideo's single stage-based pipeline architecture with exact file paths and registration hooks.", "path": "add-model/SKILL.md", "status": "draft", "trust": "low"}
+1 -1
View File
@@ -12,7 +12,7 @@ automates the boilerplate of setting environment variables, picking the right
entrypoint, and applying defaults from the closest example script.
## Prerequisites
- The repo is cloned and `fastvideo` is installed (`uv pip install -e ".[dev]"`).
- The repo is cloned and `fastvideo` is installed (`uv pip install -e .[dev]`).
- Dataset is preprocessed (see `docs/training/data_preprocess.md`).
- `WANDB_API_KEY` is set in the environment (or `WANDB_MODE=offline` for local).
- GPU resources are available (multi-GPU requires NCCL).
@@ -1,343 +0,0 @@
---
name: reseed-ssim-references
description: Re-seed HF reference videos for a single existing SSIM test on Modal L40S. Always backs up current refs locally first, regenerates on Modal, pauses for the user to eyeball before-vs-after quality, then overwrites the targeted `<model_id>` subtree on `FastVideo/ssim-reference-videos` with `--force`. Use when an intentional code change (model port fix, attention backend swap, kernel upgrade, hyperparameter change) has invalidated existing refs and they need to be regenerated. Pairs with `seed-ssim-references`, which is for first-time seeding only.
---
# Re-seed SSIM Reference Videos
## Purpose
Replace the existing SSIM reference videos for a single `(test_file, model_id)`
pair on the HF dataset (`FastVideo/ssim-reference-videos`). This is **destructive**
on HF — the old refs are overwritten — so the skill always:
1. Confirms intent with a one-liner the user has to type.
2. Downloads the existing refs as a local, timestamped backup.
3. Regenerates on Modal L40S (same code path that CI uses).
4. Pauses for a side-by-side eyeball of backup vs new mp4s.
5. Uploads with `--force`, scoped to the single `--model-id`.
6. Reminds the user to keep the backup until the PR lands.
Pairs with `seed-ssim-references`, which is the inverse (first-time seeding
only, refuses to overwrite). Re-seeding is intentionally a separate, more
ceremonial operation because mistakenly clobbering production refs is much
harder to recover from than failing closed.
## When to use
- An intentional code change (model port fix, kernel upgrade, attention
backend swap, hyperparameter change in the test itself) has shifted the
expected SSIM output and the existing refs no longer represent the new
ground truth.
- A test is failing in CI **for the right reason** (the new code is correct,
the old refs are stale).
## When not to use
- A test is failing for the **wrong** reason (the port is buggy, not the
refs). Fix the port; re-seeding hides the bug.
- A brand-new test that has no refs on HF yet. Use `seed-ssim-references`.
- "Just to clean up drift" without a concrete code change to point at. The
PR description has to justify *why* refs changed; without a concrete
change, there's nothing to write.
## Inputs
| Parameter | Required | Description |
|-----------|----------|-------------|
| `test_file` | Yes | Path to the SSIM test, e.g. `fastvideo/tests/ssim/test_matrixgame_similarity.py`. Validated against `fastvideo/tests/ssim/test_*_similarity.py`. |
| `model_id` | Yes | Single model id from the test's `*_MODEL_TO_PARAMS`, e.g. `Matrix-Game-2.0-Diffusers-Base`. Re-seed runs are **per model**. For multi-model tests, invoke the skill once per model. |
| `intent_rationale` | Yes | One-line explanation of *why* refs are being regenerated (e.g. "Relax FA-2 head_size whitelist to include 80 — matrix_game now uses FLASH_ATTN instead of TORCH_SDPA"). Recorded in the backup directory and reused in the PR description. |
Hardcoded:
- Modal GPU: **L40S** (matches CI; re-seeding from another SKU produces refs
that L40S CI cannot match).
- Quality tier: **`default`**. `full_quality` is a separate, deliberate
operation.
- HF repo: `FastVideo/ssim-reference-videos` (override via
`FASTVIDEO_SSIM_REFERENCE_HF_REPO`).
- Device folder: `L40S_reference_videos`.
## Prerequisites
The user has confirmed:
- `modal` CLI authenticated.
- `hf` CLI authenticated, **and** `HF_API_KEY` (or `HUGGINGFACE_HUB_TOKEN` /
`HF_TOKEN`) exported with **write** access to
`FastVideo/ssim-reference-videos`.
- The current branch's code is the change that motivated the re-seed (i.e.
`git rev-parse HEAD` is the commit that intentionally invalidated refs).
Fail fast if any of these are missing.
## Steps
### 1. Validate inputs and confirm intent
- Verify `test_file` exists and matches `fastvideo/tests/ssim/test_*_similarity.py`.
- Grep the file for `*_MODEL_TO_PARAMS` and assert `model_id` is one of its
keys. If the file has only a single hardcoded model, accept that model id
as the only valid value.
- Print the rationale and ask the user to type **`confirm reseed`** (not just
`y` — make it deliberate):
> About to RE-SEED references for model `<model_id>` from test `<test_file>`.
> This will OVERWRITE existing refs on
> `FastVideo/ssim-reference-videos/reference_videos/default/L40S_reference_videos/<model_id>/`
> after backup + Modal regen + eyeball.
>
> Reason: `<intent_rationale>`
> HEAD: `<git rev-parse --short=12 HEAD>`
>
> Reply `confirm reseed` to proceed, anything else to abort.
Stop until the user types exactly `confirm reseed`. Anything else aborts
with no side effects.
### 2. Back up existing refs
Always required. The backup is the only graceful path back if anything goes
wrong later.
```bash
SHORT_COMMIT=$(git rev-parse --short=12 HEAD)
TIMESTAMP=$(date -u +%Y%m%d_%H%M%S)
MODEL_SAFE=$(echo "<model_id>" | tr '/' '_')
BACKUP_DIR="ssim_reseed_backup/${TIMESTAMP}_${SHORT_COMMIT}_${MODEL_SAFE}"
mkdir -p "$BACKUP_DIR"
hf download \
--repo-type dataset FastVideo/ssim-reference-videos \
--include "reference_videos/default/L40S_reference_videos/<model_id>/**" \
--local-dir "$BACKUP_DIR"
mp4_count=$(find "$BACKUP_DIR" -name "*.mp4" | wc -l)
echo "Backup mp4 count: $mp4_count"
[ "$mp4_count" -gt 0 ] || {
echo "ERROR: backup is empty for <model_id>. Either the model id is wrong"
echo "or there are no existing refs (use seed-ssim-references instead)."
exit 1
}
# Provenance — used in the PR description
cat > "$BACKUP_DIR/PROVENANCE.txt" <<EOF
test_file: <test_file>
model_id: <model_id>
head_commit: $(git rev-parse HEAD)
timestamp_utc: $(date -u +%FT%TZ)
reason: <intent_rationale>
EOF
```
If the `hf download` produces zero mp4s, abort — the user has either picked a
non-existent `model_id` or there are no refs yet (in which case
`seed-ssim-references` is the right tool).
### 3. Regenerate on Modal L40S
Mirror CI's exact env recipe so the regenerated refs are byte-comparable to
what CI will produce on the same commit. Two differences from CI:
1. **Pass the same env prefix CI uses** (`IMAGE_VERSION`, `BUILDKITE_*`) — see
`.buildkite/pipeline.yml:1-3` and `.buildkite/scripts/pr_test.sh:62-83`.
Without this, `ssim_test.py:17-18` resolves a different GHCR image tag
(default is `latest`, CI is `py3.12-latest`), and `ssim_test.py:38-46`
bakes different values into the image's frozen env block. **Mismatched
image or env is the most common source of SSIM drift between reseed and
CI runs.**
2. **Do not pass `--skip-reference-download`**. Letting the test fetch the
existing refs and run the full SSIM compare gives "before" SSIM numbers
for the PR description, and the test still produces the new mp4s
regardless of whether the comparison passes or fails.
```bash
SUBDIR="${TIMESTAMP}_${SHORT_COMMIT}"
IMAGE_VERSION="py3.12-latest" \
BUILDKITE_REPO="$(git config --get remote.origin.url)" \
BUILDKITE_COMMIT="$(git rev-parse HEAD)" \
BUILDKITE_PULL_REQUEST="${BUILDKITE_PULL_REQUEST:-false}" \
modal run fastvideo/tests/modal/ssim_test.py \
--git-repo="$(git config --get remote.origin.url)" \
--git-commit="$(git rev-parse HEAD)" \
--hf-api-key="$HF_API_KEY" \
--test-files="<test_file>" \
--sync-generated-to-volume \
--generated-volume-subdir="$SUBDIR" \
--no-fail-fast
```
Capture the printed `modal volume get ...` hint — its `<SUBDIR>` matches
`$SUBDIR` and is needed for step 4. Capture the SSIM numbers from the test
output (or from the JSON next to the generated mp4) for the PR description.
### 4. Download generated videos
```bash
modal volume get --force hf-model-weights \
ssim_generated_videos/default/"$SUBDIR"/generated_videos \
./generated_videos_modal/default
```
After this, the new mp4s live at:
```
./generated_videos_modal/default/generated_videos/L40S_reference_videos/<model_id>/<backend>/<prompt>.mp4
```
`--force` is required when `./generated_videos_modal/default` already exists
from a prior run; safe on the first run too.
### 5. PAUSE — user reviews quality side-by-side
Print the diff and the comparison:
```bash
echo "=== File list diff (backup vs new) ==="
diff -u \
<(find "$BACKUP_DIR/reference_videos/default/L40S_reference_videos/<model_id>" -name "*.mp4" \
| sed "s|$BACKUP_DIR/reference_videos/default/L40S_reference_videos/||" | sort) \
<(find ./generated_videos_modal/default/generated_videos/L40S_reference_videos/<model_id> -name "*.mp4" \
| sed "s|./generated_videos_modal/default/generated_videos/L40S_reference_videos/||" | sort) \
|| true
echo
echo "=== SSIM numbers from this run (paste into PR) ==="
find ./generated_videos_modal/default/generated_videos/L40S_reference_videos/<model_id> -name "*_ssim.json" -exec cat {} \;
```
Then stop and tell the user:
> Old refs backed up to `$BACKUP_DIR`.
> New videos in `./generated_videos_modal/default/generated_videos/L40S_reference_videos/<model_id>/`.
>
> Open both in a video player. Confirm the new videos:
> 1. Look correct (no obvious artifacts, no black/static frames).
> 2. Are *intentionally* different from the backup in the way described
> in `<intent_rationale>` (e.g. slight numerical drift only, not a
> different scene / different motion / corrupted output).
>
> Reply **`upload`** to overwrite HF, anything else to abort.
> Aborting leaves the backup and new videos on disk for inspection — nothing
> on HF changes.
Do not proceed until the user types exactly `upload`. If they abort, leave
everything on disk and stop here.
### 6. Copy into the local reference layout
Same as `seed-ssim-references` step 5:
```bash
python fastvideo/tests/ssim/reference_videos_cli.py copy-local \
--quality-tier default \
--device-folder L40S_reference_videos \
--generated-dir ./generated_videos_modal/default/generated_videos/L40S_reference_videos
```
Result: `fastvideo/tests/ssim/reference_videos/default/L40S_reference_videos/<model_id>/<backend>/<prompt>.mp4`.
### 7. Upload with `--force`, scoped to `--model-id`
The `--force` flag is what makes this skill different from `seed-ssim-references`.
Always pair it with `--model-id` so a typo cannot accidentally overwrite a
neighboring model's refs.
```bash
python fastvideo/tests/ssim/reference_videos_cli.py upload \
--quality-tier default \
--device-folder L40S_reference_videos \
--model-id "<model_id>" \
--force
```
The CLI's overwrite guard refuses without `--force`; with `--force` it
overwrites only files under
`reference_videos/default/L40S_reference_videos/<model_id>/`.
### 8. Report success and retention guidance
Print:
- The HF path that was overwritten (`<repo>/reference_videos/default/L40S_reference_videos/<model_id>/`).
- The local backup directory path.
- The new SSIM numbers from step 5.
- This restore command, in case the PR review surfaces a problem after
upload:
```bash
python fastvideo/tests/ssim/reference_videos_cli.py upload \
--quality-tier default \
--device-folder L40S_reference_videos \
--model-id "<model_id>" \
--reference-dir "$BACKUP_DIR/reference_videos/default/L40S_reference_videos" \
--force
```
- This PR-description checklist (see `fastvideo/tests/ssim/AGENTS.md` →
*Updating Reference Videos*):
1. Source commit that produced the new refs (HEAD at re-seed time).
2. Test command and GPU SKU (`L40S`).
3. Before/after SSIM numbers.
4. The `<intent_rationale>` from step 1.
5. A note that the backup lives at `$BACKUP_DIR` and should be retained
until CI on the PR is green.
Do **not** auto-rerun the SSIM test — the user does that as part of the PR.
## Failure modes and how to handle them
- **`HF_API_KEY` unset.** Stop before step 2.
- **Backup is empty (zero mp4s).** Stop before step 3 — the model id is
wrong or the refs don't exist yet (use `seed-ssim-references`).
- **Modal run fails before generation.** No mp4s on the volume. Don't
upload. Investigate the failure (test crash, OOM, partition exhaustion),
fix, then retry from step 3. Backup is still intact.
- **Quality regressed (visual or metric).** User aborts at step 5. Backup
retained. New videos retained on disk for inspection. Nothing on HF
changed. Either fix the underlying code change or abandon the re-seed.
- **User confirmed `upload` but later realized the new refs are wrong.**
Run the restore command from step 8 with the backup `--reference-dir`.
This is exactly why the backup exists.
- **Multi-model test, only one model is being re-seeded.** Run the skill
once per model id. The `--model-id` scope on upload guarantees the others
are untouched.
## Design notes (for future skill maintainers)
- Per-`model_id` scope is mandatory. The dataset houses many model subtrees;
re-seeding the wrong one is hard to undo without backup.
- `default` tier only; `full_quality` is a separate, deliberate operation
with different params and ~doubled runtime, and isn't what CI gates on.
- The skill deliberately does **not** pass `--skip-reference-download` to
Modal so we get pre-reseed SSIM numbers for the PR. The `seed`-skill
passes it because no refs exist yet; for re-seed, refs do exist and
exposing the comparison is informative.
- The two-token confirm (`confirm reseed`, then `upload`) is intentional.
Re-seeding is high-blast-radius and should not be one-keystroke.
- The backup directory is plain mp4s + `PROVENANCE.txt`. No HF metadata is
preserved; the restore path uses `reference_videos_cli.py upload
--reference-dir` which doesn't need it.
## References
- `.agents/skills/seed-ssim-references/SKILL.md` — the first-time seed
skill this one parallels. Read it for the Modal flag rationale shared
between the two flows.
- `fastvideo/tests/ssim/AGENTS.md` — directory rules, including the PR
expectations for any reference-video change (rationale, before/after
SSIM, source commit/model/backend).
- `fastvideo/tests/ssim/reference_videos_cli.py` — `copy-local`, `upload`
(with `--model-id`, `--force`), `download`. The overwrite guard at
`upload_reference_videos` is the safety net this skill leans on.
- `fastvideo/tests/modal/ssim_test.py` — Modal orchestrator;
`--sync-generated-to-volume`, `--generated-volume-subdir`,
`--skip-reference-download`, `--no-fail-fast`.
## Changelog
| Date | Change |
|------|--------|
| 2026-05-02 | Initial version. Sister skill to `seed-ssim-references`, scoped to single `(test_file, model_id)` re-seeds, with mandatory backup and two-token confirm. |
@@ -1,376 +0,0 @@
---
name: seed-ssim-references
description: Seed HF reference artefacts for a single newly-added SSIM test (pixel `.mp4` for `run_text_to_video_similarity_test`-style tests, or latent `.pt` for `run_text_to_latent_similarity_test`-style tests). Runs the test on Modal L40S, downloads the generated artefacts via `modal volume get`, pauses for the user to verify (visual eyeball for mp4, numerics dump for pt), then uploads only that test's files to `FastVideo/ssim-reference-videos`. Use when a new `fastvideo/tests/ssim/test_*_similarity.py` has just been added and has no references on HF yet.
---
# Seed SSIM Reference Artefacts (mp4 or pt)
## Purpose
A brand-new SSIM test in `fastvideo/tests/ssim/` fails forever until its
reference artefacts exist on the HF dataset
(`FastVideo/ssim-reference-videos`). The dataset hosts two kinds of artefacts
side-by-side per `(model_id, backend, prompt)`:
- **`.mp4`** — pixel ground-truth for tests that call
`run_text_to_video_similarity_test` / `run_image_to_video_similarity_test`
in `inference_similarity_utils.py`. Compared via SSIM.
- **`.pt`** — pre-VAE latent bundle (fp16 full latent + fp32 slice +
metadata + `slice_spec` + `format_version`) for tests that call
`run_text_to_latent_similarity_test` in `latent_similarity_utils.py`.
Compared via cosine distance on the slice and the full tensor.
This skill:
1. Detects which artefact type the test produces (pixel vs latent).
2. Runs the test on Modal's L40S pool to generate the artefacts.
3. Downloads them to the local repo via `modal volume get`.
4. Pauses so the user can verify quality:
- **mp4**: visual eyeball in a video player.
- **pt**: numerics dump (shape, slice stats, NaN/Inf check, metadata).
5. Uploads only the new test's files to HF, with a guard that refuses to
overwrite anything already present.
The skill is run **manually**, once per new test. Before invoking it, the user
has already sanity-tested the new test locally — it launches `VideoGenerator`
and writes an artefact without crashing (the missing-reference assertion at
the end is expected). The skill does not re-test locally; it goes straight
to Modal L40S (which is what CI uses).
## When to use
- A new `test_*_similarity.py` file has been added in `fastvideo/tests/ssim/`
and the HF dataset has no `reference_videos/default/L40S_reference_videos/<model_id>/`
subtree for it yet.
## When not to use
- Regular CI runs — once refs exist, `pytest fastvideo/tests/ssim/` downloads
them automatically.
- Re-seeding an existing test. That requires `--force` on the upload step, and
is out of scope here; treat as a separate, deliberate operation.
## Inputs
The skill has **one required input**: the path to the new SSIM test file.
Prompt the user for it if they didn't supply it.
| Parameter | Required | Description |
|-----------|----------|-------------|
| `test_file` | Yes | e.g. `fastvideo/tests/ssim/test_ltx2_similarity.py`. The skill's first action is to ask for this if missing. |
Everything else is fixed:
- Modal runner GPU: **L40S** (hardcoded in `fastvideo/tests/modal/ssim_test.py`).
- Device folder: `L40S_reference_videos`.
- Quality tier: `default` (the tier CI runs). The `full_quality` tier is not
seeded by this skill.
- HF repo: `FastVideo/ssim-reference-videos` (dataset).
- Multi-model test files: all model ids in `*_MODEL_TO_PARAMS` are seeded
together; the Modal run produces one mp4 per (model, prompt, backend) and
the upload scopes by `--model-id`, looping if there is more than one.
## Prerequisites
The user has confirmed:
- `modal` CLI authenticated.
- `HF_API_KEY` (or `HUGGINGFACE_HUB_TOKEN` / `HF_TOKEN`) exported with write
access to `FastVideo/ssim-reference-videos`.
- The test file runs locally end-to-end (generates an mp4; SSIM assertion
failure due to missing reference is expected and fine).
Fail fast if the token env var is missing.
## Steps
### 1. Ask for the test file, then detect artefact type
If the user didn't name one, ask: *"Which SSIM test file do you want to seed
references for? (e.g. `fastvideo/tests/ssim/test_ltx2_similarity.py`)"*.
Validate:
- Path exists and matches `fastvideo/tests/ssim/test_*_similarity.py`.
- File defines a `*_MODEL_TO_PARAMS` dict — grep it to extract the set of
model ids. Those ids drive step 5.
Detect artefact type by inspecting the file's imports / helper call:
- **latent** (`.pt`) — file imports `run_text_to_latent_similarity_test`
from `fastvideo.tests.ssim.latent_similarity_utils` (or any other helper
that ends with `_latent_similarity_test`).
- **pixel** (`.mp4`) — file imports
`run_text_to_video_similarity_test` / `run_image_to_video_similarity_test`
from `fastvideo.tests.ssim.inference_similarity_utils`, OR uses the
legacy custom-inline helper pattern (see `test_gamecraft`,
`test_longcat`, etc.). Default to pixel when both heuristics fail.
Record `ARTEFACT_TYPE ∈ {pixel, latent}` for use in step 4. Steps 2, 3, 5,
and 6 are artefact-type-agnostic — `_iter_reference_files`,
`copy_generated_to_reference`, and `upload_reference_videos` already walk
both `.mp4` and `.pt` (see `reference_videos_cli.py`).
If either check fails, stop and tell the user what's wrong.
### 2. Run the test on Modal L40S
Pick a subdir name so repeated runs don't collide:
```bash
SHORT_COMMIT=$(git rev-parse --short=12 HEAD)
TIMESTAMP=$(date -u +%Y%m%d_%H%M%S)
SUBDIR="${TIMESTAMP}_${SHORT_COMMIT}"
```
Then launch the Modal run. The `IMAGE_VERSION` and `BUILDKITE_*` env-prefix
**must** match what CI exports in `.buildkite/scripts/pr_test.sh`, otherwise
`fastvideo/tests/modal/ssim_test.py` resolves a different GHCR image tag
(default is `latest`, CI is `py3.12-latest`) and bakes different values into
the image's frozen env block (`ssim_test.py:17-18, 38-46`). Mismatched image
or env produces SSIM drift that doesn't show up until the same commit runs
in CI.
```bash
IMAGE_VERSION="py3.12-latest" \
BUILDKITE_REPO="$(git config --get remote.origin.url)" \
BUILDKITE_COMMIT="$(git rev-parse HEAD)" \
BUILDKITE_PULL_REQUEST="${BUILDKITE_PULL_REQUEST:-false}" \
modal run fastvideo/tests/modal/ssim_test.py \
--git-repo="$(git config --get remote.origin.url)" \
--git-commit="$(git rev-parse HEAD)" \
--hf-api-key="$HF_API_KEY" \
--test-files="<test_file>" \
--sync-generated-to-volume \
--generated-volume-subdir="$SUBDIR" \
--skip-reference-download \
--no-fail-fast
```
Env prefix rationale (parity with CI; see `.buildkite/pipeline.yml:1-3` and
`.buildkite/scripts/pr_test.sh:62-83`):
- `IMAGE_VERSION=py3.12-latest`: pins the Modal image tag to the same one CI
uses. Without this, `ssim_test.py:17` falls back to `latest`, which on
GHCR is built from `Dockerfile.python3.10` — different Python, torch, and
flash-attn wheel than CI's `py3.12-latest` (`infra-build-image.yml:51-67`,
`_template-build-image.yml:65-101`).
- `BUILDKITE_REPO`/`BUILDKITE_COMMIT`/`BUILDKITE_PULL_REQUEST`: mirror what
Buildkite exports. `ssim_test.py:38-46` bakes these into the image's
`.env(...)` block; mismatched values can perturb in-container code paths
that branch on PR-vs-non-PR. `false` for `BUILDKITE_PULL_REQUEST` matches
Buildkite's "non-PR build" sentinel.
Flag rationale:
- `--skip-reference-download`: no refs exist yet, so conftest must not try to
pull them.
- `--no-fail-fast`: lets the test finish generation before `_assert_similarity`
raises `FileNotFoundError: Reference video folder does not exist`. The
expected failure is what we want — the mp4 has already been written.
- `--sync-generated-to-volume` + `--generated-volume-subdir`: copies the
generated mp4s to the `hf-model-weights` Modal volume under
`ssim_generated_videos/default/<SUBDIR>/generated_videos/` so we can pull
them locally.
The Modal run will end with a nonzero exit (expected) and print a
`modal volume get hf-model-weights ssim_generated_videos/default/<SUBDIR>/generated_videos ./generated_videos_modal/default`
command. Capture that `<SUBDIR>` — you need it for step 3.
### 3. Download generated videos locally
```bash
modal volume get --force hf-model-weights \
ssim_generated_videos/default/"$SUBDIR"/generated_videos \
./generated_videos_modal/default
```
`--force` is required when the parent `./generated_videos_modal/default`
already exists; without it, `modal volume get` errors with `[Errno 21] Is a
directory`. Safe to pass on the first run too.
After this, the mp4s live at
`./generated_videos_modal/default/generated_videos/L40S_reference_videos/<model_id>/<backend>/<prompt>.mp4`.
The extra `generated_videos/` level comes from the volume layout in
`_sync_generated_videos_to_volume` (`ssim_test.py`) — the command copies
`<repo>/fastvideo/tests/ssim/generated_videos/<tier>` to
`ssim_generated_videos/<tier>/<SUBDIR>/generated_videos/`, and `modal volume
get` preserves that trailing `generated_videos/` segment.
### 4. PAUSE — user reviews quality
Type-aware verification.
**For `ARTEFACT_TYPE = pixel`** — list the downloaded mp4s and ask the user to
open them in a video player:
> "Generated videos downloaded to `./generated_videos_modal/default/generated_videos/L40S_reference_videos/`. Please open them and confirm the quality looks correct. Reply **`upload`** to continue, or anything else to abort."
**For `ARTEFACT_TYPE = latent`** — `.pt` files are not human-watchable. Print
a numerics dump for each `.pt` so the user can sanity-check shape, distribution,
and metadata:
```python
import torch
from pathlib import Path
ROOT = Path("./generated_videos_modal/default/generated_videos/L40S_reference_videos")
for p in sorted(ROOT.rglob("*.pt")):
d = torch.load(p, map_location="cpu", weights_only=False)
s = d["expected_slice"]
L = d["latent"].float()
print(f"=== {p.relative_to(ROOT)} ===")
print(f" format_version: {d['format_version']}")
print(f" shape: {d['shape']}")
print(f" dtype_original: {d['dtype_original']}")
print(f" slice_spec: {d['slice_spec']}")
print(f" slice shape={tuple(s.shape)} mean={s.mean():+.4f} std={s.std():.4f} min={s.min():+.4f} max={s.max():+.4f}")
print(f" latent shape={tuple(L.shape)} mean={L.mean():+.4f} std={L.std():.4f} min={L.min():+.4f} max={L.max():+.4f}")
print(f" finite: latent NaN={torch.isnan(L).any().item()} Inf={torch.isinf(L).any().item()}; "
f"slice NaN={torch.isnan(s).any().item()} Inf={torch.isinf(s).any().item()}")
print(f" metadata: {d['metadata']}\n")
```
Sanity criteria:
- `format_version == 1` (matches `LATENT_REFERENCE_FORMAT_VERSION`).
- `shape` matches what the model produces (e.g. LTX-2 distilled =
`[1, 128, T_lat, H_lat, W_lat]`; Stable Audio Open 1.0 = `[1, 64, 1024]`).
- `slice_spec.kind` matches a registered kind (`corner_3x3_first_frame`
for video, `audio_first_8_timesteps` for audio).
- No `NaN`/`Inf`. `mean ≈ 0`, `std ≈ 1` (denoised latents stay close to
the initial Gaussian distribution; very wide deviations suggest
numerical drift).
- `metadata.prompt` matches the test's prompt.
Then ask:
> "Numerics look right? Reply **`upload`** to continue, or anything else to abort."
Do not proceed until the user explicitly says `upload`. If they abort, leave
everything on disk so they can inspect further — no cleanup.
### 5. Copy into the local reference layout
Scoped copy — only the new test's artefacts. Single command works for both
artefact types because `_iter_reference_files` walks `.mp4` and `.pt`:
```bash
python fastvideo/tests/ssim/reference_videos_cli.py copy-local \
--quality-tier default \
--device-folder L40S_reference_videos \
--generated-dir ./generated_videos_modal/default/generated_videos/L40S_reference_videos
```
(The `--generated-dir` points at the device-folder root inside the
downloaded tree; `copy-local` walks all `<model>/<backend>/*.{mp4,pt}`
underneath it. Since the Modal run was scoped to a single test file via
`--test-files`, only that test's model(s) are present — so the copy is
implicitly per-test.)
Result for pixel: `fastvideo/tests/ssim/reference_videos/default/L40S_reference_videos/<model_id>/<backend>/<prompt>.mp4`.
Result for latent: same path with `.pt` extension.
### 6. Upload to HF — scoped per model_id, with overwrite guard
For each `<model_id>`:
```bash
python fastvideo/tests/ssim/reference_videos_cli.py upload \
--quality-tier default \
--device-folder L40S_reference_videos \
--model-id "<model_id>"
```
The upload command:
- Uploads **only** `reference_videos/default/L40S_reference_videos/<model_id>/`.
- **Refuses** if any file already exists at that path on HF (this is the
guard — seeding a new test should never clobber existing refs). To override,
the user must re-run with `--force`. If the guard fires, stop and report
exactly which files exist; do not silently `--force`.
Reads the HF token from `HF_API_KEY` / `HUGGINGFACE_HUB_TOKEN` / `HF_TOKEN`.
### 7. Report success
List what was uploaded (paths in repo) and remind the user to push any
related code changes. Do **not** auto-verify by re-running Modal — the user
can run `pytest fastvideo/tests/ssim/<test_file>` later to confirm end-to-end;
it will auto-download the refs they just uploaded.
## Failure modes and how to handle them
- **`HF_API_KEY` unset.** Stop before step 2. The Modal run needs it (passed
via `--hf-api-key`), and step 6 needs it for upload. If the user
ran `hf auth login` instead of exporting an env var, read the cached
token via `huggingface_hub.get_token()` and forward it to Modal as
`--hf-api-key="$CACHED_TOKEN"`.
- **Modal run fails before generation.** No artefacts on the volume — nothing
to download. Fix the test locally (`pytest fastvideo/tests/ssim/<test_file>`)
and retry from step 2.
- **`./generated_videos_modal/default/L40S_reference_videos/` missing after
`modal volume get`.** The run didn't produce artefacts (most likely the
test crashed before writing, or `REQUIRED_GPUS` exceeded the partition
capacity — see Modal logs).
- **Latent test crashed with FSDP / inference_mode error
(`RuntimeError: Inference tensors do not track version counter`).** The
test must pass `init_kwargs_override={"use_fsdp_inference": False}` when
`sp_size == 1` — see `test_stable_audio_similarity.py` for the pattern.
Fix in the test, push, retry.
- **Upload guard fires (files already exist).** The test name / model id
collides with something already on HF. Verify the user actually wants to
replace existing refs; if so, re-run the upload with `--force`. If not,
rename the model id in `*_MODEL_TO_PARAMS` and re-seed.
- **Quality looks wrong in step 4.** Abort. The artefacts stay on disk for
inspection. The fix is usually in the test's params (resolution, steps,
seed) — edit the test, then re-run the skill.
- For latent: also check `slice_spec.kind` matches the latent rank
(`corner_3x3_first_frame` requires 5-D, `audio_first_8_timesteps`
requires 3-D); a rank/kind mismatch raises in `_extract_expected_slice`.
## Design notes (for future skill maintainers)
- The skill deliberately runs on Modal, **not** locally, because the CI
runner is L40S. Seeding from a different GPU SKU produces refs that CI's
L40S runs can't match (pixel SSIM drifts across SKUs; latent cosine has
tighter cross-SKU bf16 drift but the configured tolerances assume
same-SKU seed → same-SKU verify).
- The skill is default-tier only. `full_quality` refs are seeded by a
separate, deliberate operation — they double runtime and aren't what CI
gates on.
- The overwrite guard in `reference_videos_cli.py upload` is default-on
specifically because this skill exists. Re-seeding is a distinct operation
that requires explicit `--force`.
- Both artefact types share the same Modal flow: the orchestrator sets
`--skip-reference-download` + `--no-fail-fast`, runs pytest, the test's
helper writes the artefact (`.mp4` via `imageio` for pixel,
`save_latent_reference` → `torch.save` for latent) BEFORE the
missing-reference assertion raises. `_sync_generated_videos_to_volume` in
`ssim_test.py` does a `shutil.copytree` of the whole `generated_videos/`
tree, picking up `.mp4`, `.pt`, and the `*_ssim.json` / `*_latent.json`
metric files alongside.
## References
- `fastvideo/tests/modal/ssim_test.py` — Modal orchestrator; see
`--sync-generated-to-volume`, `--generated-volume-subdir`,
`--skip-reference-download`, `--no-fail-fast`.
- `fastvideo/tests/ssim/reference_videos_cli.py` — `copy-local`, `upload`
(with `--model-id`, `--force`), `download`, `ensure` subcommands.
Extension allowlist is `REFERENCE_EXTENSIONS = VIDEO_EXTENSIONS +
LATENT_EXTENSIONS` (`.pt`).
- `fastvideo/tests/ssim/README.md` — reference layout, HF repo conventions.
- `fastvideo/tests/ssim/inference_similarity_utils.py` — pixel helpers
(`run_text_to_video_similarity_test`,
`run_image_to_video_similarity_test`, `build_init_kwargs`).
- `fastvideo/tests/ssim/latent_similarity_utils.py` — latent helper
(`run_text_to_latent_similarity_test`), slice spec dispatch
(`_extract_expected_slice`), reference schema
(`save_latent_reference` / `load_latent_reference`),
`LATENT_REFERENCE_FORMAT_VERSION`.
## Changelog
| Date | Change |
|------|--------|
| 2026-04-17 | Initial version (Modal sync-to-volume flow). |
| 2026-04-21 | Rewrite: single-test scope, explicit user-review pause, per-`model_id` upload, HF overwrite guard. Dropped `scripts/seed_ssim.sh`. |
| 2026-04-21 | Post-first-run fixes: `modal volume get` needs `--force` when parent exists; download tree has an extra `generated_videos/` level so `--generated-dir` must reflect it. |
| 2026-05-01 | Latent (`*.pt`) artefact support: artefact-type detection in step 1, type-aware verification (visual eyeball for mp4, numerics dump for pt) in step 4, FSDP+inference_mode failure-mode added, design notes for the unified Modal flow. Triggered by PR #1253 (LTX-2 latent migration + Stable Audio latent test). |
@@ -29,8 +29,8 @@
"Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of a bustling street food market. The scene captures a mix of humor and authenticity. Mid-shot framing, vibrant lighting."
],
"run_config": {
"num_warmup_runs": 2,
"num_measurement_runs": 5,
"num_warmup_runs": 1,
"num_measurement_runs": 3,
"required_gpus": 2
},
"thresholds": {
+2 -186
View File
@@ -9,183 +9,11 @@ notify:
- github_commit_status:
context: "full-suite-passed"
if: build.env("TEST_SCOPE") == "full"
- github_commit_status:
context: "direct-test-completed"
if: build.env("TEST_SCOPE") == "direct"
steps:
# ============================================================
# Direct test: triggered by /test <name> slash command.
# Labels match fastcheck/full-suite counterparts so the GitHub
# check status overwrites the original failed check.
# Only ONE step executes per build (gated by TEST_TYPE).
# ============================================================
# --- Fastcheck-scope direct tests ---
- label: ":microscope: Encoder Tests"
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "encoder"
command: "timeout 90m .buildkite/scripts/pr_test.sh"
retry:
automatic:
- exit_status: 128
limit: 3
- exit_status: -1
limit: 2
agents:
queue: "default"
- label: ":microscope: VAE Tests"
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "vae"
command: "timeout 90m .buildkite/scripts/pr_test.sh"
retry:
automatic:
- exit_status: 128
limit: 3
- exit_status: -1
limit: 2
agents:
queue: "default"
- label: ":microscope: Transformer Tests"
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "transformer"
command: "timeout 90m .buildkite/scripts/pr_test.sh"
retry:
automatic:
- exit_status: 128
limit: 3
- exit_status: -1
limit: 2
agents:
queue: "default"
- label: ":microscope: Kernel Tests"
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "kernel_tests"
command: "timeout 90m .buildkite/scripts/pr_test.sh"
retry:
automatic:
- exit_status: 128
limit: 3
- exit_status: -1
limit: 2
agents:
queue: "default"
- label: ":microscope: Unit Tests"
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "unit_test"
command: "timeout 90m .buildkite/scripts/pr_test.sh"
retry:
automatic:
- exit_status: 128
limit: 3
- exit_status: -1
limit: 2
agents:
queue: "default"
# --- Full-suite-scope direct tests ---
- label: ":bar_chart: SSIM Tests"
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "ssim"
command: "timeout 90m .buildkite/scripts/pr_test.sh"
retry:
automatic:
- exit_status: 128
limit: 3
- exit_status: -1
limit: 2
- exit_status: 1
limit: 2
agents:
queue: "default"
- label: ":test_tube: LoRA Inference Tests"
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "inference_lora"
command: "timeout 90m .buildkite/scripts/pr_test.sh"
retry:
automatic:
- exit_status: 128
limit: 3
- exit_status: -1
limit: 2
agents:
queue: "default"
- label: ":test_tube: Training Tests"
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "training"
command: "timeout 90m .buildkite/scripts/pr_test.sh"
retry:
automatic:
- exit_status: 128
limit: 3
- exit_status: -1
limit: 2
agents:
queue: "default"
- label: ":test_tube: Distillation DMD Tests"
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "distillation_dmd"
command: "timeout 90m .buildkite/scripts/pr_test.sh"
retry:
automatic:
- exit_status: 128
limit: 3
- exit_status: -1
limit: 2
agents:
queue: "default"
- label: ":test_tube: Self-Forcing Tests"
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "self_forcing"
command: "timeout 90m .buildkite/scripts/pr_test.sh"
retry:
automatic:
- exit_status: 128
limit: 3
- exit_status: -1
limit: 2
agents:
queue: "default"
- label: ":test_tube: LoRA Training Tests"
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "training_lora"
command: "timeout 90m .buildkite/scripts/pr_test.sh"
retry:
automatic:
- exit_status: 128
limit: 3
- exit_status: -1
limit: 2
- exit_status: 1
limit: 2
agents:
queue: "default"
- label: ":test_tube: Training Tests VSA"
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "training_vsa"
command: "timeout 90m .buildkite/scripts/pr_test.sh"
retry:
automatic:
- exit_status: 128
limit: 3
- exit_status: -1
limit: 2
- exit_status: 1
limit: 2
agents:
queue: "default"
- label: ":test_tube: Inference Tests VMoBA"
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "inference_vmoba"
command: "timeout 90m .buildkite/scripts/pr_test.sh"
retry:
automatic:
- exit_status: 128
limit: 3
- exit_status: -1
limit: 2
agents:
queue: "default"
- label: ":test_tube: Performance Tests"
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "performance"
command: "timeout 90m .buildkite/scripts/pr_test.sh"
retry:
automatic:
- exit_status: 128
limit: 3
- exit_status: -1
limit: 2
agents:
queue: "default"
- label: ":test_tube: API Server Tests"
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "api_server"
- label: ":dart: Direct Test (${TEST_TYPE})"
if: build.env("TEST_SCOPE") == "direct"
command: "timeout 90m .buildkite/scripts/pr_test.sh"
retry:
automatic:
@@ -307,10 +135,6 @@ steps:
label: ":bar_chart: SSIM Tests"
env:
- TEST_TYPE=ssim
retry:
automatic:
- exit_status: 1
limit: 2
agents:
queue: "default"
- path:
@@ -371,10 +195,6 @@ steps:
label: ":test_tube: LoRA Training Tests"
env:
- TEST_TYPE=training_lora
retry:
automatic:
- exit_status: 1
limit: 2
agents:
queue: "default"
- path:
@@ -387,10 +207,6 @@ steps:
label: ":test_tube: Training Tests VSA"
env:
- TEST_TYPE=training_vsa
retry:
automatic:
- exit_status: 1
limit: 2
agents:
queue: "default"
- path:
+4 -88
View File
@@ -15,21 +15,8 @@ log "Project root: $PROJECT_ROOT"
# Install Modal if not available
if ! python3 -m modal --version &> /dev/null; then
log "Modal not found, installing..."
if ! command -v uv &> /dev/null; then
log "uv not found, bootstrapping..."
if ! curl -LsSf https://astral.sh/uv/install.sh | sh; then
log "Error: Failed to bootstrap uv via astral.sh installer."
exit 1
fi
export PATH="$HOME/.local/bin:$PATH"
if ! command -v uv &> /dev/null; then
log "Error: uv still not on PATH after bootstrap."
exit 1
fi
fi
# --break-system-packages preserves prior `pip install --user` semantics on PEP 668 agents.
uv pip install --system --break-system-packages modal
python3 -m pip install modal
# Verify installation
if ! python3 -m modal --version &> /dev/null; then
log "Error: Failed to install modal. Please install it manually."
@@ -76,72 +63,7 @@ EFFECTIVE_PR=${BUILDKITE_PULL_REQUEST:-false}
if [ "$EFFECTIVE_PR" = "false" ] && [ -n "${PR_NUMBER:-}" ]; then
EFFECTIVE_PR=$PR_NUMBER
fi
MODAL_ENV="BUILDKITE_REPO=$BUILDKITE_REPO BUILDKITE_COMMIT=$BUILDKITE_COMMIT BUILDKITE_PULL_REQUEST=$EFFECTIVE_PR BUILDKITE_BRANCH=${BUILDKITE_BRANCH:-} TEST_SCOPE=${TEST_SCOPE:-} IMAGE_VERSION=$IMAGE_VERSION"
POST_RUN_HOOK=""
upload_performance_artifacts() {
SHORT_SHA=${BUILDKITE_COMMIT:0:7}
LOCAL_DIR="downloaded_reports"
_download_reports() {
log "Downloading perf_reports/ from Modal Volume..."
mkdir -p "$LOCAL_DIR"
if ! modal volume get hf-model-weights "perf_reports/" "$LOCAL_DIR"; then
log "Error: Failed to download perf_reports/ from Modal Volume."
return 1
fi
}
_upload_dashboard() {
local target
target=$(find "$LOCAL_DIR" -name "dashboard_${SHORT_SHA}_*" | head -n 1)
log "TARGET dashboard: '$target'"
if [ -n "$target" ]; then
log "Found dashboard: $target. Uploading to Buildkite..."
buildkite-agent artifact upload "$target"
buildkite-agent annotate --style info --context "perf-dashboard" < "$target"
else
log "Warning: Could not find a dashboard file matching $SHORT_SHA"
fi
}
_upload_perf_summary() {
local target
target=$(find "$LOCAL_DIR" -name "perf_${SHORT_SHA}_*" | head -n 1)
log "TARGET perf summary: '$target'"
if [ -n "$target" ]; then
log "Found perf summary: $target. Uploading to Buildkite..."
buildkite-agent artifact upload "$target"
buildkite-agent annotate --style info --context "perf-summary" < "$target"
else
log "Warning: Could not find a perf summary file matching $SHORT_SHA"
fi
}
_cleanup_modal_volume() {
log "Cleaning up perf_reports/ from Modal Volume..."
if modal volume rm hf-model-weights "perf_reports/" --recursive; then
log "Successfully deleted perf_reports/ from Modal Volume."
else
log "Warning: Failed to delete perf_reports/ from Modal Volume. Manual cleanup may be required."
fi
}
_cleanup_local() {
log "Cleaning up local download directory..."
rm -rf "$LOCAL_DIR"
}
# --- Main flow ---
_download_reports || { _cleanup_local; return 1; }
_upload_dashboard
_upload_perf_summary
_cleanup_modal_volume
_cleanup_local
}
MODAL_ENV="BUILDKITE_REPO=$BUILDKITE_REPO BUILDKITE_COMMIT=$BUILDKITE_COMMIT BUILDKITE_PULL_REQUEST=$EFFECTIVE_PR IMAGE_VERSION=$IMAGE_VERSION"
case "$TEST_TYPE" in
"encoder")
@@ -202,9 +124,8 @@ case "$TEST_TYPE" in
MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_lora_extraction_tests"
;;
"performance")
log "Running performance tests on Modal..."
log "Running performance tests..."
MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_performance_tests"
POST_RUN_HOOK="upload_performance_artifacts"
;;
"api_server")
log "Running API server integration tests..."
@@ -226,10 +147,5 @@ else
log "Error: Modal test failed with exit code: $TEST_EXIT_CODE"
fi
if [ -n "$POST_RUN_HOOK" ]; then
log "Executing post-run hook: $POST_RUN_HOOK"
"$POST_RUN_HOOK"
fi
log "=== Test execution completed with exit code: $TEST_EXIT_CODE ==="
exit $TEST_EXIT_CODE
+2 -15
View File
@@ -13,21 +13,8 @@ log "Project root: $PROJECT_ROOT"
if ! python3 -m pre_commit --version &> /dev/null; then
log "pre-commit not found, installing..."
if ! command -v uv &> /dev/null; then
log "uv not found, bootstrapping..."
if ! curl -LsSf https://astral.sh/uv/install.sh | sh; then
log "Error: Failed to bootstrap uv via astral.sh installer."
exit 1
fi
export PATH="$HOME/.local/bin:$PATH"
if ! command -v uv &> /dev/null; then
log "Error: uv still not on PATH after bootstrap."
exit 1
fi
fi
# --break-system-packages preserves prior `pip install --user` semantics on PEP 668 agents.
uv pip install --system --break-system-packages pre-commit==4.0.1
python3 -m pip install --user pre-commit==4.0.1
if ! python3 -m pre_commit --version &> /dev/null; then
log "Error: Failed to install pre-commit."
exit 1
+12 -7
View File
@@ -4,10 +4,8 @@ merge_protections:
- base = main
success_conditions:
- "title~=(?i)^\\[(feat|feature|bugfix|fix|refactor|perf|ci|doc|docs|misc|chore|kernel|new.?model)\\]"
- "#approved-reviews-by>=1"
- check-success~=pre-commit
- check-success=fastcheck-passed
- check-success=full-suite-passed
pull_request_rules:
@@ -105,7 +103,7 @@ pull_request_rules:
- files~=^fastvideo/pipelines/samplers/
- files~=^fastvideo/entrypoints/
- files~=^fastvideo/worker/
- files~=^fastvideo/api/sampling_param
- files~=^fastvideo/configs/sample/
- files~=^fastvideo/configs/pipelines/
- files~=^examples/inference/
- -closed
@@ -274,15 +272,24 @@ pull_request_rules:
merge:
method: squash
- name: auto-update when ready
- name: auto-rebase when ready and Full Suite passed
conditions:
- label=ready
- "#approved-reviews-by>=1"
- check-success=full-suite-passed
- -conflict
- -closed
- -draft
actions:
update: {}
rebase: {}
- name: remove ready label on Full Suite failure
conditions:
- label=ready
- check-failure=full-suite-passed
actions:
label:
remove: [ready]
# ============================================================
# PR title format help
@@ -312,5 +319,3 @@ pull_request_rules:
Please update your PR title and the merge protection check will pass automatically.
merge_protections_settings:
reporting_method: check-runs
-80
View File
@@ -1,80 +0,0 @@
name: Aggregate Test Status
on:
status:
permissions:
statuses: write
jobs:
aggregate:
if: >-
github.event.context == 'direct-test-completed'
&& github.event.state == 'success'
runs-on: ubuntu-latest
steps:
- name: Check and update aggregate status
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
script: |
const sha = context.payload.sha;
const { data } = await github.rest.repos.getCombinedStatusForRef({
owner: context.repo.owner,
repo: context.repo.repo,
ref: sha,
per_page: 100,
});
const bkStatuses = data.statuses.filter(
s => s.context.startsWith('buildkite/ci/')
);
const FASTCHECK_PREFIX = 'buildkite/ci/microscope-';
const FULL_SUITE_PREFIXES = [
'buildkite/ci/test-tube-',
'buildkite/ci/bar-chart-',
];
const fastcheck = bkStatuses.filter(
s => s.context.startsWith(FASTCHECK_PREFIX)
);
const fullSuite = bkStatuses.filter(
s => FULL_SUITE_PREFIXES.some(p => s.context.startsWith(p))
);
if (
fastcheck.length > 0
&& fastcheck.every(s => s.state === 'success')
) {
core.info(
`All ${fastcheck.length} fastcheck tests passed — updating fastcheck-passed`
);
await github.rest.repos.createCommitStatus({
owner: context.repo.owner,
repo: context.repo.repo,
sha,
state: 'success',
context: 'fastcheck-passed',
description:
`All ${fastcheck.length} fastcheck tests passed`,
});
}
if (
fullSuite.length > 0
&& fullSuite.every(s => s.state === 'success')
) {
core.info(
`All ${fullSuite.length} full suite tests passed — updating full-suite-passed`
);
await github.rest.repos.createCommitStatus({
owner: context.repo.owner,
repo: context.repo.repo,
sha,
state: 'success',
context: 'full-suite-passed',
description:
`All ${fullSuite.length} full suite tests passed`,
});
}
+4 -7
View File
@@ -4,11 +4,10 @@ on:
pull_request:
branches: [main]
workflow_call:
inputs:
ref:
description: 'Git ref to checkout (defaults to github.ref)'
required: false
type: string
concurrency:
group: pre-commit-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
permissions:
contents: read
@@ -19,8 +18,6 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || '' }}
- uses: actions/setup-python@v5
with:
python-version: "3.12"
+14 -55
View File
@@ -7,7 +7,6 @@ on:
permissions:
contents: read
pull-requests: write
statuses: write
jobs:
handle-merge:
@@ -33,7 +32,6 @@ jobs:
core.setOutput('has_write', String(hasWrite));
- name: Add ready label and react
id: label
if: steps.perm.outputs.has_write == 'true'
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
@@ -41,6 +39,7 @@ jobs:
const owner = context.repo.owner;
const repo = context.repo.repo;
const prNumber = context.payload.issue.number;
// Remove ready first to allow re-trigger (labeled event fires on add, not if already present)
try { await github.rest.issues.removeLabel({ owner, repo, issue_number: prNumber, name: 'ready' }); } catch {}
await github.rest.issues.addLabels({ owner, repo, issue_number: prNumber, labels: ['ready'] });
await github.rest.reactions.createForIssueComment({
@@ -48,44 +47,6 @@ jobs:
comment_id: context.payload.comment.id,
content: 'rocket',
});
const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: prNumber });
core.setOutput('pr_sha', pr.head.sha);
core.setOutput('pr_branch', pr.head.ref);
core.setOutput('pr_number', String(prNumber));
- name: Trigger Full Suite
if: steps.perm.outputs.has_write == 'true'
env:
BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
PR_SHA: ${{ steps.label.outputs.pr_sha }}
PR_BRANCH: ${{ steps.label.outputs.pr_branch }}
PR_NUMBER: ${{ steps.label.outputs.pr_number }}
BK_ORG: ${{ vars.BUILDKITE_ORG_SLUG }}
BK_PIPELINE: ${{ vars.BUILDKITE_PIPELINE_SLUG }}
run: |
curl -sS --fail-with-body -X POST \
"https://api.buildkite.com/v2/organizations/${BK_ORG}/pipelines/${BK_PIPELINE}/builds" \
-H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
-H "Content-Type: application/json" \
--data-raw "$(jq -n \
--arg commit "$PR_SHA" \
--arg branch "$PR_BRANCH" \
--arg message "Full Suite for PR #${PR_NUMBER} (via /merge)" \
--argjson pr_id "$PR_NUMBER" \
'{
commit: $commit,
branch: $branch,
message: $message,
ignore_pipeline_branch_filters: true,
pull_request_id: $pr_id,
pull_request_base_branch: "main",
env: {
TEST_SCOPE: "full",
FULL_SUITE: "true",
PR_NUMBER: ($pr_id | tostring)
}
}')"
parse-command:
if: >-
github.event.issue.pull_request != null
@@ -181,33 +142,20 @@ jobs:
core.setOutput('sha', pr.head.sha);
core.setOutput('branch', pr.head.ref);
- name: React to comment
if: steps.perm.outputs.has_write == 'true'
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
script: |
await github.rest.reactions.createForIssueComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: context.payload.comment.id,
content: 'rocket',
});
pre-commit:
needs: parse-command
if: >-
needs.parse-command.outputs.has_write == 'true'
&& needs.parse-command.outputs.test_scope == 'precommit'
uses: ./.github/workflows/ci-precommit.yml
with:
ref: refs/pull/${{ github.event.issue.number }}/merge
post-precommit-status:
needs: [parse-command, pre-commit]
if: always() && needs.parse-command.outputs.test_scope == 'precommit'
runs-on: ubuntu-latest
steps:
- uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
- name: Post commit status
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
env:
PR_SHA: ${{ needs.parse-command.outputs.pr_sha }}
RESULT: ${{ needs.pre-commit.result }}
@@ -230,6 +178,17 @@ jobs:
&& needs.parse-command.outputs.test_type != ''
runs-on: ubuntu-latest
steps:
- name: React to comment
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
script: |
await github.rest.reactions.createForIssueComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: context.payload.comment.id,
content: 'rocket',
});
- name: Trigger Buildkite
env:
BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
+3 -3
View File
@@ -1,7 +1,7 @@
name: Trigger Full Suite
on:
pull_request_target:
pull_request:
types: [labeled, synchronize]
permissions:
@@ -10,7 +10,7 @@ permissions:
concurrency:
group: full-suite-${{ github.event.pull_request.number }}
cancel-in-progress: false
cancel-in-progress: true
jobs:
trigger:
@@ -42,7 +42,7 @@ jobs:
# Find running builds for this branch with TEST_SCOPE=full and cancel them
builds=$(curl -sS -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
"https://api.buildkite.com/v2/organizations/${{ vars.BUILDKITE_ORG_SLUG }}/pipelines/${{ vars.BUILDKITE_PIPELINE_SLUG }}/builds?branch=${PR_BRANCH}&state=running,scheduled" \
| jq -r '.[] | select(try (.env.TEST_SCOPE == "full") catch false) | .number')
| jq -r '.[] | select(.env.TEST_SCOPE == "full") | .number')
for build_num in $builds; do
echo "Cancelling Buildkite build #$build_num"
curl -sS -X PUT -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
+3 -4
View File
@@ -37,11 +37,10 @@ jobs:
with:
python-version: '3.12'
- name: Install uv
uses: astral-sh/setup-uv@v3
- name: Install dependencies
run: uv pip install --system -r requirements-mkdocs.txt
run: |
python -m pip install --upgrade pip
pip install -r requirements-mkdocs.txt
- name: Setup Pages
uses: actions/configure-pages@v4
+3 -4
View File
@@ -56,11 +56,10 @@ jobs:
with:
python-version: '3.10'
- name: Install uv
uses: astral-sh/setup-uv@v3
- name: Install build dependencies
run: uv pip install --system build twine wheel
run: |
python -m pip install --upgrade pip
pip install build twine wheel
- name: Build package
run: |
+11 -16
View File
@@ -131,13 +131,11 @@ jobs:
clang-11 --version
nvcc --version
- name: Install uv
uses: astral-sh/setup-uv@v3
- name: Install PyTorch ${{ matrix.torch-cuda.torch-version }}+cu${{ matrix.torch-cuda.cuda-version }}
run: |
uv pip install --system typing-extensions==4.12.2
uv pip install --system --no-cache-dir torch==${{ matrix.torch-cuda.torch-version }} --index-url https://download.pytorch.org/whl/${{matrix.torch-cuda.torch-cuda-short}}
pip install --upgrade pip
pip install typing-extensions==4.12.2
pip install --no-cache-dir torch==${{ matrix.torch-cuda.torch-version }} --index-url https://download.pytorch.org/whl/${{matrix.torch-cuda.torch-cuda-short}}
nvcc --version
python --version
python -c "import torch; print('PyTorch:', torch.__version__)"
@@ -147,20 +145,20 @@ jobs:
- name: Build wheel
run: |
export PYTHONPATH=$GITHUB_WORKSPACE:$PYTHONPATH
uv pip install --system setuptools ninja packaging wheel triton scikit-build-core cmake build
pip install setuptools ninja packaging wheel triton scikit-build-core cmake build
cd fastvideo-kernel
git submodule update --init --recursive # Ensure ThunderKittens submodule is initialized
# Release builds are produced on GPU-less runners, so force-enable TK and target Hopper.
export TORCH_CUDA_ARCH_LIST="9.0a"
export CMAKE_ARGS="${CMAKE_ARGS:-} -DFASTVIDEO_KERNEL_BUILD_TK=ON -DCMAKE_CUDA_ARCHITECTURES=90a"
# Build standard wheel (no local version suffix) for PyPI
python -m build --wheel --outdir dist
# Fix the wheel to be manylinux compliant
uv pip install --system auditwheel
pip install auditwheel
# Point auditwheel at torch libs, but do not vendor them into the wheel.
TORCH_LIB_DIR=$(python - <<'PY'
import os
@@ -213,13 +211,10 @@ jobs:
pattern: 'fastvideo_kernel-py*'
merge-multiple: true
- name: Install uv
uses: astral-sh/setup-uv@v3
- name: Build source distribution
run: |
uv pip install --system build scikit-build-core cmake ninja
pip install build scikit-build-core cmake ninja
cd fastvideo-kernel
# We don't need full CUDA/Torch to just package the source (sdist)
python -m build --sdist --outdir dist
-9
View File
@@ -85,18 +85,9 @@ docs/distillation/examples/
dmd_t2v_output/
preprocess_output_text/
# Next.js / Node artifacts under ui/: see ui/.gitignore
.claude/
.codex/
.sisyphus/
openspec/
fastvideo/tests/ssim/reference_videos/**
# Local clones of upstream repos used only for parity testing.
/stable-audio-tools/
/daVinci-MagiHuman/
# Converted model weights (produced by scripts/checkpoint_conversion/*).
# Tens of GB; should live on HF, not in git.
/converted_weights/
-1
View File
@@ -1 +0,0 @@
WRN 2026-03-26T13:46:33.469 ?.19646 server_start:193: Failed to start server: operation not permitted: /var/folders/z_/h_6myyk14d1b7z87z3vy4mjh0000gn/T/nvim.dsynkd/iSe0el/nvim.19646.0
+9 -2
View File
@@ -7,13 +7,20 @@ exclude: |
fastvideo-kernel/.*|
assets/.*|
tests/.*|
demo/.*|
predict\.py|
scripts/.*|
assets/prompts/.*|
fastvideo/data_preprocess/.*|
fastvideo/dataset/.*|
fastvideo/models/.*|
fastvideo/sample/.*|
fastvideo/train\.py|
fastvideo/utils/.*|
examples/.*|
\.agents/.*|
.github/workflows/publish-fastvideo.yml|
.github/workflows/_template-build-image.yml
.github/workflows/_template-build-image.yml|
docs/source/inference/support_matrix.md
)
repos:
- repo: https://github.com/google/yapf
-1
View File
@@ -1 +0,0 @@
3.12
+2 -31
View File
@@ -11,7 +11,7 @@
- Static assets: `assets/` (including `assets/images/`, `assets/videos/`, and `assets/prompts/`) and `comfyui/assets/`.
## Build, Test, and Development Commands
- `uv pip install -e ".[dev]"`: editable install with lint/test extras.
- `uv pip install -e .[dev]`: editable install with lint/test extras.
- `pre-commit install --hook-type pre-commit --hook-type commit-msg`: enable local hooks.
- `pre-commit run --all-files`: run formatter/lint/type/spelling checks.
- `pytest tests/`: run top-level test suite.
@@ -23,8 +23,7 @@
- Python 3.10+; 4-space indentation; keep code and imports readable and explicit.
- Style tools are configured in `pyproject.toml` and `.pre-commit-config.yaml`:
- `yapf` (format), `ruff` (lint, auto-fix), `mypy` (typing), `codespell`.
- Lint via `pre-commit run --files <changed paths>` (or `pre-commit run --all-files` for a full sweep) before committing. Do not shell out to `yapf`/`ruff`/`codespell`/`mypy` directly — pre-commit chains them with the project's config and respects the `.pre-commit-config.yaml` excludes (e.g. `fastvideo/tests/` is intentionally skipped). If pre-commit reports `(no files to check)` for your paths, that exclude is deliberate — don't bypass it.
- Target line length is 120 (configured in `pyproject.toml` for ruff, yapf, and isort).
- Target line length is 80.
- Naming: `snake_case` for functions/files, `PascalCase` for classes, `UPPER_SNAKE_CASE` for constants.
## Testing Guidelines
@@ -55,31 +54,3 @@ This repository is agent-friendly. Before doing any work, read:
If you are exploring a new procedure that has no existing SOP, document your
progress in `.agents/exploration/` and flag it for review at the end of your
session.
## Per-Directory AGENTS.md
Local guidance lives next to the code. Read the in-scope file before editing:
| Directory | What it covers |
|-----------|----------------|
| `fastvideo/AGENTS.md` | Core package map, public API, registry-driven model dispatch |
| `fastvideo/configs/AGENTS.md` | Arch + pipeline config dataclasses, `param_names_mapping` |
| `fastvideo/models/AGENTS.md` | DiT / VAE / encoder / scheduler / loader layout (pre-commit excluded) |
| `fastvideo/layers/AGENTS.md` | Tensor-parallel linear/attention layer rules for ports |
| `fastvideo/attention/AGENTS.md` | Backend registry + env-var override |
| `fastvideo/pipelines/AGENTS.md` | Stage ABC, `basic/<model>/`, `preprocess/`, presets |
| `fastvideo/training/AGENTS.md` | Legacy monolithic pipelines (frozen for existing models) |
| `fastvideo/train/AGENTS.md` | New modular trainer (methods × models × callbacks, YAML) |
| `fastvideo/tests/AGENTS.md` | Test taxonomy, conftest, pre-commit-excluded path |
| `fastvideo/tests/ssim/AGENTS.md` | GPU SSIM regression authoring + reference video sync |
| `scripts/checkpoint_conversion/AGENTS.md` | Adding a converter for a new HF/official checkpoint |
## Critical: Two Training Stacks Coexist
- `fastvideo/training/` — legacy, monolithic per-model `*_training_pipeline.py` and
`*_distillation_pipeline.py`. Still authoritative for shipped models.
- `fastvideo/train/` — new modular framework (composable methods × models × callbacks
driven by YAML). Preferred for new training work.
Pick the matching stack before editing. Do not migrate a pipeline between them
without an explicit ask — the conventions and config surfaces differ.
+2 -2
View File
@@ -128,7 +128,7 @@ class CLIPFeatureExtractor(BaseFeatureExtractor):
def __init__(self, device: str = 'cuda', model_name: str = "openai/clip-vit-base-patch32"):
if not TRANSFORMERS_AVAILABLE:
raise ImportError("Please install transformers: uv pip install transformers")
raise ImportError("Please install transformers: pip install transformers")
super().__init__(device)
self.processor = CLIPProcessor.from_pretrained(model_name)
self.model = CLIPModel.from_pretrained(model_name).to(self.device)
@@ -171,7 +171,7 @@ class VideoMAEFeatureExtractor(BaseFeatureExtractor):
def __init__(self, device: str = 'cuda', model_name: str = "MCG-NJU/videomae-base"):
if not TRANSFORMERS_AVAILABLE:
raise ImportError("Please install transformers: uv pip install transformers")
raise ImportError("Please install transformers: pip install transformers")
super().__init__(device)
self.model = VideoMAEModel.from_pretrained(model_name).to(self.device)
self.model.eval()
+1 -1
View File
@@ -57,7 +57,7 @@ class I3DFeatureExtractor(nn.Module):
except Exception as e:
raise RuntimeError(f"Failed to load I3D model from Hugging Face Hub. Error: {e}\n"
f"Ensure you have internet connection and huggingface_hub installed:\n"
f"uv pip install huggingface_hub") from e
f"pip install huggingface_hub") from e
def preprocess(self, videos: torch.Tensor) -> torch.Tensor:
"""
+1 -1
View File
@@ -1,7 +1,7 @@
#!/bin/bash
# 1. Install missing dependency
uv pip install -q opencv-python-headless transformers huggingface_hub
pip install -q opencv-python-headless transformers huggingface_hub
# 2. Run FVD script
python benchmarks/fvd/run_fvd.py
+1 -1
View File
@@ -1,4 +1,4 @@
#!/bin/bash
# 1. Install missing dependency
uv pip install -q opencv-python-headless
pip install -q opencv-python-headless
+2 -2
View File
@@ -38,10 +38,10 @@ cp -r /path/to/FastVideo/comfyui /path/to/ComfyUI/custom_nodes/FastVideo
#### Install dependencies:
Currently, the only dependency is `fastvideo`, which can be installed with `uv`.
Currently, the only dependency is `fastvideo`, which can be installed using pip.
```bash
uv pip install fastvideo
pip install fastvideo
```
#### Install missing custom nodes:
+3 -3
View File
@@ -42,15 +42,15 @@ RUN source $HOME/.local/bin/env && \
uv venv --python 3.10 --seed /opt/venv && \
source /opt/venv/bin/activate && \
uv pip install --no-cache-dir --upgrade pip && \
uv pip install --no-cache-dir ".[dev]" && \
uv pip install --no-cache-dir https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.4/flash_attn-2.8.3+cu128torch2.11-cp310-cp310-linux_x86_64.whl
uv pip install --no-cache-dir .[dev] && \
uv pip install --no-cache-dir https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.7.16/flash_attn-2.8.3+cu128torch2.10-cp310-cp310-linux_x86_64.whl
COPY . .
# Install dependencies using uv and set up shell configuration
RUN source $HOME/.local/bin/env && \
source /opt/venv/bin/activate && \
uv pip install --no-cache-dir -e ".[dev]" && \
uv pip install --no-cache-dir -e .[dev] && \
git config --unset-all http.https://github.com/.extraheader || true && \
echo 'source /opt/venv/bin/activate' >> /root/.bashrc && \
echo 'if [ -n "$ZSH_VERSION" ] && [ -f ~/.zshrc ]; then . ~/.zshrc; elif [ -f ~/.bashrc ]; then . ~/.bashrc; fi' > /root/.profile
+3 -3
View File
@@ -42,15 +42,15 @@ RUN source $HOME/.local/bin/env && \
uv venv --python 3.11 --seed /opt/venv && \
source /opt/venv/bin/activate && \
uv pip install --no-cache-dir --upgrade pip && \
uv pip install --no-cache-dir ".[dev]" && \
uv pip install --no-cache-dir https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.4/flash_attn-2.8.3+cu128torch2.11-cp311-cp311-linux_x86_64.whl
uv pip install --no-cache-dir .[dev] && \
uv pip install --no-cache-dir https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.7.16/flash_attn-2.8.3+cu128torch2.10-cp311-cp311-linux_x86_64.whl
COPY . .
# Install dependencies using uv and set up shell configuration
RUN source $HOME/.local/bin/env && \
source /opt/venv/bin/activate && \
uv pip install --no-cache-dir -e ".[dev]" && \
uv pip install --no-cache-dir -e .[dev] && \
git config --unset-all http.https://github.com/.extraheader || true && \
echo 'source /opt/venv/bin/activate' >> /root/.bashrc && \
echo 'if [ -n "$ZSH_VERSION" ] && [ -f ~/.zshrc ]; then . ~/.zshrc; elif [ -f ~/.bashrc ]; then . ~/.bashrc; fi' > /root/.profile
+3 -3
View File
@@ -42,15 +42,15 @@ RUN source $HOME/.local/bin/env && \
uv venv --python 3.12 --seed /opt/venv && \
source /opt/venv/bin/activate && \
uv pip install --no-cache-dir --upgrade pip && \
uv pip install --no-cache-dir ".[dev]" && \
uv pip install --no-cache-dir https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.4/flash_attn-2.8.3+cu128torch2.11-cp312-cp312-linux_x86_64.whl
uv pip install --no-cache-dir .[dev] && \
uv pip install --no-cache-dir https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.7.16/flash_attn-2.8.3+cu128torch2.10-cp312-cp312-linux_x86_64.whl
COPY . .
# Install dependencies using uv and set up shell configuration
RUN source $HOME/.local/bin/env && \
source /opt/venv/bin/activate && \
uv pip install --no-cache-dir -e ".[dev]" && \
uv pip install --no-cache-dir -e .[dev] && \
git config --unset-all http.https://github.com/.extraheader || true && \
echo 'source /opt/venv/bin/activate' >> /root/.bashrc && \
echo 'if [ -n "$ZSH_VERSION" ] && [ -f ~/.zshrc ]; then . ~/.zshrc; elif [ -f ~/.bashrc ]; then . ~/.bashrc; fi' > /root/.profile
+2 -2
View File
@@ -42,7 +42,7 @@ RUN source $HOME/.local/bin/env && \
uv venv --python 3.12 --seed /opt/venv && \
source /opt/venv/bin/activate && \
uv pip install --no-cache-dir --upgrade pip && \
uv pip install --no-cache-dir ".[dev]" && \
uv pip install --no-cache-dir .[dev] && \
uv pip install --no-cache-dir flash-attn==2.8.3 --no-build-isolation
COPY . .
@@ -50,7 +50,7 @@ COPY . .
# Install dependencies using uv and set up shell configuration
RUN source $HOME/.local/bin/env && \
source /opt/venv/bin/activate && \
uv pip install --no-cache-dir -e ".[dev]" && \
uv pip install --no-cache-dir -e .[dev] && \
git config --unset-all http.https://github.com/.extraheader || true && \
echo 'source /opt/venv/bin/activate' >> /root/.bashrc && \
echo 'if [ -n "$ZSH_VERSION" ] && [ -f ~/.zshrc ]; then . ~/.zshrc; elif [ -f ~/.bashrc ]; then . ~/.bashrc; fi' > /root/.profile
+1 -1
View File
@@ -43,7 +43,7 @@ COPY . .
# Install dependencies using uv and set up shell configuration
RUN source $HOME/.local/bin/env && \
source /opt/venv/bin/activate && \
uv pip install --no-cache-dir -e ".[rocm]" && \
uv pip install --no-cache-dir -e .[rocm] && \
git config --unset-all http.https://github.com/.extraheader || true && \
echo 'source /opt/venv/bin/activate' >> /root/.bashrc && \
echo 'if [ -n "$ZSH_VERSION" ] && [ -f ~/.zshrc ]; then . ~/.zshrc; elif [ -f ~/.bashrc ]; then . ~/.bashrc; fi' > /root/.profile
+1 -1
View File
@@ -6,7 +6,7 @@ This directory contains the FastVideo documentation built with MkDocs.
```bash
# Install dependencies
uv pip install -r requirements-mkdocs.txt
pip install -r requirements-mkdocs.txt
# Serve docs with live reload (recommended for development)
mkdocs serve
+2 -2
View File
@@ -62,9 +62,9 @@ This page contains the complete API reference for the FastVideo library.
show_root_toc_entry: true
heading_level: 4
#### fastvideo.api.sampling_param
#### fastvideo.configs.sample
::: fastvideo.api.sampling_param
::: fastvideo.configs.sample
options:
show_source: true
show_root_heading: true
-210
View File
@@ -1,210 +0,0 @@
# Activation Trace Mode
!!! note
This page covers Extension 0 (module forward hooks), which is the implemented
tracing mechanism. Extensions 1-3 are design sketches for future work and are
**not yet implemented**.
## Overview
Activation trace mode is a zero-overhead-when-off, env-gated mechanism for
dumping per-layer activation statistics during FastVideo inference. Its primary
use case is **parity debugging across model ports**: enable tracing on both
FastVideo and the upstream reference implementation, then `diff` the resulting
JSONL files to find the first divergent layer.
The mechanism is intentionally narrow. It doesn't replace general logging,
profiling, or function tracing. It answers one question: "at which layer do
FastVideo and the reference model first produce different numbers?"
## When to use
- Investigating numerical drift between FastVideo and an upstream reference.
- Debugging mid-pipeline divergence (e.g., one block produces wrong output while earlier blocks match).
- Validating that a refactor preserves bf16 noise-floor behavior across many layers.
## When NOT to use
| Goal | Use instead |
|---|---|
| General logging | `init_logger(__name__)` |
| Per-stage timing | `FASTVIDEO_STAGE_LOGGING` |
| Profiling kernel timings | `FASTVIDEO_TORCH_PROFILER_DIR` (see [Profiling](profiling.md)) |
| Function-call tracing | `FASTVIDEO_TRACE_FUNCTION` (heavy) |
## Quickstart
```bash
FASTVIDEO_TRACE_ACTIVATIONS=1 \
FASTVIDEO_TRACE_LAYERS="^block\.layers\.[0-9]+$" \
FASTVIDEO_TRACE_STATS="abs_mean,sum,max,shape" \
FASTVIDEO_TRACE_OUTPUT="/tmp/fv_trace.jsonl" \
python examples/inference/basic/basic_magi_human.py
```
Each line in `/tmp/fv_trace.jsonl` is a JSON record:
```json
{"module": "block.layers.0", "tensor": "out", "step": 0, "abs_mean": 1.234, "sum": -5.678, "max": 9.012, "shape": [1, 4096, 5120]}
```
## Configuration
| Env var | Default | Description |
|---|---|---|
| `FASTVIDEO_TRACE_ACTIVATIONS` | `False` | Master toggle. When unset or false, **zero overhead** in the production hot path. |
| `FASTVIDEO_TRACE_LAYERS` | `""` (all) | Python regex filter applied to `model.named_modules()` names. Empty string matches all modules. |
| `FASTVIDEO_TRACE_STATS` | `"abs_mean,sum"` | Comma-separated stats to compute. Available: `abs_mean`, `sum`, `min`, `max`, `mean`, `std`, `shape`, `dtype`. |
| `FASTVIDEO_TRACE_OUTPUT` | `"/tmp/fv_trace_<pid>.jsonl"` | Output file path. `<pid>` is replaced with the process ID at runtime. |
| `FASTVIDEO_TRACE_STEPS` | `""` (all) | Comma-separated denoising step indices to capture. Empty string captures all steps. |
## Workflow: parity-debug a model port
1. Set up a tightly-controlled comparison: a parity test or a small standalone
script that loads both the FastVideo model and the upstream reference with
identical inputs and seeds.
2. Run the FastVideo side with tracing on:
```bash
FASTVIDEO_TRACE_ACTIVATIONS=1 \
FASTVIDEO_TRACE_LAYERS="<your regex>" \
FASTVIDEO_TRACE_OUTPUT="/tmp/fv_trace_fv.jsonl" \
python <fv_runner.py>
```
3. Run the upstream side. The upstream repo needs separate instrumentation. See
"Hooking the upstream side" below.
4. Sort both files by `(module, step)` if needed, then diff:
```bash
diff /tmp/fv_trace_fv.jsonl /tmp/fv_trace_upstream.jsonl
```
5. The first divergent line identifies the first layer where FastVideo and the
upstream produce different outputs. Start debugging there.
## Architecture (Extension 0: module forward hooks)
At pipeline initialization, `attach_activation_trace()` reads the env vars once.
If `FASTVIDEO_TRACE_ACTIVATIONS` is unset or false, the function returns
immediately and no hooks are registered. If tracing is on, it walks
`model.named_modules()`, filters by the layer regex, and registers an
`ActivationStatHook` on each matching module.
During the forward pass, each hook fires after its module completes, computes
the requested stats on the output tensor, and appends a JSON record to the
output file.
```
ComposedPipelineBase
└─ attach_activation_trace()
├─ reads env vars (once at startup)
├─ if off: returns None immediately
└─ if on: walks named_modules()
└─ registers ActivationStatHook on matching modules
└─ on each forward: compute stats → append JSONL
```
### Zero-overhead-when-off guarantee
- The env var check happens **once at startup** inside `attach_activation_trace()`.
- If the env var is unset or false, the function returns `None` immediately.
- No hooks are registered. No branches are added to the production forward path.
- The only cost when tracing is off is one env var lookup at pipeline
initialization, which takes under a microsecond.
### Hooking the upstream side
The upstream reference repo isn't part of FastVideo, so it can't read FastVideo
env vars directly. Two options:
**Option 1: Inline patch** in your local clone of the upstream repo. Add
`register_forward_hook` calls in the same shape as `ActivationStatHook`. Clean
up afterward with `git stash` or `git checkout HEAD -- <file>`.
**Option 2: Wrapper script**. Write a small Python harness that imports the
upstream model, walks its `named_modules()`, and attaches hooks externally.
This is the same pattern used in
`tests/local_tests/transformers/_debug_magi_human_block_parity.py`.
The `add-model-trace` skill at `~/.config/opencode/skill/add-model-trace/`
provides a script template for this purpose.
## Future extensions (design only, not yet implemented)
### Extension 1: FX/Dynamo backend graph rewrite
**Granularity**: per-FX-node (every matmul, every add).
**Mechanism**: a `torch.compile` backend that takes the captured `GraphModule`
and inserts logger nodes after each op. Compiles into a separate artifact from
the production graph.
**Off semantics**: zero overhead. The production compile path is untouched.
**When to add**: if you need to trace inside a `torch.compile`'d graph and
Extension 0 is too coarse.
**Build cost**: roughly 1-2 days. Reference:
`torchao.quantization.pt2e._numeric_debugger`.
### Extension 2: AST source injection at import time
**Granularity**: per-line (between any two Python statements).
**Mechanism**: an importlib loader hook rewrites Python source AST at module
import time, inserting `if TRACE: dump(...)` statements. The decision is made
once at import.
**Off semantics**: zero overhead. If the env var is off at import time, source
is loaded as-is.
**When to add**: if you need per-line granularity that even FX-node-level can't
provide. This is almost never the right choice.
**Build cost**: roughly 1 week. Brittle and hard to debug.
### Extension 3: `__torch_dispatch__` / `TorchDispatchMode`
**Granularity**: per-op (every dispatcher call: matmul, add, view, etc.).
**Mechanism**: a `TorchDispatchMode` context manager that intercepts all ops at
the dispatcher level.
**Off semantics**: zero overhead. PyTorch's dispatcher only invokes mode hooks
when a mode is active.
**When on**: significant overhead. Every op pays a Python callback cost. Triton
kernels bypass it.
**When to add**: useful for quantization or dtype debugging where module-level
granularity isn't enough.
**Build cost**: roughly 1 day. Reference:
`torch.utils._python_dispatch.TorchDispatchMode`.
## Comparison with similar tools
| Tool | Pattern | FastVideo equivalent |
|---|---|---|
| SGLang `--debug-tensor-dump-output-folder` | env-gated forward hooks at startup | Extension 0 (this) |
| TransformerEngine `DumpTensors` | config-driven selective dumps | Extension 0 (env-driven) |
| HuggingFace `output_hidden_states=True` | source-level boolean gating | Not used; Extension 0 avoids model code edits |
| torchao numeric debugger | FX pass + node-level loggers | Extension 1 (future) |
| W&B `wandb.watch()` | runtime forward hooks (always on once registered) | Extension 0 has a similar mechanism, but gated off by default |
## Implementation references
- Module: `fastvideo/hooks/activation_trace.py`
- Env vars: `fastvideo/envs.py` (`FASTVIDEO_TRACE_ACTIVATIONS` and friends)
- Pipeline integration: `fastvideo/pipelines/composed_pipeline_base.py`
- Tests: `fastvideo/tests/hooks/test_activation_trace.py`
- Companion skill (for ad-hoc port investigations): `~/.config/opencode/skill/add-model-trace/`
## Changelog
| Date | Change |
|---|---|
| 2026-05-01 | Initial Extension 0 (module forward hooks) implementation. Extensions 1-3 designed but not implemented. |
+6 -31
View File
@@ -24,7 +24,7 @@ PR push
Runs on the PR branch directly
│
pass ──► Mergify auto-squash-merges to main, branch deleted
fail ──► fix the regression, push, and /merge again
fail ──► Mergify removes 'ready' label; fix and /merge again
```
---
@@ -102,8 +102,8 @@ failing test's output.
| Performance Tests | `performance` | 30 min |
| API Server Tests | `api_server` | 30 min |
If a Full Suite test fails, check the Buildkite build log for the failing step's output.
Fix the regression, push, and comment `/merge` again to re-trigger.
A Full Suite failure removes the `ready` label automatically. A Mergify comment links to
the Buildkite build. Fix the regression, push, and comment `/merge` again.
---
@@ -129,8 +129,8 @@ Suite passing directly on the PR branch.
- No merge conflicts
5. If all conditions pass, Mergify squash-merges to `main` automatically. The branch is
deleted after merge.
6. If the Full Suite fails, the developer fixes the issue, pushes, and comments `/merge`
again to re-trigger.
6. If the Full Suite fails, Mergify removes the `ready` label and posts a comment linking to
the Buildkite build. The developer fixes the issue, pushes, and comments `/merge` again.
**Merge conditions summary:**
@@ -173,7 +173,7 @@ Applied by Mergify based on which paths you modified. Multiple scope labels can
| Label | File paths that trigger it |
|-------|---------------------------|
| `scope: training` | `fastvideo/train/`, `fastvideo/training/`, `fastvideo/distillation/`, `examples/train/`, `examples/training/`, `examples/distill/` |
| `scope: inference` | `fastvideo/pipelines/basic/`, `fastvideo/pipelines/stages/`, `fastvideo/pipelines/samplers/`, `fastvideo/entrypoints/`, `fastvideo/worker/`, `fastvideo/api/sampling_param.py`, `fastvideo/configs/pipelines/`, `examples/inference/` |
| `scope: inference` | `fastvideo/pipelines/basic/`, `fastvideo/pipelines/stages/`, `fastvideo/pipelines/samplers/`, `fastvideo/entrypoints/`, `fastvideo/worker/`, `fastvideo/configs/sample/`, `fastvideo/configs/pipelines/`, `examples/inference/` |
| `scope: attention` | `fastvideo/attention/` |
| `scope: kernel` | `fastvideo-kernel/`, `csrc/` |
| `scope: data` | `fastvideo/dataset/`, `fastvideo/pipelines/preprocess/`, `examples/preprocessing/` |
@@ -279,30 +279,6 @@ Triggers a specific Buildkite test or suite on the current PR branch.
| `/test api` | API server integration tests | `api_server` |
| `/test full` | Entire Full Suite | all (with `TEST_SCOPE=full`) |
| `/test fastcheck` | Entire Fastcheck suite | fastcheck (with `TEST_SCOPE=fastcheck`) |
| `/test pre-commit` | Pre-commit checks on PR code | — (runs `ci-precommit.yml` via `workflow_call`) |
**Re-running failed tests:** When you use `/test <name>` to re-run a specific failed test,
the resulting Buildkite check uses the same name as the original (e.g., `/test encoder`
creates `buildkite/ci/microscope-encoder-tests`). This overwrites the failed check status.
Once all tests in a tier pass, the aggregate status (`fastcheck-passed` or
`full-suite-passed`) is automatically updated to `success` by the `ci-aggregate-status.yml`
workflow.
**How aggregate status refresh works:**
1. `/test <name>` triggers a Buildkite build with `TEST_SCOPE=direct`. The test step uses
the same label as its fastcheck/full-suite counterpart, so the resulting GitHub check
overwrites the original.
2. When the build completes, Buildkite's `notify` posts a `direct-test-completed` commit
status. This is the only signal that triggers the aggregate workflow — intermediate step
status updates do not trigger it.
3. `ci-aggregate-status.yml` fires, calls `getCombinedStatusForRef` to fetch the latest
status for every context on that commit (each context returns only its most recent
state), groups them by prefix (`microscope-*` → fastcheck, `test-tube-*`/`bar-chart-*`
→ full suite), and posts `fastcheck-passed: success` or `full-suite-passed: success` if
all entries in the group are `success`.
4. Tests that were never triggered (skipped by monorepo-diff) have no status entry and do
not block the aggregate.
---
@@ -320,7 +296,6 @@ Protected branches (`main`, `master`, `release/*`) are never deleted.
| `ci-precommit.yml` | Every push / PR against `main` | Runs pre-commit hooks (yapf, ruff, mypy, codespell, pymarkdown, actionlint, check-filenames) |
| `ci-trigger-full-suite.yml` | `ready` label added to a PR | Calls Buildkite API to run Full Suite on the PR branch |
| `ci-slash-commands.yml` | PR comment starting with `/merge` or `/test` | Handles slash commands; adds `ready` label or triggers Buildkite |
| `ci-aggregate-status.yml` | Any Buildkite commit status update | Checks if all tests in a tier passed; updates `fastcheck-passed` or `full-suite-passed` |
| `community-issue-labeler.yml` | Issue opened or edited | Auto-labels issues by keyword matching against title and body |
| `community-welcome.yml` | First contribution | Posts a welcome comment for first-time contributors |
| `community-stale.yml` | Scheduled | Marks and closes stale issues and PRs |
+7 -11
View File
@@ -44,7 +44,7 @@ FastVideo maps a Diffusers-style repo into a pipeline like:
- `fastvideo/configs/models/*`: arch configs and `param_names_mapping` for
weight name translation.
- `fastvideo/configs/pipelines/*`: pipeline wiring (component classes + names).
- `fastvideo/api/sampling_param.py`: runtime sampling parameters.
- `fastvideo/configs/sample/*`: default runtime sampling parameters.
- `fastvideo/pipelines/basic/*`: end-to-end pipeline logic built from stages.
- `model_index.json`: the HF repo entrypoint that maps component names to
classes and weight files.
@@ -55,7 +55,7 @@ Minimal usage example (based on `examples/inference/basic/basic.py`):
```python
from fastvideo import VideoGenerator
from fastvideo.api.sampling_param import SamplingParam
from fastvideo.configs.sample import SamplingParam
model_id = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" # or official_weights/<model_name>/
generator = VideoGenerator.from_pretrained(model_id, num_gpus=1)
@@ -296,10 +296,8 @@ Action:
- Add or reuse a numerical parity test that loads the official model and the
FastVideo model and compares outputs.
- See examples in `tests/local_tests/` organized by model family
(e.g., `tests/local_tests/sd35/`, `tests/local_tests/ltx2/`,
`tests/local_tests/stable_audio/`) and the navigation index in
`tests/local_tests/README.md`.
- See examples in `tests/local_tests/` (e.g., `tests/local_tests/upsamplers/`)
and the commands in `tests/local_tests/README.md`.
- If there are discrepancies, add opt‑in logging to both models and compare
activation summaries (layer output sums, per‑stage logs).
- First align the loaded weights (validate `param_names_mapping`).
@@ -321,8 +319,7 @@ Purpose:
- `fastvideo/configs/pipelines/` describes pipeline wiring and model module
names.
- `fastvideo/api/sampling_param.py` defines runtime sampling parameters.
Defaults come from profiles in `fastvideo/pipelines/basic/<family>/profiles.py`.
- `fastvideo/configs/sample/` defines default runtime parameters.
Action:
@@ -350,8 +347,7 @@ Purpose:
Action:
- Add a pipeline parity test under `tests/local_tests/<family>/`
(e.g., `tests/local_tests/<family>/test_<family>_pipeline_parity.py`).
- Add a pipeline parity test under `tests/local_tests/pipelines/`.
- See the [Testing Guide](testing.md) for test conventions.
### 7) Add user‑facing examples
@@ -478,7 +474,7 @@ FastVideo integration.
3. Pipeline wiring.
- Pipeline: `fastvideo/pipelines/basic/wan/wan_pipeline.py`
- Pipeline config: `fastvideo/configs/pipelines/wan.py`
- Sampling defaults: `fastvideo/pipelines/basic/wan/profiles.py`
- Sampling defaults: `fastvideo/configs/sample/wan.py`
4. Minimal example.
- Script: `examples/inference/basic/basic.py`
+1 -1
View File
@@ -99,7 +99,7 @@ cd /FastVideo
**Install the package**
```bash
uv pip install -e ".[dev]"
uv pip install -e .[dev]
```
The Docker image already includes Flash Attention and most heavy dependencies, so this is fast.
+1 -1
View File
@@ -49,7 +49,7 @@ git clone https://github.com/hao-ai-lab/FastVideo.git && cd FastVideo
Install FastVideo in editable mode and set up hooks:
```bash
uv pip install -e ".[dev]"
uv pip install -e .[dev]
# Optional: FlashAttention (builds native kernels)
uv pip install flash-attn --no-build-isolation -v
+5 -10
View File
@@ -104,9 +104,8 @@ distillation, self-forcing, VSA, VMoBA, performance benchmarks, and API server t
8. If all Full Suite tests pass and all merge conditions are met (approval, valid title,
pre-commit green, fastcheck green, no draft, no conflicts), Mergify squash-merges to
`main` automatically. Your branch is deleted.
9. If a Full Suite test fails, check the Buildkite build log for the failing step. Fix the
issue, push, and comment `/merge` again. You can also re-run individual failed tests
with `/test <name>` — see below.
9. If a Full Suite test fails, Mergify removes the `ready` label and posts a comment with a
link to the Buildkite build. Fix the issue, push, and comment `/merge` again.
!!! note
Only contributors with write permission to the repository can trigger slash commands.
@@ -150,15 +149,10 @@ Comment on your PR to trigger specific tests independently of the auto-merge flo
/test vmoba # VMoBA inference tests
/test performance # Performance benchmarks
/test api # API server integration tests
/test pre-commit # Pre-commit checks on PR code
```
The workflow reacts with a 🚀 emoji to confirm the command was received.
When you re-run an individual test with `/test <name>`, the new result overwrites the
original failed check (same Buildkite check name). Once all tests in a tier pass, the
`fastcheck-passed` or `full-suite-passed` status is automatically updated.
---
## Troubleshooting
@@ -205,8 +199,9 @@ Mergify removes the `needs-rebase` label automatically once conflicts are resolv
### Full Suite failed after `/merge`
The Full Suite found a regression. Check the failing Buildkite step's output for assertion
errors or tracebacks.
The Full Suite found a regression. Mergify removes the `ready` label and posts a comment
linking to the Buildkite build. Check the failing step's output for assertion errors or
tracebacks.
Common causes:
@@ -1,480 +0,0 @@
status_definitions:
kept: "Public field remains on a public adapter surface with the same meaning."
moved: "Public field remains supported but normalizes into a different nested path."
preset_owned: "Public field remains supported only through a model/preset-specific surface."
compatibility_only: "Legacy public field remains adapter-only during migration and is not part of the canonical typed schema."
private_only: "Field should only be handled by private adapters and is not a public FastVideo compatibility promise."
internal_only: "Field is runtime/config plumbing and should not be part of the new public typed inference API."
surfaces:
fastvideo_args:
moved:
model_path: generator.model_path
workload_type: generator.pipeline.workload_type
distributed_executor_backend: generator.engine.execution_backend
trust_remote_code: generator.trust_remote_code
revision: generator.revision
num_gpus: generator.engine.num_gpus
tp_size: generator.engine.parallelism.tp_size
sp_size: generator.engine.parallelism.sp_size
hsdp_replicate_dim: generator.engine.parallelism.hsdp_replicate_dim
hsdp_shard_dim: generator.engine.parallelism.hsdp_shard_dim
dist_timeout: generator.engine.parallelism.dist_timeout
lora_path: generator.pipeline.components.lora_path
dit_cpu_offload: generator.engine.offload.dit
use_fsdp_inference: generator.engine.use_fsdp_inference
dit_layerwise_offload: generator.engine.offload.dit_layerwise
text_encoder_cpu_offload: generator.engine.offload.text_encoder
image_encoder_cpu_offload: generator.engine.offload.image_encoder
vae_cpu_offload: generator.engine.offload.vae
pin_cpu_memory: generator.engine.offload.pin_cpu_memory
enable_torch_compile: generator.engine.compile.enabled
torch_compile_kwargs: generator.engine.compile.backend,fullgraph,mode,dynamic,extras
disable_autocast: generator.engine.disable_autocast
enable_stage_verification: generator.engine.enable_stage_verification
prompt_txt: request.inputs.prompt_path
override_text_encoder_safetensors: generator.pipeline.components.text_encoder_weights
override_text_encoder_quant: generator.engine.quantization.text_encoder_quant
override_transformer_cls_name: generator.pipeline.components.override_transformer_cls_name
init_weights_from_safetensors: generator.pipeline.components.transformer_weights
init_weights_from_safetensors_2: generator.pipeline.components.transformer_2_weights
override_pipeline_cls_name: generator.pipeline.components.override_pipeline_cls_name
boundary_ratio: request.sampling.boundary_ratio
ltx2_vae_tiling: generator.pipeline.vae_tiling
preset_owned:
ltx2_vae_spatial_tile_size_in_pixels: generator.pipeline.preset_overrides.ltx2.vae.spatial_tile_size_in_pixels
ltx2_vae_spatial_tile_overlap_in_pixels: generator.pipeline.preset_overrides.ltx2.vae.spatial_tile_overlap_in_pixels
ltx2_vae_temporal_tile_size_in_frames: generator.pipeline.preset_overrides.ltx2.vae.temporal_tile_size_in_frames
ltx2_vae_temporal_tile_overlap_in_frames: generator.pipeline.preset_overrides.ltx2.vae.temporal_tile_overlap_in_frames
ltx2_initial_latent_path: request.extensions.ltx2.initial_latent_path
compatibility_only:
mode: "Legacy multi-mode FastVideoArgs switch; typed inference config should not expose execution mode."
inference_mode: "Legacy boolean mirror of mode; kept only through adapters while FastVideoArgs remains."
lora_nickname: "Legacy adapter-selection surface pending LoRA API cleanup."
lora_target_modules: "Legacy LoRA configuration surface pending dedicated component API."
output_type: "Legacy output formatting surface pending GenerationResult cleanup."
VSA_sparsity: "Model-specific inference optimization not yet represented in the typed public schema."
moba_config_path: "Model-specific MoBA optimization surface not yet represented in the typed public schema."
master_port: "Executor/bootstrap compatibility field; not part of the canonical inference schema."
private_only:
ray_placement_group: "Ray deployment-only field."
ray_runtime_env: "Ray deployment-only field."
internal_only:
pipeline_config: "Legacy internal carrier object."
preprocess_config: "Legacy preprocess carrier object."
moba_config: "Derived runtime config loaded from moba_config_path."
model_paths: "Runtime bookkeeping."
model_loaded: "Runtime bookkeeping."
pipeline_config_base:
moved:
pipeline_config_path: generator.pipeline.components.pipeline_config_path
preset_owned:
embedded_cfg_scale: generator.pipeline.preset_overrides.embedded_cfg_scale
flow_shift: generator.pipeline.preset_overrides.flow_shift
flow_shift_sr: generator.pipeline.preset_overrides.flow_shift_sr
is_causal: generator.pipeline.preset_overrides.is_causal
vae_tiling: generator.pipeline.preset_overrides.vae_tiling
vae_sp: generator.pipeline.preset_overrides.vae_sp
dmd_denoising_steps: generator.pipeline.preset_overrides.dmd_denoising_steps
ti2v_task: generator.pipeline.preset_overrides.ti2v_task
boundary_ratio: generator.pipeline.preset_overrides.boundary_ratio
compatibility_only:
model_path: "Redundant with generator.model_path."
disable_autocast: "Duplicated by generator.engine.disable_autocast during migration."
dit_precision: "Precision override pending dedicated typed component precision design."
upsampler_precision: "Precision override pending dedicated typed component precision design."
vae_precision: "Precision override pending dedicated typed component precision design."
image_encoder_precision: "Precision override pending dedicated typed component precision design."
text_encoder_precisions: "Precision override pending dedicated typed component precision design."
internal_only:
dit_config: "Legacy internal component config object."
upsampler_config: "Legacy internal component config object."
vae_config: "Legacy internal component config object."
image_encoder_config: "Legacy internal component config object."
text_encoder_configs: "Legacy internal component config object."
preprocess_text_funcs: "Internal text preprocessing hooks."
postprocess_text_funcs: "Internal text postprocessing hooks."
pipeline_config_extensions:
preset_owned:
conditioning_strategy:
sources:
- fastvideo.configs.pipelines.cosmos.CosmosConfig
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
max_num_conditional_frames:
sources:
- fastvideo.configs.pipelines.cosmos.CosmosConfig
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
min_num_conditional_frames:
sources:
- fastvideo.configs.pipelines.cosmos.CosmosConfig
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
sigma_conditional:
sources:
- fastvideo.configs.pipelines.cosmos.CosmosConfig
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
sigma_data:
sources:
- fastvideo.configs.pipelines.cosmos.CosmosConfig
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
state_ch:
sources:
- fastvideo.configs.pipelines.cosmos.CosmosConfig
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
state_t:
sources:
- fastvideo.configs.pipelines.cosmos.CosmosConfig
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
text_encoder_class:
sources:
- fastvideo.configs.pipelines.cosmos.CosmosConfig
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
autoregressive_chunk_frames:
sources:
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
autoregressive_overlap_frames:
sources:
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
cfg_behavior:
sources:
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
default_camera_rotation:
sources:
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
default_movement_distance:
sources:
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
default_negative_prompt:
sources:
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
default_trajectory_type:
sources:
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
filter_points_threshold:
sources:
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
fps:
sources:
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
frame_buffer_max:
sources:
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
moge_model_name:
sources:
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
noise_aug_strength:
sources:
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
num_frames:
sources:
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
offload_moge_after_depth:
sources:
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
use_moge_depth:
sources:
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
video_resolution:
sources:
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
text_encoder_crop_start:
sources:
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15I2V480PStepDistilledConfig
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15I2V720PConfig
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15SR1080PConfig
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15T2V480PConfig
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15T2V720PConfig
- fastvideo.configs.pipelines.hyworld.HYWorldConfig
- fastvideo.configs.pipelines.hyworld.Hunyuan15T2V480PConfig
text_encoder_max_lengths:
sources:
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15I2V480PStepDistilledConfig
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15I2V720PConfig
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15SR1080PConfig
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15T2V480PConfig
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15T2V720PConfig
- fastvideo.configs.pipelines.hyworld.HYWorldConfig
- fastvideo.configs.pipelines.hyworld.Hunyuan15T2V480PConfig
precision:
sources:
- fastvideo.configs.pipelines.lingbotworld.LingBotWorldI2V480PConfig
- fastvideo.configs.pipelines.lingbotworld.Wan2_2_I2V_A14B_Config
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionI2VConfig
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionI2V_A14B_Config
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionT2VConfig
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionT2V_14B_Config
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionT2V_1_3B_Config
- fastvideo.configs.pipelines.wan.FastWan2_1_T2V_480P_Config
- fastvideo.configs.pipelines.wan.FastWan2_2_TI2V_5B_Config
- fastvideo.configs.pipelines.wan.MatrixGameBaseI2V480PConfig
- fastvideo.configs.pipelines.wan.MatrixGameI2V480PConfig
- fastvideo.configs.pipelines.wan.SelfForcingWan2_2_T2V480PConfig
- fastvideo.configs.pipelines.wan.SelfForcingWanT2V480PConfig
- fastvideo.configs.pipelines.wan.WANV2VConfig
- fastvideo.configs.pipelines.wan.Wan2_2_I2V_A14B_Config
- fastvideo.configs.pipelines.wan.Wan2_2_T2V_A14B_Config
- fastvideo.configs.pipelines.wan.Wan2_2_TI2V_5B_Config
- fastvideo.configs.pipelines.wan.WanI2V480PConfig
- fastvideo.configs.pipelines.wan.WanI2V720PConfig
- fastvideo.configs.pipelines.wan.WanT2V480PConfig
- fastvideo.configs.pipelines.wan.WanT2V720PConfig
warp_denoising_step:
sources:
- fastvideo.configs.pipelines.lingbotworld.LingBotWorldI2V480PConfig
- fastvideo.configs.pipelines.lingbotworld.Wan2_2_I2V_A14B_Config
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionI2VConfig
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionI2V_A14B_Config
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionT2VConfig
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionT2V_14B_Config
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionT2V_1_3B_Config
- fastvideo.configs.pipelines.wan.FastWan2_1_T2V_480P_Config
- fastvideo.configs.pipelines.wan.FastWan2_2_TI2V_5B_Config
- fastvideo.configs.pipelines.wan.MatrixGameBaseI2V480PConfig
- fastvideo.configs.pipelines.wan.MatrixGameI2V480PConfig
- fastvideo.configs.pipelines.wan.SelfForcingWan2_2_T2V480PConfig
- fastvideo.configs.pipelines.wan.SelfForcingWanT2V480PConfig
- fastvideo.configs.pipelines.wan.WANV2VConfig
- fastvideo.configs.pipelines.wan.Wan2_2_I2V_A14B_Config
- fastvideo.configs.pipelines.wan.Wan2_2_T2V_A14B_Config
- fastvideo.configs.pipelines.wan.Wan2_2_TI2V_5B_Config
- fastvideo.configs.pipelines.wan.WanI2V480PConfig
- fastvideo.configs.pipelines.wan.WanI2V720PConfig
- fastvideo.configs.pipelines.wan.WanT2V480PConfig
- fastvideo.configs.pipelines.wan.WanT2V720PConfig
bsa_cdf_threshold:
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
bsa_chunk_k:
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
bsa_chunk_q:
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
bsa_params:
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
bsa_sparsity:
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
enable_bsa:
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
enable_kv_cache:
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
enhance_hf:
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
offload_kv_cache:
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
t_thresh:
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
use_distill:
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
scheduler_arch:
sources: [fastvideo.configs.pipelines.sd35.SD35Config]
text_encoder_archs:
sources: [fastvideo.configs.pipelines.sd35.SD35Config]
tokenizer_archs:
sources: [fastvideo.configs.pipelines.sd35.SD35Config]
transformer_arch:
sources: [fastvideo.configs.pipelines.sd35.SD35Config]
vae_arch:
sources: [fastvideo.configs.pipelines.sd35.SD35Config]
expand_timesteps:
sources:
- fastvideo.configs.pipelines.wan.FastWan2_2_TI2V_5B_Config
- fastvideo.configs.pipelines.wan.Wan2_2_TI2V_5B_Config
context_noise:
sources: [fastvideo.configs.pipelines.wan.MatrixGameI2V480PConfig]
num_frames_per_block:
sources: [fastvideo.configs.pipelines.wan.MatrixGameI2V480PConfig]
audio_channels:
sources:
- fastvideo.configs.pipelines.stable_audio.StableAudioT2AConfig
- fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig
audio_end_in_s:
sources:
- fastvideo.configs.pipelines.stable_audio.StableAudioT2AConfig
- fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig
audio_start_in_s:
sources:
- fastvideo.configs.pipelines.stable_audio.StableAudioT2AConfig
- fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig
max_audio_duration_s:
sources:
- fastvideo.configs.pipelines.stable_audio.StableAudioT2AConfig
- fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig
sample_size:
sources:
- fastvideo.configs.pipelines.stable_audio.StableAudioT2AConfig
- fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig
sampling_rate:
sources:
- fastvideo.configs.pipelines.stable_audio.StableAudioT2AConfig
- fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig
compatibility_only:
batch_size: "Gen3C inference-only tuning field pending typed batching design."
gradient_checkpointing: "Gen3C inference-only compatibility field pending typed batching design."
guidance_scale: "Gen3C pipeline-level default pending preset/default-request cleanup."
num_inference_steps: "Gen3C pipeline-level default pending preset/default-request cleanup."
internal_only:
audio_decoder_config: "Legacy internal component config object."
audio_decoder_precision: "Precision override pending dedicated component precision design."
vocoder_config: "Legacy internal component config object."
vocoder_precision: "Precision override pending dedicated component precision design."
sampling_param_base:
moved:
image_path: request.inputs.image_path
pil_image: request.inputs.pil_image
video_path: request.inputs.video_path
mouse_cond: request.inputs.mouse_cond
keyboard_cond: request.inputs.keyboard_cond
grid_sizes: request.inputs.grid_sizes
pose: request.inputs.pose
c2ws_plucker_emb: request.inputs.c2ws_plucker_emb
refine_from: request.inputs.refine_from
stage1_video: request.inputs.stage1_video
prompt: request.prompt
negative_prompt: request.negative_prompt
prompt_path: request.inputs.prompt_path
output_path: request.output.output_path
output_video_name: request.output.output_video_name
num_videos_per_prompt: request.sampling.num_videos_per_prompt
seed: request.sampling.seed
num_frames: request.sampling.num_frames
height: request.sampling.height
width: request.sampling.width
height_sr: request.sampling.height_sr
width_sr: request.sampling.width_sr
fps: request.sampling.fps
num_inference_steps: request.sampling.num_inference_steps
num_inference_steps_sr: request.sampling.num_inference_steps_sr
guidance_scale: request.sampling.guidance_scale
guidance_scale_2: request.sampling.guidance_scale_2
guidance_rescale: request.sampling.guidance_rescale
boundary_ratio: request.sampling.boundary_ratio
sigmas: request.sampling.sigmas
enable_teacache: request.runtime.enable_teacache
save_video: request.output.save_video
return_frames: request.output.return_frames
return_trajectory_latents: request.runtime.return_trajectory_latents
return_trajectory_decoded: request.runtime.return_trajectory_decoded
continuation_state: request.state
return_continuation_state: request.output.return_state
preset_owned:
t_thresh: request.stage_overrides.refine.t_thresh
spatial_refine_only: request.stage_overrides.refine.spatial_refine_only
num_cond_frames: request.stage_overrides.refine.num_cond_frames
trajectory_type: request.extensions.gen3c.trajectory_type
movement_distance: request.extensions.gen3c.movement_distance
camera_rotation: request.extensions.gen3c.camera_rotation
prompt_attention_mask: request.extensions.hyworld.prompt_attention_mask
negative_attention_mask: request.extensions.hyworld.negative_attention_mask
camera_states: request.extensions.hunyuangamecraft.camera_states
camera_trajectory: request.extensions.hunyuangamecraft.camera_trajectory
action_list: request.extensions.hunyuangamecraft.action_list
action_speed_list: request.extensions.hunyuangamecraft.action_speed_list
gt_latents: request.extensions.hunyuangamecraft.gt_latents
conditioning_mask: request.extensions.hunyuangamecraft.conditioning_mask
ltx2_cfg_scale_video: request.extensions.ltx2.cfg_scale_video
ltx2_cfg_scale_audio: request.extensions.ltx2.cfg_scale_audio
ltx2_modality_scale_video: request.extensions.ltx2.modality_scale_video
ltx2_modality_scale_audio: request.extensions.ltx2.modality_scale_audio
ltx2_rescale_scale: request.extensions.ltx2.rescale_scale
ltx2_stg_scale_video: request.extensions.ltx2.stg_scale_video
ltx2_stg_scale_audio: request.extensions.ltx2.stg_scale_audio
ltx2_stg_blocks_video: request.extensions.ltx2.stg_blocks_video
ltx2_stg_blocks_audio: request.extensions.ltx2.stg_blocks_audio
audio_start_in_s: request.extensions.stable_audio.audio_start_in_s
audio_end_in_s: request.extensions.stable_audio.audio_end_in_s
init_audio: request.extensions.stable_audio.init_audio
init_audio_strength: request.extensions.stable_audio.init_audio_strength
init_noise_level: request.extensions.stable_audio.init_noise_level
inpaint_audio: request.extensions.stable_audio.inpaint_audio
inpaint_mask: request.extensions.stable_audio.inpaint_mask
internal_only:
data_type: "Derived from the request shape and not a public input."
sampling_param_extensions: {}
openai_image_request:
kept:
model: "HTTP adapter model-routing field."
response_format: "HTTP adapter response formatting field."
output_format: "HTTP adapter output-format field."
background: "HTTP adapter output-format field."
quality: "Compatibility field currently accepted by the adapter."
style: "Compatibility field currently accepted by the adapter."
user: "Compatibility field currently accepted by the adapter."
moved:
prompt: request.prompt
n: request.sampling.num_videos_per_prompt
size:
target: request.sampling.width,height
note: "Adapter parses OpenAI size strings as WIDTHxHEIGHT and forwards width then height."
num_inference_steps: request.sampling.num_inference_steps
guidance_scale: request.sampling.guidance_scale
true_cfg_scale: request.sampling.true_cfg_scale
seed: request.sampling.seed
negative_prompt: request.negative_prompt
enable_teacache: request.runtime.enable_teacache
openai_video_request:
kept:
model: "HTTP adapter model-routing field."
moved:
prompt: request.prompt
input_reference: request.inputs.image_path
reference_url: request.inputs.image_path
size:
target: request.sampling.width,height
note: "Adapter parses OpenAI size strings as WIDTHxHEIGHT and forwards width then height."
fps: request.sampling.fps
num_frames: request.sampling.num_frames
seed: request.sampling.seed
num_inference_steps: request.sampling.num_inference_steps
guidance_scale: request.sampling.guidance_scale
guidance_scale_2: request.sampling.guidance_scale_2
true_cfg_scale: request.sampling.true_cfg_scale
negative_prompt: request.negative_prompt
enable_teacache: request.runtime.enable_teacache
output_path: request.output.output_path
compatibility_only:
seconds:
target: request.sampling.num_frames
note: "HTTP adapter duration convenience field. If num_frames is omitted, the adapter computes num_frames = fps * seconds."
cli:
notes:
- "CLI parity is checked against the actual generate/serve parser dest sets."
- "The inventory tracks parser dest names, excluding argparse's implicit help action."
- "The refactored inference CLI is config-only: subcommands expose only --config, and any additional CLI input must use dotted override paths."
generate:
explicit_local_fields:
- config
expected_dests:
- config
serve:
explicit_local_fields:
- config
expected_dests:
- config
+5 -6
View File
@@ -12,7 +12,7 @@ FastVideo maps a Diffusers-style repo into a pipeline like this:
- `fastvideo/configs/models/*`: arch configs and `param_names_mapping` for
weight name translation.
- `fastvideo/configs/pipelines/*`: pipeline wiring (component classes + names).
- `fastvideo/api/sampling_param.py`: runtime sampling parameters.
- `fastvideo/configs/sample/*`: default runtime sampling parameters.
- `fastvideo/pipelines/basic/*`: end-to-end pipelines.
- `fastvideo/pipelines/stages/*`: reusable pipeline stages.
- `fastvideo/models/loader/*`: component loaders for Diffusers-style repos.
@@ -26,7 +26,7 @@ Minimal usage (from `examples/inference/basic/basic.py`):
```python
from fastvideo import VideoGenerator
from fastvideo.api.sampling_param import SamplingParam
from fastvideo.configs.sample import SamplingParam
model_id = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" # or official_weights/<model_name>/
generator = VideoGenerator.from_pretrained(model_id, num_gpus=1)
@@ -49,9 +49,8 @@ runtime parameters consistent:
- `fastvideo/configs/models/`: architecture definitions, layer shapes, and
`param_names_mapping` rules for key renaming.
- `fastvideo/configs/pipelines/`: pipeline wiring and required components.
- `fastvideo/api/sampling_param.py`: sampling parameters (steps, frames,
guidance scale, resolution, fps). Defaults come from profiles in
`fastvideo/pipelines/basic/<family>/profiles.py`.
- `fastvideo/configs/sample/`: default sampling parameters (steps, frames,
guidance scale, resolution, fps).
- `fastvideo/registry.py`: unified registry for pipeline config + sampling
defaults and model metadata resolution, defined via explicit
`register_configs(...)` blocks (no separate dict registries).
@@ -143,7 +142,7 @@ How this maps to FastVideo:
- `T5TokenizerFast` -> loaded via HF in `fastvideo/models/loader/`
- `UniPCMultistepScheduler` -> loaded via Diffusers scheduler utilities
- Pipeline defaults -> `fastvideo/configs/pipelines/wan.py`
- Sampling defaults -> `fastvideo/pipelines/basic/wan/profiles.py`
- Sampling defaults -> `fastvideo/configs/sample/wan.py`
## Pipeline system
-177
View File
@@ -1,177 +0,0 @@
# Streaming WebSocket Server Contract
The streaming server (`fastvideo/entrypoints/streaming/server.py`) speaks
a JSON-over-WebSocket protocol with binary fMP4 chunks for media. This
document is the authoritative spec for the message catalogue and the
session state machine. Any change to either must update this document
in the same PR that touches `protocol.py` or `session.py`.
## Endpoint
| Path | Protocol | Purpose |
|---|---|---|
| `WS /v1/stream` | WebSocket (JSON + binary) | Per-session realtime streaming |
| `GET /health` | HTTP | Liveness probe (`status`, `stream_mode`, active `sessions`) |
The server is launched by `fastvideo serve --config <serve.yaml>` when
the config carries a `streaming:` block. Without that block the same CLI
launches the OpenAI stateless HTTP server instead.
## Connection lifecycle
Every WebSocket connection holds exactly one `Session`. Sessions move
through the states in `SessionState` (`fastvideo/entrypoints/streaming/session.py`).
```
┌──────────────┐
│ INITIALIZING │ ← WebSocket accepted, before init frame
└──────┬───────┘
│ session_init_v2 received
┌──────────────┼──────────────┐
▼ ▼ ▼
QUEUED GPU_BINDING REJECTED
│ │ ↑
│ slot ready │ │ max-sessions hit
▼ ▼ │ or invalid init
┌────────┐ │
│ ACTIVE │ ────────┘
└────┬───┘
segment loop │
│
┌───────────┼───────────┐
▼ ▼ ▼
COMPLETE ERROR TIMEOUT
(clean leave) (any failure) (idle / segment_cap reached)
```
Terminal states (`COMPLETE`, `ERROR`, `TIMEOUT`, `REJECTED`) are sinks —
no transitions out. The transition matrix is enforced in
`session.py::_VALID_TRANSITIONS`; bad transitions raise.
`SessionManager` enforces the per-process budgets pulled from
`StreamingConfig`:
- `session_timeout_seconds` — idle reaper drops sessions that haven't
advanced; non-terminal sessions transition to `TIMEOUT`.
- `generation_segment_cap` — a session that hits the cap transitions to
`COMPLETE` after the last segment ships.
## Message catalogue
Every JSON frame carries `{"type": <str>, ...}`. Pydantic models in
`protocol.py` are the source of truth; this table is the human-readable
view.
### Client → server
| `type` | Required fields | Purpose |
|---|---|---|
| `session_init_v2` | — | Opening frame. Carries preset, curated prompts, optional initial image, feature toggles, optional `continuation_state` to resume from a snapshot. |
| `segment_prompt_source` | `prompt` | Request the next segment using the supplied prompt; optional sampling overrides (`seed`, `num_inference_steps`, `guidance_scale`, `negative_prompt`). |
| `seed_prompts_updated` | `seed_prompts` | Replace the session's seed-prompt list; takes effect on the next segment. |
| `enhancement_updated` | `enabled` | Toggle prompt enhancement for subsequent segments. |
| `auto_extension_updated` | `enabled` | Toggle automatic per-segment prompt extension. |
| `loop_generation_updated` | `enabled` | Toggle loop-generation mode. |
| `generation_paused_updated` | `paused` | Pause/resume segment generation; queued requests defer. |
| `snapshot_state` | — | Request the current `ContinuationState` for export; server replies with `continuation_state_snapshot`. |
The opening frame must be `session_init_v2`. Any other first frame is
rejected with an `error` (code `invalid_message`) and the WebSocket is
closed.
### Server → client
| `type` | Carries | When emitted |
|---|---|---|
| `queue_status` | `position`, `queue_depth` | After `session_init_v2` accepted, before GPU binding. |
| `gpu_assigned` | GPU id, model id | Once a generator slot is bound. |
| `ltx2_stream_start` | session-level metadata | Once the session enters `ACTIVE`. |
| `ltx2_segment_start` | `segment_idx`, `prompt`, prompt source | When a `segment_prompt_source` request begins generation. |
| `step_complete` | `segment_idx`, denoise timings | After the segment's denoising loop finishes (before media emission). |
| `media_init` | `segment_idx`, mime, stream id | First frame of fMP4 output for the segment. |
| binary frame | fMP4 fragment bytes | Subsequent media chunks; the protocol enforces that `media_init` precedes any binary frames. |
| `media_segment_complete` | `segment_idx`, chunk count, byte count | Last media chunk for the segment. |
| `ltx2_segment_complete` | `segment_idx`, segment summary | Segment fully shipped; ready for the next `segment_prompt_source`. |
| `ltx2_stream_complete` | session summary | Session reached `generation_segment_cap` or client requested clean shutdown. |
| `session_timeout` | reason | Session hit `session_timeout_seconds`; immediately followed by close. |
| `continuation_state_snapshot` | `kind`, `payload` | Reply to `snapshot_state`. The payload is the same shape produced by `LTX2ContinuationState.to_continuation_state(...)`. |
| `error` | `code`, `message` | Any validation/runtime error. Non-fatal errors keep the connection open; fatal errors precede a `close`. |
## Continuation state
The session optionally accepts a `continuation_state` dict inside the
opening `session_init_v2` frame. When present, the server hydrates it
into a `ContinuationState(kind, payload)` envelope and feeds it as the
`request.state` on the first segment's `GenerationRequest` — letting a
client resume after a disconnect, migrate sessions across processes,
or replay a prior session.
After every segment, if the runtime returns a fresh state, the server
persists it to the `SessionStore` so a `snapshot_state` request can
export it. The store and serialization contracts live with the model
family (e.g. `fastvideo/pipelines/basic/ltx2/continuation.py` for LTX-2).
## Example flow
```
client server
────── ──────
WS /v1/stream ─────── connect ─────────────────────────►
◄────── (accept)
{"type": "session_init_v2",
"preset": "ltx2_two_stage",
"curated_prompts": ["a fox in snow", "the fox jumps"],
"initial_image": {...},
"stream_mode": "av_fmp4"} ─────────────────────────────►
(validate, queue, bind)
◄──── {"type": "queue_status",
"position": 0, "queue_depth": 0}
◄──── {"type": "gpu_assigned",
"gpu_id": 0, "model_id": "..."}
◄──── {"type": "ltx2_stream_start", ...}
{"type": "segment_prompt_source",
"prompt": "a fox in snow",
"source": "curated"} ───────────────────────────────────►
(run pipeline)
◄──── {"type": "ltx2_segment_start",
"segment_idx": 1, ...}
◄──── {"type": "step_complete",
"segment_idx": 1, "timings": {...}}
◄──── {"type": "media_init",
"segment_idx": 1,
"mime": "video/mp4", ...}
◄──── <binary fMP4 init segment>
◄──── <binary fMP4 fragment>
◄──── <binary fMP4 fragment>
◄──── {"type": "media_segment_complete",
"segment_idx": 1, "chunks": 12}
◄──── {"type": "ltx2_segment_complete",
"segment_idx": 1, ...}
{"type": "segment_prompt_source",
"prompt": "the fox jumps"} ─────────────────────────────►
(segment 2 …)
{"type": "snapshot_state"} ──────────────────────────────►
◄──── {"type": "continuation_state_snapshot",
"kind": "ltx2.v1",
"payload": {"schema_version": 1, ...}}
(close) ──────────────────────────────────────────────────►
(session → COMPLETE)
```
## Backward / forward compatibility
- Adding a new client message: append a Pydantic model to `protocol.py`
with a unique `type`; add the discriminator entry to `ClientMessage`;
add a row to the table above. Old clients that don't send the new
message remain compatible.
- Adding a new server message: emit only when a new feature flag is
enabled (or always emit, since clients ignore unknown types).
- Changing an existing message: bump the `type` (e.g. `session_init_v2`
→ `session_init_v3`) and accept both for one release cycle. Never
silently change field semantics under the same `type`.
+1 -1
View File
@@ -243,7 +243,7 @@ for step in range(start_step, max_steps):
```bash
# Install
uv pip install -e ".[dev]"
uv pip install -e .[dev]
# Run DMD2 distillation on Wan 2.1
torchrun --nproc_per_node=8 -m fastvideo.train.entrypoint.train \
+1 -24
View File
@@ -16,8 +16,7 @@ Both models are trained on **61×448×832** resolution but support generating vi
First install [VSA](../attention/vsa/index.md). Set `MODEL_BASE` to your own model path and run:
```bash
FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN \
fastvideo generate --config scripts/inference/inference_wan_VSA_DMD_1_3B.yaml
bash scripts/inference/v1_inference_wan_dmd.sh
```
## 🗂️ Dataset
@@ -86,25 +85,3 @@ sbatch examples/distill/Wan2.2-TI2V-5B-Diffusers/Data-free/distill_dmd_t2v_5B.sh
- Learning rate: 2e-5
- Training steps: 3000 (~12 hours)
- HSDP shard dim: 1
## 🧭 Note on `real_score_guidance_scale`
The teacher CFG used inside the DMD loss follows the DMD2 reference
implementation and uses the parameterization
```
x = x_cond + w * (x_cond - x_uncond)
```
rather than the Ho & Salimans form `x_uncond + w * (x_cond - x_uncond)`. The
two are mathematically equivalent up to a constant offset:
| `real_score_guidance_scale` (`w`) | Equivalent standard CFG (`w + 1`) | Output |
|-----------------------------------|-----------------------------------|-----------------------|
| `-1` | `0` | unconditional |
| `0` | `1` | conditional |
| `3.5` (default) | `4.5` | strong guidance |
So `real_score_guidance_scale` should be read as the **extra** guidance
strength added on top of the conditional prediction. When porting values
from a paper that uses the Ho & Salimans form, subtract 1.
+4 -4
View File
@@ -27,7 +27,7 @@ uv pip install fastvideo
conda create -n fastvideo python=3.12 -y
conda activate fastvideo
uv pip install fastvideo
pip install fastvideo
```
### From source
@@ -41,11 +41,11 @@ uv pip install -e .
uv pip install flash-attn --no-build-isolation -v
```
Alternative with Conda environment (still drives installs through `uv`):
Alternative with Conda environment:
```bash
uv pip install -e .
uv pip install flash-attn --no-build-isolation -v
pip install -e .
pip install flash-attn --no-build-isolation -v
```
## Hardware Requirements
+4 -6
View File
@@ -58,16 +58,14 @@ uv pip install flash-attn --no-build-isolation -v
#### With Conda environment (alternative)
`uv` works inside an active conda env too, so prefer `uv pip` for the actual install:
```bash
uv pip install fastvideo
pip install fastvideo
```
Also optionally install FlashAttention:
```bash
uv pip install flash-attn --no-build-isolation -v
pip install flash-attn --no-build-isolation -v
```
### Installation from Source
@@ -89,7 +87,7 @@ uv pip install -e .
Alternative with Conda environment:
```bash
uv pip install -e .
pip install -e .
```
### Optional Dependencies
@@ -103,7 +101,7 @@ uv pip install flash-attn --no-build-isolation -v
Alternative with Conda environment:
```bash
uv pip install flash-attn --no-build-isolation -v
pip install flash-attn --no-build-isolation -v
```
## Set up using Docker
+2 -4
View File
@@ -57,10 +57,8 @@ uv pip install fastvideo
#### With Conda environment (alternative)
`uv` works inside an active conda env too, so prefer `uv pip` for the actual install:
```bash
uv pip install fastvideo
pip install fastvideo
```
### Installation from Source
@@ -82,7 +80,7 @@ uv pip install -e .
Alternative with Conda environment:
```bash
uv pip install -e .
pip install -e .
```
## Development Environment Setup
+1 -1
View File
@@ -33,7 +33,7 @@ The following two classes `PipelineConfig` and `SamplingParam` are used to confi
### SamplingParam
::: fastvideo.api.sampling_param.SamplingParam
::: fastvideo.configs.sample.base.SamplingParam
options:
show_root_heading: true
show_source: false
+15 -10
View File
@@ -128,14 +128,19 @@ Concrete hierarchy: `DiTConfig` → `DiTArchConfig`, `VAEConfig` →
- `dump_to_json()` / `load_from_json()` — JSON persistence. Callable
fields and `arch_config` are excluded from dumps.
### SamplingParam (`fastvideo/api/sampling_param.py`)
### SamplingParam (`fastvideo/configs/sample/`)
Generation parameters separate from pipeline config. Each model family
provides defaults via a profile (see `fastvideo/pipelines/basic/<family>/profiles.py`):
provides defaults:
```python
sp = SamplingParam.from_pretrained("Wan-AI/Wan2.1-T2V-1.3B-Diffusers")
# sp.height == 480, sp.width == 832, sp.num_frames == 81, etc.
@dataclass
class WanT2V_1_3B_SamplingParam(SamplingParam):
height: int = 480
width: int = 832
num_frames: int = 81
guidance_scale: float = 3.0
num_inference_steps: int = 50
```
## Component Loading
@@ -425,9 +430,9 @@ User: generator.generate_video(prompt, ...)
`fastvideo/configs/pipelines/<model>.py`. Set DiT/VAE/encoder configs,
flow_shift, precision defaults.
2. **Sampling param profile** — Create a profile in
`fastvideo/pipelines/basic/<model>/profiles.py` with default height,
width, num_frames, guidance_scale, num_inference_steps.
2. **Sampling param** — Create a `SamplingParam` subclass in
`fastvideo/configs/sample/<model>.py`. Set default height, width,
num_frames, guidance_scale, num_inference_steps.
3. **Register configs** — In `fastvideo/registry.py`, add a
`register_configs()` call inside `_register_configs()` with
@@ -450,6 +455,6 @@ User: generator.generate_video(prompt, ...)
`fastvideo/pipelines/stages/`, implement `forward()`, optionally
implement `verify_input()`/`verify_output()`.
7. **Verify** — Run `fastvideo generate --config <config.yaml>` with a
minimal nested config to confirm the pipeline loads and generates
output.
7. **Verify** — Run `fastvideo generate --model-path <path> --prompt
"test" --num-inference-steps 2` to confirm the pipeline loads and
generates output.
+81 -42
View File
@@ -1,29 +1,71 @@
# FastVideo CLI Inference
The FastVideo CLI is config-first. Inference runs are driven by a nested JSON or
YAML config, with optional dotted-path overrides on the command line. The
contract matches training: use an explicit subcommand plus `--config`, then add
any dotted overrides you need.
The FastVideo CLI exposes the same core inference controls as the Python API.
## Basic Usage
Use either:
1. `--model-path` + `--prompt`
2. `--model-path` + `--prompt-txt` (batch prompts, one line per prompt)
3. `--config` (JSON/YAML)
```bash
fastvideo generate --config config.yaml
fastvideo serve --config serve.yaml
fastvideo generate --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \
--prompt "A cat playing with a ball of yarn"
```
```bash
fastvideo generate --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \
--prompt-txt prompts.txt
```
You cannot provide both `--prompt` and `--prompt-txt` in the same run.
## View All Arguments
```bash
fastvideo generate --help
```
The subcommands intentionally expose only `--config`. Any per-run CLI changes
must use dotted override paths such as:
Arguments come from:
- `--generator.engine.num_gpus 2`
- `--request.sampling.seed 42`
- `--server.port 9000`
- FastVideo runtime args (`FastVideoArgs`)
- Sampling args (`SamplingParam`)
- Pipeline config args (`PipelineConfig`)
## Common Arguments
### Parallelism
- `--num-gpus`
- `--sp-size`
- `--tp-size`
### Sampling
- `--num-frames`
- `--height` / `--width`
- `--num-inference-steps`
- `--guidance-scale`
- `--seed`
- `--negative-prompt`
### Output
- `--output-path`
- `--save-video` / `--no-save-video`
- `--return-frames`
### Offloading and Performance
- `--dit-layerwise-offload`
- `--use-fsdp-inference`
- `--text-encoder-cpu-offload`
- `--image-encoder-cpu-offload`
- `--vae-cpu-offload`
- `--enable-torch-compile`
- `--torch-compile-kwargs`
## Using Config Files
@@ -31,53 +73,50 @@ must use dotted override paths such as:
fastvideo generate --config config.yaml
```
Config files can be JSON or YAML. Dotted CLI overrides take precedence over
config-file values.
Config files can be JSON or YAML. CLI flags override config-file values.
Example `config.yaml`:
```yaml
generator:
model_path: FastVideo/FastHunyuan-diffusers
engine:
num_gpus: 2
parallelism:
sp_size: 2
tp_size: 1
request:
prompt: A capybara lounging in a hammock
sampling:
num_frames: 45
height: 720
width: 1280
num_inference_steps: 6
seed: 1024
output:
output_path: outputs/
model_path: "FastVideo/FastHunyuan-diffusers"
prompt: "A capybara lounging in a hammock"
output_path: "outputs/"
num_gpus: 2
sp_size: 2
tp_size: 1
num_frames: 45
height: 720
width: 1280
num_inference_steps: 6
seed: 1024
dit_precision: "bf16"
vae_precision: "fp16"
vae_tiling: true
vae_sp: true
enable_torch_compile: false
```
Notes:
- `generator` and `request` are the top-level keys for generation configs.
- `serve` configs use `generator`, `server`, and optional `default_request`.
- Prompt text files belong under `request.inputs.prompt_path`.
- Use `dit_precision` / `vae_precision` (not `precision`).
- Nested config objects are supported, for example `vae_config` and
`dit_config`.
## Examples
Simple generation:
```bash
fastvideo generate --config config.yaml
fastvideo generate \
--model-path FastVideo/FastHunyuan-diffusers \
--prompt "A cat playing with a ball of yarn" \
--num-frames 45 --height 720 --width 1280 \
--num-inference-steps 6 --seed 1024 \
--output-path outputs/
```
Config + dotted override:
Config + CLI override:
```bash
fastvideo generate --config config.yaml --request.prompt "A panda skiing at sunset"
```
Helper wrapper with positional config path:
```bash
bash scripts/inference/run.sh scripts/inference/inference_wan.yaml
fastvideo generate --config config.yaml --prompt "A panda skiing at sunset"
```
+19 -27
View File
@@ -73,40 +73,32 @@ if __name__ == '__main__':
## JSON/YAML Config Files (CLI)
The inference CLI is config-first. Use an explicit subcommand with `--config`,
then apply optional dotted overrides on top, matching the training CLI style.
By default, CLI generation uses `return_frames=false` unless you set
`request.output.return_frames: true` in config or via a dotted override.
The CLI supports `--config` with JSON or YAML. Command-line arguments override
config file values.
By default, `fastvideo generate` uses `return_frames=false` unless you set
`--return-frames` (or `return_frames: true` in config).
```bash
fastvideo generate --config config.yaml
```
Example nested config:
Use CLI argument names as keys (underscore or hyphen is accepted). Example:
```yaml
generator:
model_path: FastVideo/FastHunyuan-diffusers
engine:
num_gpus: 2
parallelism:
sp_size: 2
request:
prompt: A capybara relaxing in a hammock
sampling:
num_frames: 45
height: 720
width: 1280
num_inference_steps: 6
seed: 1024
output:
output_path: outputs/
```
Override individual values from the CLI with dotted paths:
```bash
fastvideo generate --config config.yaml --request.sampling.seed 42
model_path: "FastVideo/FastHunyuan-diffusers"
prompt: "A capybara relaxing in a hammock"
num_gpus: 2
sp_size: 2
num_frames: 45
height: 720
width: 1280
num_inference_steps: 6
seed: 1024
dit_precision: "bf16"
vae_precision: "fp16"
vae_tiling: true
vae_sp: true
enable_torch_compile: false
```
## Performance Optimization
-129
View File
@@ -1,129 +0,0 @@
# GEN3C: 3D-Informed Camera-Controlled Video Generation
[GEN3C](https://arxiv.org/abs/2503.03751) is NVIDIA's Cosmos-7B-based video model for camera-controlled generation from a single image. The FastVideo integration supports the GEN3C I2V workflow, including 3D cache conditioning and tokenizer-based conditioning latents.
## Key Features
- **Camera trajectory control**: `left/right/up/down/zoom_in/zoom_out/clockwise/counterclockwise`
- **3D cache conditioning**: depth prediction -> point cloud cache -> forward warping -> latent conditioning
- **Single-image to video generation**: 121-frame generation with camera motion
- **Official raw checkpoint conversion**: `model.pt` -> Diffusers/FastVideo layout
## Model Sources
- Official raw checkpoint (not Diffusers): `nvidia/GEN3C-Cosmos-7B`
- Diffusers-format checkpoint: `FastVideo/GEN3C-Cosmos-7B-Diffusers`
## Prerequisites
- Install MoGe:
```bash
uv pip install git+https://github.com/microsoft/MoGe.git
```
- If you hit `ImportError: libGL.so.1` (common on Ubuntu/headless nodes), you can try installing OpenCV runtime libs:
```bash
sudo apt-get update
sudo apt-get install -y libgl1 libglib2.0-0 libsm6 libxext6 libxrender1
```
## Quick Start
### Option A: Use Diffusers-format weights directly
```bash
python examples/inference/basic/basic_gen3c.py \
--model_path FastVideo/GEN3C-Cosmos-7B-Diffusers \
--image_path /path/to/input.png \
--prompt "" \
--trajectory left \
--movement_distance 0.3 \
--camera_rotation center_facing \
--num_inference_steps 35 \
--guidance_scale 1.0 \
--output_path outputs_video/gen3c_output.mp4
```
### Option B: Convert official raw checkpoint locally
1. Download:
```bash
huggingface-cli download nvidia/GEN3C-Cosmos-7B --local-dir official_weights/GEN3C-Cosmos-7B
```
1. Convert:
```bash
python scripts/checkpoint_conversion/convert_gen3c_to_fastvideo.py \
--source official_weights/GEN3C-Cosmos-7B/model.pt \
--output converted_weights/GEN3C-Cosmos-7B
```
1. Run:
```bash
python examples/inference/basic/basic_gen3c.py \
--model_path converted_weights/GEN3C-Cosmos-7B \
--image_path /path/to/input.png \
--prompt "" \
--trajectory left \
--movement_distance 0.3 \
--camera_rotation center_facing \
--num_inference_steps 35 \
--guidance_scale 1.0 \
--output_path outputs_video/gen3c_output.mp4
```
## FastVideo Defaults
GEN3C defaults in FastVideo:
- `height=704`, `width=1280`
- `num_frames=121`
- `num_inference_steps=35`
- `guidance_scale=1.0`
- `fps=24`
These values are defined in:
- `fastvideo/pipelines/basic/gen3c/profiles.py`
- `fastvideo/configs/pipelines/gen3c.py`
and align with the official GEN3C inference defaults in:
- `tmp/GEN3C/cosmos_predict1/diffusion/inference/inference_utils.py`
## Scheduler Note
The converted GEN3C Diffusers layout may include a FlowMatch scheduler config, but GEN3C denoising uses EDM preconditioning behavior. FastVideo's GEN3C pipeline enforces an EDM scheduler at runtime for parity with official inference behavior.
Implementation path:
- `fastvideo/pipelines/basic/gen3c/gen3c_pipeline.py`
## 3D Cache Conditioning Path
FastVideo GEN3C conditioning stage performs:
1. MoGe depth estimation from input image
2. 3D cache initialization
3. Camera trajectory generation
4. Forward rendering of warped frames + masks
5. VAE/tokenizer encoding of conditioning buffers
6. Denoising with condition mask + condition pose channels
Main implementation:
- `fastvideo/pipelines/basic/gen3c/gen3c_pipeline.py`
- `fastvideo/pipelines/basic/gen3c/cache_3d.py`
- `fastvideo/pipelines/basic/gen3c/depth_estimation.py`
- `fastvideo/models/vaes/gen3c_tokenizer_vae.py`
## References
- [GEN3C Paper](https://arxiv.org/abs/2503.03751)
- [Official Repository](https://github.com/nv-tlabs/GEN3C)
- [Official Checkpoint (raw)](https://huggingface.co/nvidia/GEN3C-Cosmos-7B)
+3 -3
View File
@@ -54,7 +54,7 @@ FASTVIDEO_ATTENTION_BACKEND=SAGE_ATTN python example.py
We recommend always installing [Flash Attention 2](https://github.com/Dao-AILab/flash-attention):
```bash
uv pip install flash-attn==2.7.4.post1 --no-build-isolation
pip install flash-attn==2.7.4.post1 --no-build-isolation
```
And if using a Hopper+ GPU (ie H100), installing [Flash Attention 3](https://github.com/Dao-AILab/flash-attention?tab=readme-ov-file#flashattention-3-beta-release) by compiling it from source (takes about 10 minutes for me):
@@ -63,7 +63,7 @@ And if using a Hopper+ GPU (ie H100), installing [Flash Attention 3](https://git
git clone https://github.com/Dao-AILab/flash-attention.git && cd flash-attention
cd hopper
uv pip install ninja
pip install ninja
python setup.py install
```
@@ -98,7 +98,7 @@ To use [SageAttention](https://github.com/thu-ml/SageAttention) 2.1.1, please co
```bash
git clone https://github.com/thu-ml/SageAttention.git
cd sageattention
python setup.py install # or uv pip install -e .
python setup.py install # or pip install -e .
```
### Sage Attention 3
-6
View File
@@ -73,7 +73,6 @@ pipeline initialization and sampling.
| Matrix Game 2.0 Base | `FastVideo/Matrix-Game-2.0-Base-Diffusers` | 352x640 | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ |
| Matrix Game 2.0 GTA | `FastVideo/Matrix-Game-2.0-GTA-Diffusers` | 352x640 | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ |
| Matrix Game 2.0 TempleRun | `FastVideo/Matrix-Game-2.0-TempleRun-Diffusers` | 352x640 | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ |
| GEN3C Cosmos 7B | `FastVideo/GEN3C-Cosmos-7B-Diffusers` | 704px1280p | ❌ | ❌ | ❌ | ⭕ | ⭕ |
**Note**: Wan2.2 TI2V 5B has some quality issues when performing I2V generation. We are working on fixing this issue.
@@ -86,11 +85,6 @@ The authoritative source for model-ID recognition is
`fastvideo/registry.py`. If a model ID is registered there, FastVideo can
resolve default pipeline and sampling configuration for it.
**Note (GEN3C)**: The official `nvidia/GEN3C-Cosmos-7B` repo provides a raw
`model.pt` checkpoint. Use a Diffusers-format repo (for example,
`FastVideo/GEN3C-Cosmos-7B-Diffusers`) or convert locally with
`scripts/checkpoint_conversion/convert_gen3c_to_fastvideo.py`.
## Special requirements
### Sliding Tile Attention
@@ -4,7 +4,7 @@ These are end-to-end example scripts for distilling Wan2.1 T2V 1.3B model using
### 0. Make sure you have installed VSA
```bash
uv pip install vsa
pip install vsa
```
### 1. Download dataset:
@@ -4,7 +4,7 @@ These are end-to-end example scripts for distilling Wan2.2 TI2V 5B model DMD+VSA
### 0. Make sure you have installed VSA
```bash
uv pip install vsa
pip install vsa
```
### Data-free Distillation
@@ -4,7 +4,7 @@ These are end-to-end example scripts for distilling Wan2.2 TI2V 5B model DMD+VSA
### 0. Make sure you have installed VSA
```bash
uv pip install vsa
pip install vsa
```
### 1. Download dataset:
-5
View File
@@ -28,11 +28,6 @@ For an example running DMD+VSA inference:
python examples/inference/basic/basic_dmd.py
```
For the typed config/request path added during the inference API refactor:
```
python examples/inference/basic/basic_dmd_new_api.py
```
## Basic Walkthrough
All you need to generate videos using multi-gpus from state-of-the-art diffusion pipelines is the following few lines!
+1 -1
View File
@@ -1,6 +1,6 @@
from fastvideo import VideoGenerator
# from fastvideo.api.sampling_param import SamplingParam
# from fastvideo.configs.sample import SamplingParam
OUTPUT_PATH = "video_samples"
def main():
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
from fastvideo import VideoGenerator
from fastvideo.api.sampling_param import SamplingParam
from fastvideo.configs.sample import SamplingParam
def main():
@@ -1,5 +1,5 @@
from fastvideo import VideoGenerator
from fastvideo.api.sampling_param import SamplingParam
from fastvideo.configs.sample import SamplingParam
def main():
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
from fastvideo import VideoGenerator
from fastvideo.api.sampling_param import SamplingParam
from fastvideo.configs.sample import SamplingParam
def main():
+1 -1
View File
@@ -2,7 +2,7 @@ import os
import time
from fastvideo import VideoGenerator
from fastvideo.api.sampling_param import SamplingParam
from fastvideo.configs.sample import SamplingParam
OUTPUT_PATH = "video_samples_dmd2"
def main():
@@ -1,98 +0,0 @@
import os
import time
from fastvideo import VideoGenerator
from fastvideo.api import (
EngineConfig,
GenerationRequest,
GeneratorConfig,
OffloadConfig,
OutputConfig,
PipelineSelection,
)
OUTPUT_PATH = "video_samples_dmd2_typed"
def main():
os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "VIDEO_SPARSE_ATTN"
model_name = "FastVideo/FastWan2.1-T2V-1.3B-Diffusers"
generator_config = GeneratorConfig(
model_path=model_name,
engine=EngineConfig(
num_gpus=1,
use_fsdp_inference=False,
offload=OffloadConfig(
text_encoder=True,
pin_cpu_memory=True,
dit=False,
vae=False,
),
),
# PR 2 still routes a few advanced inference knobs through the
# compatibility bridge until they get first-class typed fields.
pipeline=PipelineSelection(
experimental={
"VSA_sparsity": 0.8,
},
),
)
load_start_time = time.perf_counter()
generator = VideoGenerator.from_config(generator_config)
load_end_time = time.perf_counter()
load_time = load_end_time - load_start_time
prompt = (
"A neon-lit alley in futuristic Tokyo during a heavy rainstorm at night. "
"The puddles reflect glowing signs in kanji, advertising ramen, karaoke, "
"and VR arcades. A woman in a translucent raincoat walks briskly with an "
"LED umbrella. Steam rises from a street food cart, and a cat darts "
"across the screen. Raindrops are visible on the camera lens, creating "
"a cinematic bokeh effect."
)
request = GenerationRequest(
prompt=prompt,
output=OutputConfig(
output_path=OUTPUT_PATH,
save_video=True,
return_frames=False,
),
)
start_time = time.perf_counter()
result = generator.generate(request)
end_time = time.perf_counter()
gen_time = end_time - start_time
prompt2 = (
"A majestic lion strides across the golden savanna, its powerful frame "
"glistening under the warm afternoon sun. The tall grass ripples gently "
"in the breeze, enhancing the lion's commanding presence. The tone is "
"vibrant, embodying the raw energy of the wild. Low angle, steady "
"tracking shot, cinematic."
)
request2 = GenerationRequest(
prompt=prompt2,
output=OutputConfig(
output_path=OUTPUT_PATH,
save_video=True,
return_frames=False,
),
)
start_time = time.perf_counter()
result2 = generator.generate(request2)
end_time = time.perf_counter()
gen_time2 = end_time - start_time
print(f"Time taken to load model: {load_time} seconds")
print(f"Time taken to generate video: {gen_time} seconds")
print(f"First output written to: {result.video_path}")
print(f"Time taken to generate video2: {gen_time2} seconds")
print(f"Second output written to: {result2.video_path}")
if __name__ == "__main__":
main()
-109
View File
@@ -1,109 +0,0 @@
"""
GEN3C: 3D-aware camera-controlled video generation.
This example generates a video from a single input image with camera control.
The pipeline uses MoGe depth estimation, 3D point cloud forward warping,
and the GEN3C diffusion model.
Requirements:
1. Install MoGe:
uv pip install git+https://github.com/microsoft/MoGe.git
If you hit `ImportError: libGL.so.1`, install:
sudo apt-get update && sudo apt-get install -y libgl1 libglib2.0-0 libsm6 libxext6 libxrender1
2. Download and convert weights:
huggingface-cli download nvidia/GEN3C-Cosmos-7B --local-dir official_weights/GEN3C-Cosmos-7B
python scripts/checkpoint_conversion/convert_gen3c_to_fastvideo.py \
--source ./official_weights/GEN3C-Cosmos-7B/model.pt \
--output ./converted_weights/GEN3C-Cosmos-7B \
--components-source nvidia/Cosmos-Predict2-2B-Video2World
3. Provide an input image for 3D-conditioned generation.
"""
import argparse
from fastvideo import VideoGenerator
def main():
parser = argparse.ArgumentParser(description="GEN3C video generation")
parser.add_argument("--model_path",
type=str,
default="converted_weights/GEN3C-Cosmos-7B")
parser.add_argument("--image_path",
type=str,
default=None,
help="Input image for 3D cache conditioning")
parser.add_argument("--prompt",
type=str,
default="A slow camera pan over a sunlit landscape.")
parser.add_argument(
"--negative_prompt",
type=str,
default=(
"The video captures a series of frames showing ugly scenes, static with no motion, motion blur, "
"over-saturation, shaky footage, low resolution, grainy texture, pixelated images, poorly lit areas, "
"underexposed and overexposed scenes, poor color balance, washed out colors, choppy sequences, "
"jerky movements, low frame rate, artifacting, color banding, unnatural transitions, outdated special "
"effects, fake elements, unconvincing visuals, poorly edited content, jump cuts, visual noise, and "
"flickering. Overall, the video is of poor quality."
),
)
parser.add_argument("--trajectory",
type=str,
default="left",
choices=[
"left", "right", "up", "down", "zoom_in",
"zoom_out", "clockwise", "counterclockwise", "none"
])
parser.add_argument("--movement_distance", type=float, default=0.3)
parser.add_argument("--camera_rotation",
type=str,
default="center_facing",
choices=[
"center_facing", "no_rotation",
"trajectory_aligned"
])
parser.add_argument("--height", type=int, default=704)
parser.add_argument("--width", type=int, default=1280)
parser.add_argument("--num_frames", type=int, default=121)
parser.add_argument("--num_inference_steps", type=int, default=35)
parser.add_argument("--guidance_scale", type=float, default=1.0)
parser.add_argument("--output_path",
type=str,
default="outputs_video/gen3c.mp4")
parser.add_argument("--seed", type=int, default=42)
args = parser.parse_args()
generator = VideoGenerator.from_pretrained(
args.model_path,
num_gpus=1,
use_fsdp_inference=False,
dit_cpu_offload=False,
vae_cpu_offload=True,
text_encoder_cpu_offload=True,
pin_cpu_memory=True,
)
video = generator.generate_video(
args.prompt,
negative_prompt=args.negative_prompt,
image_path=args.image_path,
trajectory_type=args.trajectory,
movement_distance=args.movement_distance,
camera_rotation=args.camera_rotation,
height=args.height,
width=args.width,
num_frames=args.num_frames,
num_inference_steps=args.num_inference_steps,
guidance_scale=args.guidance_scale,
fps=24,
seed=args.seed,
output_path=args.output_path,
save_video=True,
)
generator.shutdown()
if __name__ == "__main__":
main()
+1 -1
View File
@@ -1,6 +1,6 @@
from fastvideo import VideoGenerator
import json
# from fastvideo.api.sampling_param import SamplingParam
# from fastvideo.configs.sample import SamplingParam
OUTPUT_PATH = "video_samples_hy15"
def main():
+1 -1
View File
@@ -1,6 +1,6 @@
from fastvideo import VideoGenerator
import json
# from fastvideo.api.sampling_param import SamplingParam
# from fastvideo.configs.sample import SamplingParam
OUTPUT_PATH = "video_samples_hy15_1080p"
def main():
@@ -1,7 +1,7 @@
from fastvideo import VideoGenerator
from fastvideo.models.dits.lingbotworld.cam_utils import prepare_camera_embedding
# from fastvideo.api.sampling_param import SamplingParam
# from fastvideo.configs.sample import SamplingParam
OUTPUT_PATH = "video_samples_lingbotworld"
def main():
# FastVideo will automatically use the optimal default arguments for the
@@ -1,51 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Minimal user-runnable example for the daVinci-MagiHuman base AV pipeline.
Produces an mp4 with both video (Wan 2.2 TI2V-5B VAE) and audio (Stable
Audio Open 1.0 VAE, first-class FastVideo port in
`fastvideo/models/vaes/oobleck.py`) muxed together via PyAV.
Prerequisites (one-off):
# Accept terms of use on the gated HF repos with your HF_TOKEN:
# - https://huggingface.co/google/t5gemma-9b-9b-ul2
# - https://huggingface.co/stabilityai/stable-audio-open-1.0
# All four cross-variant shared components (Wan 2.2 VAE, T5-Gemma
# encoder + tokenizer, Stable Audio VAE) are lazy-loaded from their
# canonical upstream HF repos on first build, so a single ~25 GB
# cache is shared across every MagiHuman variant.
The umbrella HF repo `FastVideo/MagiHuman-Diffusers` holds all four
variants (base / distill / sr_540p / sr_1080p) under sibling subfolders
and FastVideo will download just the requested subfolder. Local
conversion via `scripts/checkpoint_conversion/convert_magi_human_to_diffusers.py`
is also supported.
"""
from fastvideo import VideoGenerator
PROMPT = (
"A warm afternoon scene: a person sits on a park bench reading a book, "
"surrounded by softly swaying trees."
)
def main() -> None:
generator = VideoGenerator.from_pretrained(
"FastVideo/MagiHuman-Diffusers/base",
num_gpus=1,
)
output_path = "outputs_video/magi_human_basic/output_magi_human.mp4"
generator.generate_video(
prompt=PROMPT,
output_path=output_path,
save_video=True,
# Defaults pulled from the registered preset (magi_human_base):
# height=256, width=448, fps=25, num_inference_steps=32, seed=42.
# Override here only if you have a specific QA scenario.
)
generator.shutdown()
if __name__ == "__main__":
main()
@@ -1,53 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Minimal user-runnable example for the daVinci-MagiHuman DMD-2 distilled
text-to-AV pipeline.
Same arch as the base model (`basic_magi_human.py`) but with DMD-2 distilled
weights: 8 denoising steps, no classifier-free guidance. ~4x faster than
base at the same 256x480 resolution. Mirrors upstream
`daVinci-MagiHuman/example/distill/run_T2V.sh`.
Prerequisites (one-off):
# 1) Accept terms on the gated HF repos with your HF_TOKEN:
# - https://huggingface.co/google/t5gemma-9b-9b-ul2
# - https://huggingface.co/stabilityai/stable-audio-open-1.0
# Cross-variant shared components (Wan 2.2 VAE + T5-Gemma + Stable
# Audio VAE) are lazy-loaded from their canonical upstream HF repos
# and shared with the base variant cache.
# 2) Convert the distill subfolder of GAIR/daVinci-MagiHuman:
python scripts/checkpoint_conversion/convert_magi_human_to_diffusers.py \\
--source GAIR/daVinci-MagiHuman \\
--subfolder distill \\
--output converted_weights/magi_human_distill \\
--cast-bf16
# `--cast-bf16` is recommended (61 GB fp32 -> 30 GB bf16); the FV pipeline
# loads bf16 anyway, and the conversion keeps norms / RoPE bands fp32.
"""
from fastvideo import VideoGenerator
PROMPT = (
"A warm afternoon scene: a person sits on a park bench reading a book, "
"surrounded by softly swaying trees."
)
def main() -> None:
generator = VideoGenerator.from_pretrained(
"FastVideo/MagiHuman-Diffusers/distill",
num_gpus=1,
)
output_path = "outputs_video/magi_human_basic/output_magi_human_distill.mp4"
generator.generate_video(
prompt=PROMPT,
output_path=output_path,
save_video=True,
# Defaults pulled from the registered preset (magi_human_distill):
# height=256, width=480, fps=25, num_inference_steps=8, cfg=1, seed=42.
)
generator.shutdown()
if __name__ == "__main__":
main()
@@ -1,34 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Minimal daVinci-MagiHuman DMD-2 distilled text+image-to-AV example."""
from fastvideo import VideoGenerator
from fastvideo.pipelines.basic.magi_human.pipeline_configs import (
MagiHumanDistillI2VConfig,
)
PROMPT = (
"A cheerful saxophonist performs a short line with expressive facial "
"motion, natural head movement, and synchronized audio in a small jazz club."
)
IMAGE_PATH = "assets/images/saxophonist.jpg"
def main() -> None:
generator = VideoGenerator.from_pretrained(
"FastVideo/MagiHuman-Diffusers/distill",
num_gpus=1,
workload_type="i2v",
override_pipeline_cls_name="MagiHumanI2VPipeline",
pipeline_config=MagiHumanDistillI2VConfig(),
)
generator.generate_video(
prompt=PROMPT,
image_path=IMAGE_PATH,
output_path="outputs_video/magi_human_distill_ti2v/output_magi_human_distill_ti2v.mp4",
save_video=True,
)
generator.shutdown()
if __name__ == "__main__":
main()
@@ -1,45 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Run daVinci-MagiHuman SR-1080p text-to-AV in FastVideo.
Build the converted repo on large local storage, then symlink it into the
workspace:
python scripts/checkpoint_conversion/convert_magi_human_to_diffusers.py \
--source GAIR/daVinci-MagiHuman \
--subfolder base \
--sr-source GAIR/daVinci-MagiHuman \
--sr-subfolder 1080p_sr \
--output /raid/william5lin_converted_weights/magi_human_sr_1080p \
--cast-bf16
ln -s /raid/william5lin_converted_weights/magi_human_sr_1080p \
converted_weights/magi_human_sr_1080p
"""
from fastvideo import VideoGenerator
from fastvideo.pipelines.basic.magi_human.pipeline_configs import (
MagiHumanSR1080pConfig,
)
PROMPT = (
"A warm afternoon scene: a person sits on a park bench reading a book, "
"surrounded by softly swaying trees."
)
def main() -> None:
generator = VideoGenerator.from_pretrained(
"FastVideo/MagiHuman-Diffusers/sr_1080p",
num_gpus=1,
override_pipeline_cls_name="MagiHumanSR1080pPipeline",
pipeline_config=MagiHumanSR1080pConfig(),
)
generator.generate_video(
prompt=PROMPT,
output_path="outputs_video/magi_human_sr1080p/output_magi_human_sr1080p.mp4",
save_video=True,
)
generator.shutdown()
if __name__ == "__main__":
main()
@@ -1,34 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Run daVinci-MagiHuman SR-1080p text+image-to-AV in FastVideo."""
from fastvideo import VideoGenerator
from fastvideo.pipelines.basic.magi_human.pipeline_configs import (
MagiHumanSR1080pI2VConfig,
)
PROMPT = (
"A cheerful saxophonist performs a short line with expressive facial "
"motion, natural head movement, and synchronized audio in a small jazz club."
)
IMAGE_PATH = "assets/images/saxophonist.jpg"
def main() -> None:
generator = VideoGenerator.from_pretrained(
"FastVideo/MagiHuman-Diffusers/sr_1080p",
num_gpus=1,
workload_type="i2v",
override_pipeline_cls_name="MagiHumanSR1080pI2VPipeline",
pipeline_config=MagiHumanSR1080pI2VConfig(),
)
generator.generate_video(
prompt=PROMPT,
image_path=IMAGE_PATH,
output_path="outputs_video/magi_human_sr1080p_ti2v/output_magi_human_sr1080p_ti2v.mp4",
save_video=True,
)
generator.shutdown()
if __name__ == "__main__":
main()
@@ -1,37 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Run daVinci-MagiHuman SR-540p text-to-AV in FastVideo.
The converted repo must contain both ``transformer/`` (base DiT) and
``sr_transformer/`` (540p SR DiT). Build it with:
python scripts/checkpoint_conversion/convert_magi_human_to_diffusers.py \
--source GAIR/daVinci-MagiHuman \
--subfolder base \
--sr-source GAIR/daVinci-MagiHuman \
--sr-subfolder 540p_sr \
--output converted_weights/magi_human_sr_540p
"""
from fastvideo import VideoGenerator
PROMPT = (
"A warm afternoon scene: a person sits on a park bench reading a book, "
"surrounded by softly swaying trees."
)
def main() -> None:
generator = VideoGenerator.from_pretrained(
"FastVideo/MagiHuman-Diffusers/sr_540p",
num_gpus=1,
)
generator.generate_video(
prompt=PROMPT,
output_path="outputs_video/magi_human_sr540p/output_magi_human_sr540p.mp4",
save_video=True,
)
generator.shutdown()
if __name__ == "__main__":
main()
@@ -1,34 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Run daVinci-MagiHuman SR-540p text+image-to-AV in FastVideo."""
from fastvideo import VideoGenerator
from fastvideo.pipelines.basic.magi_human.pipeline_configs import (
MagiHumanSR540pI2VConfig,
)
PROMPT = (
"A cheerful saxophonist performs a short line with expressive facial "
"motion, natural head movement, and synchronized audio in a small jazz club."
)
IMAGE_PATH = "assets/images/saxophonist.jpg"
def main() -> None:
generator = VideoGenerator.from_pretrained(
"FastVideo/MagiHuman-Diffusers/sr_540p",
num_gpus=1,
workload_type="i2v",
override_pipeline_cls_name="MagiHumanSRI2VPipeline",
pipeline_config=MagiHumanSR540pI2VConfig(),
)
generator.generate_video(
prompt=PROMPT,
image_path=IMAGE_PATH,
output_path="outputs_video/magi_human_sr540p_ti2v/output_magi_human_sr540p_ti2v.mp4",
save_video=True,
)
generator.shutdown()
if __name__ == "__main__":
main()
@@ -1,34 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Minimal daVinci-MagiHuman base text+image-to-AV example."""
from fastvideo import VideoGenerator
from fastvideo.pipelines.basic.magi_human.pipeline_configs import (
MagiHumanBaseI2VConfig,
)
PROMPT = (
"A cheerful saxophonist performs a short line with expressive facial "
"motion, natural head movement, and synchronized audio in a small jazz club."
)
IMAGE_PATH = "assets/images/saxophonist.jpg"
def main() -> None:
generator = VideoGenerator.from_pretrained(
"FastVideo/MagiHuman-Diffusers/base",
num_gpus=1,
workload_type="i2v",
override_pipeline_cls_name="MagiHumanI2VPipeline",
pipeline_config=MagiHumanBaseI2VConfig(),
)
generator.generate_video(
prompt=PROMPT,
image_path=IMAGE_PATH,
output_path="outputs_video/magi_human_ti2v/output_magi_human_ti2v.mp4",
save_video=True,
)
generator.shutdown()
if __name__ == "__main__":
main()
+1 -1
View File
@@ -1,5 +1,5 @@
from fastvideo import VideoGenerator, PipelineConfig
from fastvideo.api.sampling_param import SamplingParam
from fastvideo.configs.sample import SamplingParam
def main():
config = PipelineConfig.from_pretrained("Wan-AI/Wan2.1-T2V-1.3B-Diffusers")
+1 -1
View File
@@ -1,6 +1,6 @@
from fastvideo import VideoGenerator
# from fastvideo.api.sampling_param import SamplingParam
# from fastvideo.configs.sample import SamplingParam
OUTPUT_PATH = "video_samples"
def main():
@@ -2,7 +2,7 @@
from fastvideo import VideoGenerator, SamplingParam
import json
# from fastvideo.api.sampling_param import SamplingParam
# from fastvideo.configs.sample import SamplingParam
OUTPUT_PATH = "video_samples_self_forcing_causal_wan2_2_14B_i2v"
def main():
@@ -2,7 +2,7 @@
from fastvideo import VideoGenerator
# from fastvideo.api.sampling_param import SamplingParam
# from fastvideo.configs.sample import SamplingParam
OUTPUT_PATH = "video_samples_self_forcing_causal_wan2_2_14B_t2v"
def main():
@@ -1,77 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Stable Audio Open 1.0 — text-to-audio (baseline) example.
User story (game-audio designer, prototyping):
"I'm prototyping a level and I need 6 seconds of background
ambience — gentle wind, distant thunder, a hint of birdsong. I
don't want to dig through a sound library; I want to type what I
hear in my head and get a wav back. If it's wrong I'll iterate
on the prompt. This is the first stop."
User story (musician sketching ideas):
"I want to bounce a 30s lo-fi drum loop to use as a placeholder
bed while I build the rest of the track. Type prompt, get audio,
drop into the DAW. The actual production beat I'll record
myself, but I need *something* to write the chords against."
User story (researcher exploring the model):
"First time touching Stable Audio Open — what does it sound
like at default settings? This is the smallest amount of code
that goes from prompt to mp4."
How it works:
Pure text-to-audio (T2A). The pipeline runs:
T5 + NumberConditioner -> StableAudioDiT -> Oobleck VAE
via the `dpmpp-3m-sde` k-diffusion sampler. All components are
FastVideo-native — no diffusers / transformers model imports at
runtime (see REVIEW item 30). Mirrors upstream
`stable_audio_tools.inference.generation.generate_diffusion_cond`
bit-for-bit (~0.2% abs_mean drift on 25 steps).
Tunable knobs (the "creative dials"):
audio_end_in_s
1–6 — quick ideation (sub-10s wall clock at 100 steps)
10–30 — full musical phrase / loop length (the README example
uses 30s)
47.5 — model maximum (full sample_size = 2097152 / 44100 Hz)
num_inference_steps
25 — fast preview, occasional artifacts
100 — preset default (matches the HF model card)
250 — diminishing returns past here
guidance_scale
3 — looser, more variation per seed
7 — preset default; matches README
12+ — sharper but can sound "fried"
Prerequisites:
1. Accept the terms on https://huggingface.co/stabilityai/stable-audio-open-1.0
and export your HF token in the shell:
export HF_TOKEN=hf_...
2. Install optional inference deps (one-time):
uv pip install k_diffusion einops_exts alias_free_torch torchsde
"""
from fastvideo import VideoGenerator
PROMPT = "Lo-fi hip hop instrumental with vinyl crackle and gentle piano."
def main() -> None:
generator = VideoGenerator.from_pretrained(
"FastVideo/stable-audio-open-1.0-Diffusers",
num_gpus=1,
)
output_path = "outputs_audio/stable_audio_basic/output_stable_audio.wav"
generator.generate_video(
prompt=PROMPT,
output_path=output_path,
save_video=True,
# 6-second clip; the model max is ~47.5s.
audio_end_in_s=6.0,
# The registered preset gives 100 steps + CFG=7.0 by default;
# override num_inference_steps / guidance_scale here for QA.
)
generator.shutdown()
if __name__ == "__main__":
main()
@@ -1,77 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Stable Audio Open 1.0 — audio-to-audio variation example.
User story (musician, late at night):
"I generated this 12-second lo-fi loop earlier and I love the chord
progression and overall vibe, but the snare hit at 0:08 sounds wrong
and the rhythm feels stiff. I don't want to start over from scratch
and lose what's working — I want the model to keep the harmony and
mood but reroll the percussion + groove."
User story (sound designer, on a deadline):
"I have one good 'sword clang' SFX. The art director wants 8 sibling
variations that all feel like the same sword from different angles —
same metal, same weight, slightly different impact. I'd rather
refine my one good take than text-prompt my way through 50 misses."
Pass `init_audio=path/to/clip` (any wav/mp3/mp4/m4a/flac the standard
deps decode) and the model will use it as a starting point for the
text prompt instead of pure noise.
Picking `init_audio_strength` (0.0 to 1.0):
Higher = closer to the source clip. Lower = more transformation.
(Same convention as the "Input Audio Strength" slider in
Stability's commercial Stable Audio web UI, so values transfer
directly.)
| strength | what you get |
|----------|----------------------------------------------------|
| 1.00 | Output ≈ reference. No transformation. |
| 0.85 | Texture micro-variation only. |
| 0.70 | Light reroll, same instruments. |
| 0.60 | Default. Instrument identity is replaceable |
| | (cello can take over from piano on the same notes).|
| 0.50 | Heavy — only melody / chord progression survives. |
| 0.30 | Reference acts as a loose mood prompt. |
| 0.00 | Plain T2A — reference ignored. |
Rule of thumb by intent:
* "Fix one part of this clip" -> 0.75 .. 0.85
* "Same notes, different instrument" -> 0.55 .. 0.65
* "Same chord progression, new content" -> 0.40 .. 0.55
* "Use this as a loose mood prompt" -> 0.20 .. 0.35
If the reference timbre is bleeding through more than you want,
lower it; if the structure is gone, raise it.
Prerequisites: same as `basic_stable_audio.py`.
"""
from fastvideo import VideoGenerator
PROMPT = "Change the piano to a cello playing the same notes"
# Path to any audio-bearing file (wav, mp3, mp4, m4a, flac, ...).
# Set to `None` to skip A2A and run plain T2A.
INIT_AUDIO_PATH: str | None = None
# Reference fidelity in [0, 1] -- higher = closer to source.
INIT_AUDIO_STRENGTH = 0.6
def main() -> None:
generator = VideoGenerator.from_pretrained(
"FastVideo/stable-audio-open-1.0-Diffusers",
num_gpus=1,
)
generator.generate_video(
prompt=PROMPT,
output_path="outputs_audio/stable_audio_a2a/output_a2a.wav",
save_video=True,
audio_end_in_s=6.0,
init_audio=INIT_AUDIO_PATH,
init_audio_strength=INIT_AUDIO_STRENGTH,
)
generator.shutdown()
if __name__ == "__main__":
main()
@@ -1,84 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Stable Audio Open 1.0 — inpainting / outpainting (loop extension) example.
User story (loop extension — the killer app):
"I have a 6-second drum loop my client likes. They want it as
background bed for a 30-second ad. I need it to loop seamlessly,
but a hard cut every 6s sounds bad. Let me extend it to 30s,
keeping the first 6s exactly as-is and letting the model continue
the groove for the remaining 24s."
User story (audio repair):
"There's a microphone bump at 0:14 in this 30-second field
recording — really obvious in headphones. Mask out 0:13 to 0:15
and let the model regenerate plausible ambience that blends in.
Everything else stays exactly as I recorded it."
User story (transition smoothing):
"I have two 10-second clips I want to crossfade. Mask out a 1s
overlap region in the middle and let the model invent a coherent
transition between the two."
How it works (RePaint-style blending):
Stable Audio Open 1.0 wasn't trained as an inpainting model
(`model_type=diffusion_cond`, not `diffusion_cond_inpaint`), so we
can't use the upstream's mask-conditioned approach directly. We
use the RePaint trick instead, which works on any v-prediction
diffusion model:
1. Encode the reference clip into latent space.
2. At every denoising step `i`, replace the kept region of the
in-flight latent (where mask == 1) with the reference
re-noised to the next timestep's sigma. Only the unkept
region (mask == 0) is freely denoised.
3. After the loop, the kept region is exactly the reference;
the unkept region is freshly generated content.
This is approximate compared to a properly trained inpainting
checkpoint — the seam between kept/unkept can have slight EQ
discontinuity — but it works on the existing public model.
Tunable: the mask is a 1-D tensor in {0, 1} at the model's sample
rate. Conventions:
1.0 = keep this sample from the reference
0.0 = regenerate this sample
Prerequisites: same as `basic_stable_audio.py`.
"""
import os
from fastvideo import VideoGenerator
PROMPT = "Steady lo-fi hip hop drum loop with vinyl crackle."
# Required: path to the reference audio file (wav, mp3, mp4, m4a, flac,
# ...) you want to extend or repair. The pipeline raises if a mask is
# passed without a reference, so this must be a real path.
REFERENCE_AUDIO_PATH = "path/to/your/loop.wav"
KEEP_SECONDS = 6.0 # first KEEP_SECONDS preserved exactly
TOTAL_SECONDS = 12.0 # extend the loop to this duration
def main() -> None:
if not os.path.isfile(REFERENCE_AUDIO_PATH):
raise FileNotFoundError(
f"REFERENCE_AUDIO_PATH={REFERENCE_AUDIO_PATH!r} does not exist. "
"Edit this script to point at a real audio file (wav/mp3/mp4/"
"m4a/flac) before running.")
generator = VideoGenerator.from_pretrained(
"FastVideo/stable-audio-open-1.0-Diffusers",
num_gpus=1,
)
generator.generate_video(
prompt=PROMPT,
output_path="outputs_audio/stable_audio_inpaint/output_inpaint.wav",
save_video=True,
audio_end_in_s=TOTAL_SECONDS,
inpaint_audio=REFERENCE_AUDIO_PATH,
# Tuple form: keep first KEEP_SECONDS, regenerate the rest.
inpaint_mask=(KEEP_SECONDS, TOTAL_SECONDS),
)
generator.shutdown()
if __name__ == "__main__":
main()
@@ -1,53 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Stable Audio Open Small — fast / lightweight T2A example.
User story (interactive UI builder):
"I'm building a sound-design UI where the user types a prompt and
we want sub-2-second feedback so the experience feels like
autocomplete, not a render queue. The full Stable Audio Open 1.0
takes ~8s on a single GPU; the small variant takes a fraction of
that — quality is lower but completely usable for real-time
iteration."
User story (overnight batch jobs):
"I'm generating 10,000 short SFX variants for a procedural game.
Wall-clock matters more than per-clip polish — give me the small
model so I can fit the run in one night instead of a week."
How it works:
The small variant is a separate Stability AI checkpoint
(`stabilityai/stable-audio-open-small`) that ships the same Oobleck
VAE as the 1.0 base model but a smaller / faster DiT (`embed_dim=1024`,
`depth=16`, `qk_norm="ln"`) and only one duration conditioner
(`seconds_total`, no `seconds_start`). FastVideo loads from the
converted Diffusers-format repo `FastVideo/stable-audio-open-small-Diffusers`
via the standard component loader; per-variant arch fields come
from `transformer/config.json` and `conditioner/config.json`.
Prerequisites: same as `basic_stable_audio.py`. The converted repo is
public so no gated-access flow is required.
"""
from fastvideo import VideoGenerator
PROMPT = "Lo-fi hip hop instrumental with vinyl crackle and gentle piano."
def main() -> None:
generator = VideoGenerator.from_pretrained(
"FastVideo/stable-audio-open-small-Diffusers",
num_gpus=1,
)
output_path = "outputs_audio/stable_audio_small/output_stable_audio_small.wav"
generator.generate_video(
prompt=PROMPT,
output_path=output_path,
save_video=True,
# Small variant trains on a ~11.9s window — keep `audio_end_in_s`
# at or below that.
audio_end_in_s=6.0,
)
generator.shutdown()
if __name__ == "__main__":
main()
+1 -1
View File
@@ -1,6 +1,6 @@
from fastvideo import VideoGenerator
# from fastvideo.api.sampling_param import SamplingParam
# from fastvideo.configs.sample import SamplingParam
OUTPUT_PATH = "video_samples_wan2_2_14B_t2v"
def main():
+1 -1
View File
@@ -1,6 +1,6 @@
from fastvideo import VideoGenerator
# from fastvideo.api.sampling_param import SamplingParam
# from fastvideo.configs.sample import SamplingParam
OUTPUT_PATH = "video_samples_wan2_1_Fun"
OUTPUT_NAME = "wan2.1_test"
+1 -1
View File
@@ -1,6 +1,6 @@
from fastvideo import VideoGenerator
# from fastvideo.api.sampling_param import SamplingParam
# from fastvideo.configs.sample import SamplingParam
OUTPUT_PATH = "video_samples_wan2_2_14B_i2v"
def main():
@@ -5,7 +5,7 @@ import time
import gradio as gr
from fastvideo.entrypoints.video_generator import VideoGenerator
from fastvideo.api.sampling_param import SamplingParam
from fastvideo.configs.sample.base import SamplingParam
from copy import deepcopy
@@ -9,7 +9,7 @@ import tempfile
import gradio as gr
from fastvideo.api.sampling_param import SamplingParam
from fastvideo.configs.sample.base import SamplingParam
MODEL_PATH_MAPPING = {
@@ -185,7 +185,7 @@ class BaseModelDeployment:
def _initialize_generator(self, config: Dict[str, Any]) -> None:
from fastvideo.entrypoints.video_generator import VideoGenerator
from fastvideo.api.sampling_param import SamplingParam
from fastvideo.configs.sample.base import SamplingParam
print(f"Initializing model: {self.model_path}")
self.generator = VideoGenerator.from_pretrained(
@@ -1,5 +1,5 @@
from fastvideo import VideoGenerator
from fastvideo.api.sampling_param import SamplingParam
from fastvideo.configs.sample import SamplingParam
OUTPUT_PATH = "./lora_out"
def main():
@@ -2,7 +2,7 @@
Inference using a LoRA checkpoint from FastVideo trainer.
"""
from fastvideo import VideoGenerator
from fastvideo.api.sampling_param import SamplingParam
from fastvideo.configs.sample import SamplingParam
OUTPUT_PATH = "./lora_out"
def main():
@@ -1,73 +0,0 @@
# Cosmos Predict2 2B T2V finetune config.
#
# Data must be preprocessed with Cosmos VAE + T5 text encoder
# into parquet format before training.
models:
student:
_target_: fastvideo.train.models.cosmos.CosmosModel
init_from: nvidia/Cosmos-Predict2-2B-Video2World
trainable: true
method:
_target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod
training:
distributed:
num_gpus: 8
sp_size: 1
tp_size: 1
hsdp_replicate_dim: 8
hsdp_shard_dim: 1
data:
data_path: data/cosmos_preprocessed
dataloader_num_workers: 4
train_batch_size: 1
training_cfg_rate: 0.0
seed: 1000
# Cosmos VAE: 4x temporal, 8x spatial compression.
# 93 frames -> 24 latent frames, 480x832 -> 60x104
num_latent_t: 24
num_height: 480
num_width: 832
num_frames: 93
optimizer:
learning_rate: 1.0e-5
betas: [0.9, 0.999]
weight_decay: 0.01
lr_scheduler: constant
lr_warmup_steps: 0
loop:
max_train_steps: 5000
gradient_accumulation_steps: 1
checkpoint:
output_dir: outputs/cosmos_finetune
training_state_checkpointing_steps: 500
checkpoints_total_limit: 3
resume_from_checkpoint: latest
tracker:
project_name: fastvideo_cosmos
run_name: cosmos_finetune
model:
enable_gradient_checkpointing_type: full
callbacks:
grad_clip:
_target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback
max_grad_norm: 1.0
validation:
_target_: fastvideo.train.callbacks.validation.ValidationCallback
pipeline_target: fastvideo.pipelines.basic.cosmos.cosmos_pipeline.Cosmos2VideoToWorldPipeline
dataset_file: data/cosmos_preprocessed/validation_prompts.json
every_steps: 100
sampling_steps: [50]
guidance_scale: 6.0
pipeline:
flow_shift: 1.0
@@ -1,79 +0,0 @@
# Cosmos-Predict2.5-2B Text-to-World overfitting test config.
#
# Overfits on a few short videos (480x832, 93 frames) to verify the
# Cosmos 2.5 training plugin works end-to-end.
#
# Preprocess data first:
# CUDA_VISIBLE_DEVICES=0 python fastvideo/pipelines/preprocess/preprocess_cosmos25_overfit.py
#
# Run:
# bash examples/train/run.sh examples/train/configs/overfit_cosmos25_t2w.yaml
models:
student:
_target_: fastvideo.train.models.cosmos.CosmosModel
init_from: KyleShao/Cosmos-Predict2.5-2B-Diffusers
trainable: true
enable_gradient_checkpointing_type: full
flow_shift: 1.0
method:
_target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod
training:
distributed:
num_gpus: 1
sp_size: 1
tp_size: 1
hsdp_replicate_dim: 1
hsdp_shard_dim: 1
data:
data_path: data/cosmos25_overfit_preprocessed
dataloader_num_workers: 0
train_batch_size: 1
training_cfg_rate: 0.0
seed: 42
num_latent_t: 24
num_height: 480
num_width: 832
num_frames: 93
optimizer:
learning_rate: 5.0e-5
betas: [0.9, 0.999]
weight_decay: 0.0
lr_scheduler: constant
lr_warmup_steps: 0
loop:
max_train_steps: 300
gradient_accumulation_steps: 1
checkpoint:
output_dir: outputs/cosmos25_overfit
training_state_checkpointing_steps: 50
checkpoints_total_limit: 2
tracker:
project_name: fastvideo_cosmos25
run_name: cosmos25_overfit
model:
precondition_outputs: false
enable_gradient_checkpointing_type: full
callbacks:
grad_clip:
_target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback
max_grad_norm: 1.0
validation:
_target_: fastvideo.train.callbacks.validation.ValidationCallback
pipeline_target: fastvideo.pipelines.basic.cosmos.cosmos2_5_pipeline.Cosmos2_5Pipeline
dataset_file: data/cosmos25_overfit_preprocessed/validation_prompts.json
every_steps: 150
sampling_steps: [35]
guidance_scale: 7.0
pipeline:
flow_shift: 1.0
+1 -1
View File
@@ -1,7 +1,7 @@
cmake_minimum_required(VERSION 3.26 FATAL_ERROR)
project(fastvideo-kernel LANGUAGES CXX)
# Prefer environment variable (used by CI or uv pip install git+repo_addr) if CMake var is not explicitly set.
# Prefer environment variable (used by CI or pip install git+repo_addr) if CMake var is not explicitly set.
if(NOT DEFINED GPU_BACKEND AND DEFINED ENV{GPU_BACKEND})
set(GPU_BACKEND "$ENV{GPU_BACKEND}")
endif()
+1 -39
View File
@@ -10,35 +10,6 @@ set -ex
echo "Building fastvideo-kernel..."
# ---------------------------------------------------------------------------
# Neutralise conda-injected compiler toolchains.
#
# Conda compiler packages (gcc_linux-aarch64, gxx_linux-64, etc.) set
# CMAKE_ARGS, CFLAGS, CXXFLAGS, and LDFLAGS on activation. When multiple
# toolchains are installed the variables can reference a *cross*-compiler
# that doesn't match the host (e.g. aarch64-conda-linux-gnu-c++ on x86_64).
# Even when the correct toolchain is active, the flags it injects
# (-march=nocona, -mtune=haswell, …) can conflict with nvcc's host-compiler
# expectations. Clear them so CMake discovers the system compiler instead.
# ---------------------------------------------------------------------------
if [[ -n "${CONDA_PREFIX:-}" ]]; then
_need_clean=0
# Detect conda cross-compiler that doesn't match the host.
_host_arch="$(uname -m)"
if [[ "${CXX:-}" == *"conda"* ]] || [[ "${CC:-}" == *"conda"* ]]; then
_need_clean=1
fi
if [[ "${CMAKE_ARGS:-}" == *"conda"* ]]; then
_need_clean=1
fi
if (( _need_clean )); then
echo "NOTE: Clearing conda-injected compiler settings (CC/CXX/CMAKE_ARGS/CFLAGS/...)"
echo " to use the system compiler for CUDA extension builds."
unset CC CXX CMAKE_ARGS CFLAGS CXXFLAGS LDFLAGS
fi
unset _need_clean _host_arch
fi
# Ensure submodules are initialized if needed (tk)
git submodule update --init --recursive
@@ -61,16 +32,7 @@ has_cmake_arg() {
}
detect_with_torch() {
# Prefer the active venv's python directly over `uv run --active --no-project`,
# which on some uv versions provisions its own interpreter and misses packages
# installed into VIRTUAL_ENV.
local py
if [[ -n "${VIRTUAL_ENV:-}" && -x "${VIRTUAL_ENV}/bin/python" ]]; then
py="${VIRTUAL_ENV}/bin/python"
else
py="$(command -v python3 || command -v python)"
fi
"${py}" -c "import torch
uv run --active --no-project python -c "import torch
if not torch.cuda.is_available():
raise RuntimeError('torch.cuda.is_available() is false')
mj, mn = torch.cuda.get_device_capability(0)
+1 -1
View File
@@ -23,7 +23,7 @@ classifiers = [
]
dependencies = [
"torch>=2.5.0",
"triton>=2.0.0; sys_platform == 'linux'",
"triton>=2.0.0",
]
[project.urls]
@@ -5,11 +5,6 @@ from fastvideo_kernel.ops import (
video_sparse_attn,
)
from fastvideo_kernel.block_sparse_attn import (
block_sparse_attn,
block_sparse_attn_from_indices,
)
from fastvideo_kernel.vmoba import (
moba_attn_varlen,
process_moba_input,
@@ -27,8 +22,6 @@ from fastvideo_kernel.turbodiffusion_ops import (
__all__ = [
"sliding_tile_attention",
"video_sparse_attn",
"block_sparse_attn",
"block_sparse_attn_from_indices",
"moba_attn_varlen",
"process_moba_input",
"process_moba_output",

Some files were not shown because too many files have changed in this diff Show More