Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1f688e1283 |
@@ -119,7 +119,7 @@ FastVideo-WorldModel/
|
|||||||
## Build & Test Commands
|
## Build & Test Commands
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv pip install -e ".[dev]" # Editable install
|
uv pip install -e .[dev] # Editable install
|
||||||
pre-commit run --all-files # Lint/format/spell
|
pre-commit run --all-files # Lint/format/spell
|
||||||
pytest tests/ # Top-level tests
|
pytest tests/ # Top-level tests
|
||||||
pytest fastvideo/tests/ -v # Package tests
|
pytest fastvideo/tests/ -v # Package tests
|
||||||
|
|||||||
@@ -1,96 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
# Sync .agents/skills/ into .claude/skills/ via per-skill symlinks.
|
|
||||||
#
|
|
||||||
# Why: Claude Code only scans .claude/skills/ and ~/.claude/skills/ for
|
|
||||||
# user-invocable skills (no skillsPath config exists — see
|
|
||||||
# https://code.claude.com/docs/en/skills.md). This repo's skills live
|
|
||||||
# in .agents/skills/ so they travel with the repo and stay under git.
|
|
||||||
# Run this once after cloning (or after adding/removing a skill) to
|
|
||||||
# expose them to Claude Code without maintaining a parallel tree.
|
|
||||||
#
|
|
||||||
# Usage:
|
|
||||||
# .agents/scripts/sync-skills.sh
|
|
||||||
#
|
|
||||||
# Idempotent and safe to re-run. Prunes stale symlinks whose source
|
|
||||||
# has been removed from .agents/skills/. Leaves hand-written
|
|
||||||
# .claude/skills/<name>/ directories untouched (only symlinks are
|
|
||||||
# managed).
|
|
||||||
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
REPO_ROOT="$(git -C "$(dirname "$0")" rev-parse --show-toplevel)"
|
|
||||||
SRC_DIR="$REPO_ROOT/.agents/skills"
|
|
||||||
DST_DIR="$REPO_ROOT/.claude/skills"
|
|
||||||
|
|
||||||
if [[ ! -d "$SRC_DIR" ]]; then
|
|
||||||
echo "Error: $SRC_DIR does not exist." >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
mkdir -p "$DST_DIR"
|
|
||||||
|
|
||||||
linked=0
|
|
||||||
unchanged=0
|
|
||||||
skipped=0
|
|
||||||
pruned=0
|
|
||||||
|
|
||||||
link_skill() {
|
|
||||||
local name="$1"
|
|
||||||
local src="$SRC_DIR/$name"
|
|
||||||
local dst="$DST_DIR/$name"
|
|
||||||
# Relative target keeps symlinks portable across clones.
|
|
||||||
local rel="../../.agents/skills/$name"
|
|
||||||
|
|
||||||
if [[ -L "$dst" ]]; then
|
|
||||||
if [[ "$(readlink "$dst")" == "$rel" ]]; then
|
|
||||||
unchanged=$((unchanged + 1))
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
rm "$dst"
|
|
||||||
elif [[ -e "$dst" ]]; then
|
|
||||||
echo "Skipped (not a symlink): .claude/skills/$name" >&2
|
|
||||||
skipped=$((skipped + 1))
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
|
|
||||||
ln -s "$rel" "$dst"
|
|
||||||
echo "Linked: .claude/skills/$name -> $rel"
|
|
||||||
linked=$((linked + 1))
|
|
||||||
}
|
|
||||||
|
|
||||||
prune_stale() {
|
|
||||||
local link="$1"
|
|
||||||
local target
|
|
||||||
target="$(readlink "$link")"
|
|
||||||
case "$target" in
|
|
||||||
../../.agents/skills/*) ;;
|
|
||||||
*) return ;;
|
|
||||||
esac
|
|
||||||
local name="${target##*/}"
|
|
||||||
if [[ ! -d "$SRC_DIR/$name" ]]; then
|
|
||||||
rm "$link"
|
|
||||||
echo "Pruned stale: .claude/skills/$(basename "$link")"
|
|
||||||
pruned=$((pruned + 1))
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
for src in "$SRC_DIR"/*/; do
|
|
||||||
[[ -d "$src" ]] || continue
|
|
||||||
name="$(basename "$src")"
|
|
||||||
# Only treat directories that actually contain a SKILL.md as skills.
|
|
||||||
[[ -f "$src/SKILL.md" ]] || continue
|
|
||||||
link_skill "$name"
|
|
||||||
done
|
|
||||||
|
|
||||||
shopt -s nullglob
|
|
||||||
for link in "$DST_DIR"/*; do
|
|
||||||
[[ -L "$link" ]] || continue
|
|
||||||
prune_stale "$link"
|
|
||||||
done
|
|
||||||
shopt -u nullglob
|
|
||||||
|
|
||||||
printf "\nSummary: %d linked, %d unchanged, %d pruned" "$linked" "$unchanged" "$pruned"
|
|
||||||
if [[ "$skipped" -gt 0 ]]; then
|
|
||||||
printf ", %d skipped (non-symlink collision)" "$skipped"
|
|
||||||
fi
|
|
||||||
printf "\n"
|
|
||||||
@@ -5,6 +5,3 @@
|
|||||||
{"name": "evaluate-video-quality", "description": "Evaluate generated video quality using available metrics (SSIM, loss trajectory, caption consistency)", "path": "evaluate-video-quality/SKILL.md", "status": "draft", "trust": "low"}
|
{"name": "evaluate-video-quality", "description": "Evaluate generated video quality using available metrics (SSIM, loss trajectory, caption consistency)", "path": "evaluate-video-quality/SKILL.md", "status": "draft", "trust": "low"}
|
||||||
{"name": "index-related-work", "description": "Ingest a paper or repository into the related work index", "path": "index-related-work/SKILL.md", "status": "draft", "trust": "low"}
|
{"name": "index-related-work", "description": "Ingest a paper or repository into the related work index", "path": "index-related-work/SKILL.md", "status": "draft", "trust": "low"}
|
||||||
{"name": "search-related-work", "description": "Query the related work index for relevant papers, repos, or comparisons", "path": "search-related-work/SKILL.md", "status": "draft", "trust": "low"}
|
{"name": "search-related-work", "description": "Query the related work index for relevant papers, repos, or comparisons", "path": "search-related-work/SKILL.md", "status": "draft", "trust": "low"}
|
||||||
{"name": "seed-ssim-references", "description": "Run a new or updated fastvideo/tests/ssim/ test on Modal, pull generated videos, and upload them to FastVideo/ssim-reference-videos so the test has a regression baseline", "path": "seed-ssim-references/SKILL.md", "status": "draft", "trust": "low"}
|
|
||||||
{"name": "reseed-ssim-references", "description": "Re-seed (overwrite) HF reference videos for an existing fastvideo/tests/ssim/ test and a single model id on Modal L40S. Always backs up current refs first, regenerates on Modal, pauses for the user to eyeball before-vs-after, then uploads with --force scoped to --model-id. Sister skill to seed-ssim-references; use when intentional code change has invalidated existing refs", "path": "reseed-ssim-references/SKILL.md", "status": "draft", "trust": "low"}
|
|
||||||
{"name": "add-model", "description": "Add a new model (or variant) to FastVideo: DiT + configs + pipeline + presets + registry + tests. Walks through FastVideo's single stage-based pipeline architecture with exact file paths and registration hooks.", "path": "add-model/SKILL.md", "status": "draft", "trust": "low"}
|
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ automates the boilerplate of setting environment variables, picking the right
|
|||||||
entrypoint, and applying defaults from the closest example script.
|
entrypoint, and applying defaults from the closest example script.
|
||||||
|
|
||||||
## Prerequisites
|
## Prerequisites
|
||||||
- The repo is cloned and `fastvideo` is installed (`uv pip install -e ".[dev]"`).
|
- The repo is cloned and `fastvideo` is installed (`uv pip install -e .[dev]`).
|
||||||
- Dataset is preprocessed (see `docs/training/data_preprocess.md`).
|
- Dataset is preprocessed (see `docs/training/data_preprocess.md`).
|
||||||
- `WANDB_API_KEY` is set in the environment (or `WANDB_MODE=offline` for local).
|
- `WANDB_API_KEY` is set in the environment (or `WANDB_MODE=offline` for local).
|
||||||
- GPU resources are available (multi-GPU requires NCCL).
|
- GPU resources are available (multi-GPU requires NCCL).
|
||||||
|
|||||||
@@ -1,343 +0,0 @@
|
|||||||
---
|
|
||||||
name: reseed-ssim-references
|
|
||||||
description: Re-seed HF reference videos for a single existing SSIM test on Modal L40S. Always backs up current refs locally first, regenerates on Modal, pauses for the user to eyeball before-vs-after quality, then overwrites the targeted `<model_id>` subtree on `FastVideo/ssim-reference-videos` with `--force`. Use when an intentional code change (model port fix, attention backend swap, kernel upgrade, hyperparameter change) has invalidated existing refs and they need to be regenerated. Pairs with `seed-ssim-references`, which is for first-time seeding only.
|
|
||||||
---
|
|
||||||
|
|
||||||
# Re-seed SSIM Reference Videos
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Replace the existing SSIM reference videos for a single `(test_file, model_id)`
|
|
||||||
pair on the HF dataset (`FastVideo/ssim-reference-videos`). This is **destructive**
|
|
||||||
on HF — the old refs are overwritten — so the skill always:
|
|
||||||
|
|
||||||
1. Confirms intent with a one-liner the user has to type.
|
|
||||||
2. Downloads the existing refs as a local, timestamped backup.
|
|
||||||
3. Regenerates on Modal L40S (same code path that CI uses).
|
|
||||||
4. Pauses for a side-by-side eyeball of backup vs new mp4s.
|
|
||||||
5. Uploads with `--force`, scoped to the single `--model-id`.
|
|
||||||
6. Reminds the user to keep the backup until the PR lands.
|
|
||||||
|
|
||||||
Pairs with `seed-ssim-references`, which is the inverse (first-time seeding
|
|
||||||
only, refuses to overwrite). Re-seeding is intentionally a separate, more
|
|
||||||
ceremonial operation because mistakenly clobbering production refs is much
|
|
||||||
harder to recover from than failing closed.
|
|
||||||
|
|
||||||
## When to use
|
|
||||||
|
|
||||||
- An intentional code change (model port fix, kernel upgrade, attention
|
|
||||||
backend swap, hyperparameter change in the test itself) has shifted the
|
|
||||||
expected SSIM output and the existing refs no longer represent the new
|
|
||||||
ground truth.
|
|
||||||
- A test is failing in CI **for the right reason** (the new code is correct,
|
|
||||||
the old refs are stale).
|
|
||||||
|
|
||||||
## When not to use
|
|
||||||
|
|
||||||
- A test is failing for the **wrong** reason (the port is buggy, not the
|
|
||||||
refs). Fix the port; re-seeding hides the bug.
|
|
||||||
- A brand-new test that has no refs on HF yet. Use `seed-ssim-references`.
|
|
||||||
- "Just to clean up drift" without a concrete code change to point at. The
|
|
||||||
PR description has to justify *why* refs changed; without a concrete
|
|
||||||
change, there's nothing to write.
|
|
||||||
|
|
||||||
## Inputs
|
|
||||||
|
|
||||||
| Parameter | Required | Description |
|
|
||||||
|-----------|----------|-------------|
|
|
||||||
| `test_file` | Yes | Path to the SSIM test, e.g. `fastvideo/tests/ssim/test_matrixgame_similarity.py`. Validated against `fastvideo/tests/ssim/test_*_similarity.py`. |
|
|
||||||
| `model_id` | Yes | Single model id from the test's `*_MODEL_TO_PARAMS`, e.g. `Matrix-Game-2.0-Diffusers-Base`. Re-seed runs are **per model**. For multi-model tests, invoke the skill once per model. |
|
|
||||||
| `intent_rationale` | Yes | One-line explanation of *why* refs are being regenerated (e.g. "Relax FA-2 head_size whitelist to include 80 — matrix_game now uses FLASH_ATTN instead of TORCH_SDPA"). Recorded in the backup directory and reused in the PR description. |
|
|
||||||
|
|
||||||
Hardcoded:
|
|
||||||
|
|
||||||
- Modal GPU: **L40S** (matches CI; re-seeding from another SKU produces refs
|
|
||||||
that L40S CI cannot match).
|
|
||||||
- Quality tier: **`default`**. `full_quality` is a separate, deliberate
|
|
||||||
operation.
|
|
||||||
- HF repo: `FastVideo/ssim-reference-videos` (override via
|
|
||||||
`FASTVIDEO_SSIM_REFERENCE_HF_REPO`).
|
|
||||||
- Device folder: `L40S_reference_videos`.
|
|
||||||
|
|
||||||
## Prerequisites
|
|
||||||
|
|
||||||
The user has confirmed:
|
|
||||||
|
|
||||||
- `modal` CLI authenticated.
|
|
||||||
- `hf` CLI authenticated, **and** `HF_API_KEY` (or `HUGGINGFACE_HUB_TOKEN` /
|
|
||||||
`HF_TOKEN`) exported with **write** access to
|
|
||||||
`FastVideo/ssim-reference-videos`.
|
|
||||||
- The current branch's code is the change that motivated the re-seed (i.e.
|
|
||||||
`git rev-parse HEAD` is the commit that intentionally invalidated refs).
|
|
||||||
|
|
||||||
Fail fast if any of these are missing.
|
|
||||||
|
|
||||||
## Steps
|
|
||||||
|
|
||||||
### 1. Validate inputs and confirm intent
|
|
||||||
|
|
||||||
- Verify `test_file` exists and matches `fastvideo/tests/ssim/test_*_similarity.py`.
|
|
||||||
- Grep the file for `*_MODEL_TO_PARAMS` and assert `model_id` is one of its
|
|
||||||
keys. If the file has only a single hardcoded model, accept that model id
|
|
||||||
as the only valid value.
|
|
||||||
- Print the rationale and ask the user to type **`confirm reseed`** (not just
|
|
||||||
`y` — make it deliberate):
|
|
||||||
|
|
||||||
> About to RE-SEED references for model `<model_id>` from test `<test_file>`.
|
|
||||||
> This will OVERWRITE existing refs on
|
|
||||||
> `FastVideo/ssim-reference-videos/reference_videos/default/L40S_reference_videos/<model_id>/`
|
|
||||||
> after backup + Modal regen + eyeball.
|
|
||||||
>
|
|
||||||
> Reason: `<intent_rationale>`
|
|
||||||
> HEAD: `<git rev-parse --short=12 HEAD>`
|
|
||||||
>
|
|
||||||
> Reply `confirm reseed` to proceed, anything else to abort.
|
|
||||||
|
|
||||||
Stop until the user types exactly `confirm reseed`. Anything else aborts
|
|
||||||
with no side effects.
|
|
||||||
|
|
||||||
### 2. Back up existing refs
|
|
||||||
|
|
||||||
Always required. The backup is the only graceful path back if anything goes
|
|
||||||
wrong later.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
SHORT_COMMIT=$(git rev-parse --short=12 HEAD)
|
|
||||||
TIMESTAMP=$(date -u +%Y%m%d_%H%M%S)
|
|
||||||
MODEL_SAFE=$(echo "<model_id>" | tr '/' '_')
|
|
||||||
BACKUP_DIR="ssim_reseed_backup/${TIMESTAMP}_${SHORT_COMMIT}_${MODEL_SAFE}"
|
|
||||||
mkdir -p "$BACKUP_DIR"
|
|
||||||
|
|
||||||
hf download \
|
|
||||||
--repo-type dataset FastVideo/ssim-reference-videos \
|
|
||||||
--include "reference_videos/default/L40S_reference_videos/<model_id>/**" \
|
|
||||||
--local-dir "$BACKUP_DIR"
|
|
||||||
|
|
||||||
mp4_count=$(find "$BACKUP_DIR" -name "*.mp4" | wc -l)
|
|
||||||
echo "Backup mp4 count: $mp4_count"
|
|
||||||
[ "$mp4_count" -gt 0 ] || {
|
|
||||||
echo "ERROR: backup is empty for <model_id>. Either the model id is wrong"
|
|
||||||
echo "or there are no existing refs (use seed-ssim-references instead)."
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# Provenance — used in the PR description
|
|
||||||
cat > "$BACKUP_DIR/PROVENANCE.txt" <<EOF
|
|
||||||
test_file: <test_file>
|
|
||||||
model_id: <model_id>
|
|
||||||
head_commit: $(git rev-parse HEAD)
|
|
||||||
timestamp_utc: $(date -u +%FT%TZ)
|
|
||||||
reason: <intent_rationale>
|
|
||||||
EOF
|
|
||||||
```
|
|
||||||
|
|
||||||
If the `hf download` produces zero mp4s, abort — the user has either picked a
|
|
||||||
non-existent `model_id` or there are no refs yet (in which case
|
|
||||||
`seed-ssim-references` is the right tool).
|
|
||||||
|
|
||||||
### 3. Regenerate on Modal L40S
|
|
||||||
|
|
||||||
Mirror CI's exact env recipe so the regenerated refs are byte-comparable to
|
|
||||||
what CI will produce on the same commit. Two differences from CI:
|
|
||||||
|
|
||||||
1. **Pass the same env prefix CI uses** (`IMAGE_VERSION`, `BUILDKITE_*`) — see
|
|
||||||
`.buildkite/pipeline.yml:1-3` and `.buildkite/scripts/pr_test.sh:62-83`.
|
|
||||||
Without this, `ssim_test.py:17-18` resolves a different GHCR image tag
|
|
||||||
(default is `latest`, CI is `py3.12-latest`), and `ssim_test.py:38-46`
|
|
||||||
bakes different values into the image's frozen env block. **Mismatched
|
|
||||||
image or env is the most common source of SSIM drift between reseed and
|
|
||||||
CI runs.**
|
|
||||||
2. **Do not pass `--skip-reference-download`**. Letting the test fetch the
|
|
||||||
existing refs and run the full SSIM compare gives "before" SSIM numbers
|
|
||||||
for the PR description, and the test still produces the new mp4s
|
|
||||||
regardless of whether the comparison passes or fails.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
SUBDIR="${TIMESTAMP}_${SHORT_COMMIT}"
|
|
||||||
|
|
||||||
IMAGE_VERSION="py3.12-latest" \
|
|
||||||
BUILDKITE_REPO="$(git config --get remote.origin.url)" \
|
|
||||||
BUILDKITE_COMMIT="$(git rev-parse HEAD)" \
|
|
||||||
BUILDKITE_PULL_REQUEST="${BUILDKITE_PULL_REQUEST:-false}" \
|
|
||||||
modal run fastvideo/tests/modal/ssim_test.py \
|
|
||||||
--git-repo="$(git config --get remote.origin.url)" \
|
|
||||||
--git-commit="$(git rev-parse HEAD)" \
|
|
||||||
--hf-api-key="$HF_API_KEY" \
|
|
||||||
--test-files="<test_file>" \
|
|
||||||
--sync-generated-to-volume \
|
|
||||||
--generated-volume-subdir="$SUBDIR" \
|
|
||||||
--no-fail-fast
|
|
||||||
```
|
|
||||||
|
|
||||||
Capture the printed `modal volume get ...` hint — its `<SUBDIR>` matches
|
|
||||||
`$SUBDIR` and is needed for step 4. Capture the SSIM numbers from the test
|
|
||||||
output (or from the JSON next to the generated mp4) for the PR description.
|
|
||||||
|
|
||||||
### 4. Download generated videos
|
|
||||||
|
|
||||||
```bash
|
|
||||||
modal volume get --force hf-model-weights \
|
|
||||||
ssim_generated_videos/default/"$SUBDIR"/generated_videos \
|
|
||||||
./generated_videos_modal/default
|
|
||||||
```
|
|
||||||
|
|
||||||
After this, the new mp4s live at:
|
|
||||||
|
|
||||||
```
|
|
||||||
./generated_videos_modal/default/generated_videos/L40S_reference_videos/<model_id>/<backend>/<prompt>.mp4
|
|
||||||
```
|
|
||||||
|
|
||||||
`--force` is required when `./generated_videos_modal/default` already exists
|
|
||||||
from a prior run; safe on the first run too.
|
|
||||||
|
|
||||||
### 5. PAUSE — user reviews quality side-by-side
|
|
||||||
|
|
||||||
Print the diff and the comparison:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
echo "=== File list diff (backup vs new) ==="
|
|
||||||
diff -u \
|
|
||||||
<(find "$BACKUP_DIR/reference_videos/default/L40S_reference_videos/<model_id>" -name "*.mp4" \
|
|
||||||
| sed "s|$BACKUP_DIR/reference_videos/default/L40S_reference_videos/||" | sort) \
|
|
||||||
<(find ./generated_videos_modal/default/generated_videos/L40S_reference_videos/<model_id> -name "*.mp4" \
|
|
||||||
| sed "s|./generated_videos_modal/default/generated_videos/L40S_reference_videos/||" | sort) \
|
|
||||||
|| true
|
|
||||||
|
|
||||||
echo
|
|
||||||
echo "=== SSIM numbers from this run (paste into PR) ==="
|
|
||||||
find ./generated_videos_modal/default/generated_videos/L40S_reference_videos/<model_id> -name "*_ssim.json" -exec cat {} \;
|
|
||||||
```
|
|
||||||
|
|
||||||
Then stop and tell the user:
|
|
||||||
|
|
||||||
> Old refs backed up to `$BACKUP_DIR`.
|
|
||||||
> New videos in `./generated_videos_modal/default/generated_videos/L40S_reference_videos/<model_id>/`.
|
|
||||||
>
|
|
||||||
> Open both in a video player. Confirm the new videos:
|
|
||||||
> 1. Look correct (no obvious artifacts, no black/static frames).
|
|
||||||
> 2. Are *intentionally* different from the backup in the way described
|
|
||||||
> in `<intent_rationale>` (e.g. slight numerical drift only, not a
|
|
||||||
> different scene / different motion / corrupted output).
|
|
||||||
>
|
|
||||||
> Reply **`upload`** to overwrite HF, anything else to abort.
|
|
||||||
> Aborting leaves the backup and new videos on disk for inspection — nothing
|
|
||||||
> on HF changes.
|
|
||||||
|
|
||||||
Do not proceed until the user types exactly `upload`. If they abort, leave
|
|
||||||
everything on disk and stop here.
|
|
||||||
|
|
||||||
### 6. Copy into the local reference layout
|
|
||||||
|
|
||||||
Same as `seed-ssim-references` step 5:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python fastvideo/tests/ssim/reference_videos_cli.py copy-local \
|
|
||||||
--quality-tier default \
|
|
||||||
--device-folder L40S_reference_videos \
|
|
||||||
--generated-dir ./generated_videos_modal/default/generated_videos/L40S_reference_videos
|
|
||||||
```
|
|
||||||
|
|
||||||
Result: `fastvideo/tests/ssim/reference_videos/default/L40S_reference_videos/<model_id>/<backend>/<prompt>.mp4`.
|
|
||||||
|
|
||||||
### 7. Upload with `--force`, scoped to `--model-id`
|
|
||||||
|
|
||||||
The `--force` flag is what makes this skill different from `seed-ssim-references`.
|
|
||||||
Always pair it with `--model-id` so a typo cannot accidentally overwrite a
|
|
||||||
neighboring model's refs.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python fastvideo/tests/ssim/reference_videos_cli.py upload \
|
|
||||||
--quality-tier default \
|
|
||||||
--device-folder L40S_reference_videos \
|
|
||||||
--model-id "<model_id>" \
|
|
||||||
--force
|
|
||||||
```
|
|
||||||
|
|
||||||
The CLI's overwrite guard refuses without `--force`; with `--force` it
|
|
||||||
overwrites only files under
|
|
||||||
`reference_videos/default/L40S_reference_videos/<model_id>/`.
|
|
||||||
|
|
||||||
### 8. Report success and retention guidance
|
|
||||||
|
|
||||||
Print:
|
|
||||||
|
|
||||||
- The HF path that was overwritten (`<repo>/reference_videos/default/L40S_reference_videos/<model_id>/`).
|
|
||||||
- The local backup directory path.
|
|
||||||
- The new SSIM numbers from step 5.
|
|
||||||
- This restore command, in case the PR review surfaces a problem after
|
|
||||||
upload:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python fastvideo/tests/ssim/reference_videos_cli.py upload \
|
|
||||||
--quality-tier default \
|
|
||||||
--device-folder L40S_reference_videos \
|
|
||||||
--model-id "<model_id>" \
|
|
||||||
--reference-dir "$BACKUP_DIR/reference_videos/default/L40S_reference_videos" \
|
|
||||||
--force
|
|
||||||
```
|
|
||||||
|
|
||||||
- This PR-description checklist (see `fastvideo/tests/ssim/AGENTS.md` →
|
|
||||||
*Updating Reference Videos*):
|
|
||||||
1. Source commit that produced the new refs (HEAD at re-seed time).
|
|
||||||
2. Test command and GPU SKU (`L40S`).
|
|
||||||
3. Before/after SSIM numbers.
|
|
||||||
4. The `<intent_rationale>` from step 1.
|
|
||||||
5. A note that the backup lives at `$BACKUP_DIR` and should be retained
|
|
||||||
until CI on the PR is green.
|
|
||||||
|
|
||||||
Do **not** auto-rerun the SSIM test — the user does that as part of the PR.
|
|
||||||
|
|
||||||
## Failure modes and how to handle them
|
|
||||||
|
|
||||||
- **`HF_API_KEY` unset.** Stop before step 2.
|
|
||||||
- **Backup is empty (zero mp4s).** Stop before step 3 — the model id is
|
|
||||||
wrong or the refs don't exist yet (use `seed-ssim-references`).
|
|
||||||
- **Modal run fails before generation.** No mp4s on the volume. Don't
|
|
||||||
upload. Investigate the failure (test crash, OOM, partition exhaustion),
|
|
||||||
fix, then retry from step 3. Backup is still intact.
|
|
||||||
- **Quality regressed (visual or metric).** User aborts at step 5. Backup
|
|
||||||
retained. New videos retained on disk for inspection. Nothing on HF
|
|
||||||
changed. Either fix the underlying code change or abandon the re-seed.
|
|
||||||
- **User confirmed `upload` but later realized the new refs are wrong.**
|
|
||||||
Run the restore command from step 8 with the backup `--reference-dir`.
|
|
||||||
This is exactly why the backup exists.
|
|
||||||
- **Multi-model test, only one model is being re-seeded.** Run the skill
|
|
||||||
once per model id. The `--model-id` scope on upload guarantees the others
|
|
||||||
are untouched.
|
|
||||||
|
|
||||||
## Design notes (for future skill maintainers)
|
|
||||||
|
|
||||||
- Per-`model_id` scope is mandatory. The dataset houses many model subtrees;
|
|
||||||
re-seeding the wrong one is hard to undo without backup.
|
|
||||||
- `default` tier only; `full_quality` is a separate, deliberate operation
|
|
||||||
with different params and ~doubled runtime, and isn't what CI gates on.
|
|
||||||
- The skill deliberately does **not** pass `--skip-reference-download` to
|
|
||||||
Modal so we get pre-reseed SSIM numbers for the PR. The `seed`-skill
|
|
||||||
passes it because no refs exist yet; for re-seed, refs do exist and
|
|
||||||
exposing the comparison is informative.
|
|
||||||
- The two-token confirm (`confirm reseed`, then `upload`) is intentional.
|
|
||||||
Re-seeding is high-blast-radius and should not be one-keystroke.
|
|
||||||
- The backup directory is plain mp4s + `PROVENANCE.txt`. No HF metadata is
|
|
||||||
preserved; the restore path uses `reference_videos_cli.py upload
|
|
||||||
--reference-dir` which doesn't need it.
|
|
||||||
|
|
||||||
## References
|
|
||||||
|
|
||||||
- `.agents/skills/seed-ssim-references/SKILL.md` — the first-time seed
|
|
||||||
skill this one parallels. Read it for the Modal flag rationale shared
|
|
||||||
between the two flows.
|
|
||||||
- `fastvideo/tests/ssim/AGENTS.md` — directory rules, including the PR
|
|
||||||
expectations for any reference-video change (rationale, before/after
|
|
||||||
SSIM, source commit/model/backend).
|
|
||||||
- `fastvideo/tests/ssim/reference_videos_cli.py` — `copy-local`, `upload`
|
|
||||||
(with `--model-id`, `--force`), `download`. The overwrite guard at
|
|
||||||
`upload_reference_videos` is the safety net this skill leans on.
|
|
||||||
- `fastvideo/tests/modal/ssim_test.py` — Modal orchestrator;
|
|
||||||
`--sync-generated-to-volume`, `--generated-volume-subdir`,
|
|
||||||
`--skip-reference-download`, `--no-fail-fast`.
|
|
||||||
|
|
||||||
## Changelog
|
|
||||||
|
|
||||||
| Date | Change |
|
|
||||||
|------|--------|
|
|
||||||
| 2026-05-02 | Initial version. Sister skill to `seed-ssim-references`, scoped to single `(test_file, model_id)` re-seeds, with mandatory backup and two-token confirm. |
|
|
||||||
@@ -1,376 +0,0 @@
|
|||||||
---
|
|
||||||
name: seed-ssim-references
|
|
||||||
description: Seed HF reference artefacts for a single newly-added SSIM test (pixel `.mp4` for `run_text_to_video_similarity_test`-style tests, or latent `.pt` for `run_text_to_latent_similarity_test`-style tests). Runs the test on Modal L40S, downloads the generated artefacts via `modal volume get`, pauses for the user to verify (visual eyeball for mp4, numerics dump for pt), then uploads only that test's files to `FastVideo/ssim-reference-videos`. Use when a new `fastvideo/tests/ssim/test_*_similarity.py` has just been added and has no references on HF yet.
|
|
||||||
---
|
|
||||||
|
|
||||||
# Seed SSIM Reference Artefacts (mp4 or pt)
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
A brand-new SSIM test in `fastvideo/tests/ssim/` fails forever until its
|
|
||||||
reference artefacts exist on the HF dataset
|
|
||||||
(`FastVideo/ssim-reference-videos`). The dataset hosts two kinds of artefacts
|
|
||||||
side-by-side per `(model_id, backend, prompt)`:
|
|
||||||
|
|
||||||
- **`.mp4`** — pixel ground-truth for tests that call
|
|
||||||
`run_text_to_video_similarity_test` / `run_image_to_video_similarity_test`
|
|
||||||
in `inference_similarity_utils.py`. Compared via SSIM.
|
|
||||||
- **`.pt`** — pre-VAE latent bundle (fp16 full latent + fp32 slice +
|
|
||||||
metadata + `slice_spec` + `format_version`) for tests that call
|
|
||||||
`run_text_to_latent_similarity_test` in `latent_similarity_utils.py`.
|
|
||||||
Compared via cosine distance on the slice and the full tensor.
|
|
||||||
|
|
||||||
This skill:
|
|
||||||
|
|
||||||
1. Detects which artefact type the test produces (pixel vs latent).
|
|
||||||
2. Runs the test on Modal's L40S pool to generate the artefacts.
|
|
||||||
3. Downloads them to the local repo via `modal volume get`.
|
|
||||||
4. Pauses so the user can verify quality:
|
|
||||||
- **mp4**: visual eyeball in a video player.
|
|
||||||
- **pt**: numerics dump (shape, slice stats, NaN/Inf check, metadata).
|
|
||||||
5. Uploads only the new test's files to HF, with a guard that refuses to
|
|
||||||
overwrite anything already present.
|
|
||||||
|
|
||||||
The skill is run **manually**, once per new test. Before invoking it, the user
|
|
||||||
has already sanity-tested the new test locally — it launches `VideoGenerator`
|
|
||||||
and writes an artefact without crashing (the missing-reference assertion at
|
|
||||||
the end is expected). The skill does not re-test locally; it goes straight
|
|
||||||
to Modal L40S (which is what CI uses).
|
|
||||||
|
|
||||||
## When to use
|
|
||||||
|
|
||||||
- A new `test_*_similarity.py` file has been added in `fastvideo/tests/ssim/`
|
|
||||||
and the HF dataset has no `reference_videos/default/L40S_reference_videos/<model_id>/`
|
|
||||||
subtree for it yet.
|
|
||||||
|
|
||||||
## When not to use
|
|
||||||
|
|
||||||
- Regular CI runs — once refs exist, `pytest fastvideo/tests/ssim/` downloads
|
|
||||||
them automatically.
|
|
||||||
- Re-seeding an existing test. That requires `--force` on the upload step, and
|
|
||||||
is out of scope here; treat as a separate, deliberate operation.
|
|
||||||
|
|
||||||
## Inputs
|
|
||||||
|
|
||||||
The skill has **one required input**: the path to the new SSIM test file.
|
|
||||||
Prompt the user for it if they didn't supply it.
|
|
||||||
|
|
||||||
| Parameter | Required | Description |
|
|
||||||
|-----------|----------|-------------|
|
|
||||||
| `test_file` | Yes | e.g. `fastvideo/tests/ssim/test_ltx2_similarity.py`. The skill's first action is to ask for this if missing. |
|
|
||||||
|
|
||||||
Everything else is fixed:
|
|
||||||
|
|
||||||
- Modal runner GPU: **L40S** (hardcoded in `fastvideo/tests/modal/ssim_test.py`).
|
|
||||||
- Device folder: `L40S_reference_videos`.
|
|
||||||
- Quality tier: `default` (the tier CI runs). The `full_quality` tier is not
|
|
||||||
seeded by this skill.
|
|
||||||
- HF repo: `FastVideo/ssim-reference-videos` (dataset).
|
|
||||||
- Multi-model test files: all model ids in `*_MODEL_TO_PARAMS` are seeded
|
|
||||||
together; the Modal run produces one mp4 per (model, prompt, backend) and
|
|
||||||
the upload scopes by `--model-id`, looping if there is more than one.
|
|
||||||
|
|
||||||
## Prerequisites
|
|
||||||
|
|
||||||
The user has confirmed:
|
|
||||||
|
|
||||||
- `modal` CLI authenticated.
|
|
||||||
- `HF_API_KEY` (or `HUGGINGFACE_HUB_TOKEN` / `HF_TOKEN`) exported with write
|
|
||||||
access to `FastVideo/ssim-reference-videos`.
|
|
||||||
- The test file runs locally end-to-end (generates an mp4; SSIM assertion
|
|
||||||
failure due to missing reference is expected and fine).
|
|
||||||
|
|
||||||
Fail fast if the token env var is missing.
|
|
||||||
|
|
||||||
## Steps
|
|
||||||
|
|
||||||
### 1. Ask for the test file, then detect artefact type
|
|
||||||
|
|
||||||
If the user didn't name one, ask: *"Which SSIM test file do you want to seed
|
|
||||||
references for? (e.g. `fastvideo/tests/ssim/test_ltx2_similarity.py`)"*.
|
|
||||||
|
|
||||||
Validate:
|
|
||||||
|
|
||||||
- Path exists and matches `fastvideo/tests/ssim/test_*_similarity.py`.
|
|
||||||
- File defines a `*_MODEL_TO_PARAMS` dict — grep it to extract the set of
|
|
||||||
model ids. Those ids drive step 5.
|
|
||||||
|
|
||||||
Detect artefact type by inspecting the file's imports / helper call:
|
|
||||||
|
|
||||||
- **latent** (`.pt`) — file imports `run_text_to_latent_similarity_test`
|
|
||||||
from `fastvideo.tests.ssim.latent_similarity_utils` (or any other helper
|
|
||||||
that ends with `_latent_similarity_test`).
|
|
||||||
- **pixel** (`.mp4`) — file imports
|
|
||||||
`run_text_to_video_similarity_test` / `run_image_to_video_similarity_test`
|
|
||||||
from `fastvideo.tests.ssim.inference_similarity_utils`, OR uses the
|
|
||||||
legacy custom-inline helper pattern (see `test_gamecraft`,
|
|
||||||
`test_longcat`, etc.). Default to pixel when both heuristics fail.
|
|
||||||
|
|
||||||
Record `ARTEFACT_TYPE ∈ {pixel, latent}` for use in step 4. Steps 2, 3, 5,
|
|
||||||
and 6 are artefact-type-agnostic — `_iter_reference_files`,
|
|
||||||
`copy_generated_to_reference`, and `upload_reference_videos` already walk
|
|
||||||
both `.mp4` and `.pt` (see `reference_videos_cli.py`).
|
|
||||||
|
|
||||||
If either check fails, stop and tell the user what's wrong.
|
|
||||||
|
|
||||||
### 2. Run the test on Modal L40S
|
|
||||||
|
|
||||||
Pick a subdir name so repeated runs don't collide:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
SHORT_COMMIT=$(git rev-parse --short=12 HEAD)
|
|
||||||
TIMESTAMP=$(date -u +%Y%m%d_%H%M%S)
|
|
||||||
SUBDIR="${TIMESTAMP}_${SHORT_COMMIT}"
|
|
||||||
```
|
|
||||||
|
|
||||||
Then launch the Modal run. The `IMAGE_VERSION` and `BUILDKITE_*` env-prefix
|
|
||||||
**must** match what CI exports in `.buildkite/scripts/pr_test.sh`, otherwise
|
|
||||||
`fastvideo/tests/modal/ssim_test.py` resolves a different GHCR image tag
|
|
||||||
(default is `latest`, CI is `py3.12-latest`) and bakes different values into
|
|
||||||
the image's frozen env block (`ssim_test.py:17-18, 38-46`). Mismatched image
|
|
||||||
or env produces SSIM drift that doesn't show up until the same commit runs
|
|
||||||
in CI.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
IMAGE_VERSION="py3.12-latest" \
|
|
||||||
BUILDKITE_REPO="$(git config --get remote.origin.url)" \
|
|
||||||
BUILDKITE_COMMIT="$(git rev-parse HEAD)" \
|
|
||||||
BUILDKITE_PULL_REQUEST="${BUILDKITE_PULL_REQUEST:-false}" \
|
|
||||||
modal run fastvideo/tests/modal/ssim_test.py \
|
|
||||||
--git-repo="$(git config --get remote.origin.url)" \
|
|
||||||
--git-commit="$(git rev-parse HEAD)" \
|
|
||||||
--hf-api-key="$HF_API_KEY" \
|
|
||||||
--test-files="<test_file>" \
|
|
||||||
--sync-generated-to-volume \
|
|
||||||
--generated-volume-subdir="$SUBDIR" \
|
|
||||||
--skip-reference-download \
|
|
||||||
--no-fail-fast
|
|
||||||
```
|
|
||||||
|
|
||||||
Env prefix rationale (parity with CI; see `.buildkite/pipeline.yml:1-3` and
|
|
||||||
`.buildkite/scripts/pr_test.sh:62-83`):
|
|
||||||
- `IMAGE_VERSION=py3.12-latest`: pins the Modal image tag to the same one CI
|
|
||||||
uses. Without this, `ssim_test.py:17` falls back to `latest`, which on
|
|
||||||
GHCR is built from `Dockerfile.python3.10` — different Python, torch, and
|
|
||||||
flash-attn wheel than CI's `py3.12-latest` (`infra-build-image.yml:51-67`,
|
|
||||||
`_template-build-image.yml:65-101`).
|
|
||||||
- `BUILDKITE_REPO`/`BUILDKITE_COMMIT`/`BUILDKITE_PULL_REQUEST`: mirror what
|
|
||||||
Buildkite exports. `ssim_test.py:38-46` bakes these into the image's
|
|
||||||
`.env(...)` block; mismatched values can perturb in-container code paths
|
|
||||||
that branch on PR-vs-non-PR. `false` for `BUILDKITE_PULL_REQUEST` matches
|
|
||||||
Buildkite's "non-PR build" sentinel.
|
|
||||||
|
|
||||||
Flag rationale:
|
|
||||||
- `--skip-reference-download`: no refs exist yet, so conftest must not try to
|
|
||||||
pull them.
|
|
||||||
- `--no-fail-fast`: lets the test finish generation before `_assert_similarity`
|
|
||||||
raises `FileNotFoundError: Reference video folder does not exist`. The
|
|
||||||
expected failure is what we want — the mp4 has already been written.
|
|
||||||
- `--sync-generated-to-volume` + `--generated-volume-subdir`: copies the
|
|
||||||
generated mp4s to the `hf-model-weights` Modal volume under
|
|
||||||
`ssim_generated_videos/default/<SUBDIR>/generated_videos/` so we can pull
|
|
||||||
them locally.
|
|
||||||
|
|
||||||
The Modal run will end with a nonzero exit (expected) and print a
|
|
||||||
`modal volume get hf-model-weights ssim_generated_videos/default/<SUBDIR>/generated_videos ./generated_videos_modal/default`
|
|
||||||
command. Capture that `<SUBDIR>` — you need it for step 3.
|
|
||||||
|
|
||||||
### 3. Download generated videos locally
|
|
||||||
|
|
||||||
```bash
|
|
||||||
modal volume get --force hf-model-weights \
|
|
||||||
ssim_generated_videos/default/"$SUBDIR"/generated_videos \
|
|
||||||
./generated_videos_modal/default
|
|
||||||
```
|
|
||||||
|
|
||||||
`--force` is required when the parent `./generated_videos_modal/default`
|
|
||||||
already exists; without it, `modal volume get` errors with `[Errno 21] Is a
|
|
||||||
directory`. Safe to pass on the first run too.
|
|
||||||
|
|
||||||
After this, the mp4s live at
|
|
||||||
`./generated_videos_modal/default/generated_videos/L40S_reference_videos/<model_id>/<backend>/<prompt>.mp4`.
|
|
||||||
The extra `generated_videos/` level comes from the volume layout in
|
|
||||||
`_sync_generated_videos_to_volume` (`ssim_test.py`) — the command copies
|
|
||||||
`<repo>/fastvideo/tests/ssim/generated_videos/<tier>` to
|
|
||||||
`ssim_generated_videos/<tier>/<SUBDIR>/generated_videos/`, and `modal volume
|
|
||||||
get` preserves that trailing `generated_videos/` segment.
|
|
||||||
|
|
||||||
### 4. PAUSE — user reviews quality
|
|
||||||
|
|
||||||
Type-aware verification.
|
|
||||||
|
|
||||||
**For `ARTEFACT_TYPE = pixel`** — list the downloaded mp4s and ask the user to
|
|
||||||
open them in a video player:
|
|
||||||
|
|
||||||
> "Generated videos downloaded to `./generated_videos_modal/default/generated_videos/L40S_reference_videos/`. Please open them and confirm the quality looks correct. Reply **`upload`** to continue, or anything else to abort."
|
|
||||||
|
|
||||||
**For `ARTEFACT_TYPE = latent`** — `.pt` files are not human-watchable. Print
|
|
||||||
a numerics dump for each `.pt` so the user can sanity-check shape, distribution,
|
|
||||||
and metadata:
|
|
||||||
|
|
||||||
```python
|
|
||||||
import torch
|
|
||||||
from pathlib import Path
|
|
||||||
ROOT = Path("./generated_videos_modal/default/generated_videos/L40S_reference_videos")
|
|
||||||
for p in sorted(ROOT.rglob("*.pt")):
|
|
||||||
d = torch.load(p, map_location="cpu", weights_only=False)
|
|
||||||
s = d["expected_slice"]
|
|
||||||
L = d["latent"].float()
|
|
||||||
print(f"=== {p.relative_to(ROOT)} ===")
|
|
||||||
print(f" format_version: {d['format_version']}")
|
|
||||||
print(f" shape: {d['shape']}")
|
|
||||||
print(f" dtype_original: {d['dtype_original']}")
|
|
||||||
print(f" slice_spec: {d['slice_spec']}")
|
|
||||||
print(f" slice shape={tuple(s.shape)} mean={s.mean():+.4f} std={s.std():.4f} min={s.min():+.4f} max={s.max():+.4f}")
|
|
||||||
print(f" latent shape={tuple(L.shape)} mean={L.mean():+.4f} std={L.std():.4f} min={L.min():+.4f} max={L.max():+.4f}")
|
|
||||||
print(f" finite: latent NaN={torch.isnan(L).any().item()} Inf={torch.isinf(L).any().item()}; "
|
|
||||||
f"slice NaN={torch.isnan(s).any().item()} Inf={torch.isinf(s).any().item()}")
|
|
||||||
print(f" metadata: {d['metadata']}\n")
|
|
||||||
```
|
|
||||||
|
|
||||||
Sanity criteria:
|
|
||||||
- `format_version == 1` (matches `LATENT_REFERENCE_FORMAT_VERSION`).
|
|
||||||
- `shape` matches what the model produces (e.g. LTX-2 distilled =
|
|
||||||
`[1, 128, T_lat, H_lat, W_lat]`; Stable Audio Open 1.0 = `[1, 64, 1024]`).
|
|
||||||
- `slice_spec.kind` matches a registered kind (`corner_3x3_first_frame`
|
|
||||||
for video, `audio_first_8_timesteps` for audio).
|
|
||||||
- No `NaN`/`Inf`. `mean ≈ 0`, `std ≈ 1` (denoised latents stay close to
|
|
||||||
the initial Gaussian distribution; very wide deviations suggest
|
|
||||||
numerical drift).
|
|
||||||
- `metadata.prompt` matches the test's prompt.
|
|
||||||
|
|
||||||
Then ask:
|
|
||||||
|
|
||||||
> "Numerics look right? Reply **`upload`** to continue, or anything else to abort."
|
|
||||||
|
|
||||||
Do not proceed until the user explicitly says `upload`. If they abort, leave
|
|
||||||
everything on disk so they can inspect further — no cleanup.
|
|
||||||
|
|
||||||
### 5. Copy into the local reference layout
|
|
||||||
|
|
||||||
Scoped copy — only the new test's artefacts. Single command works for both
|
|
||||||
artefact types because `_iter_reference_files` walks `.mp4` and `.pt`:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python fastvideo/tests/ssim/reference_videos_cli.py copy-local \
|
|
||||||
--quality-tier default \
|
|
||||||
--device-folder L40S_reference_videos \
|
|
||||||
--generated-dir ./generated_videos_modal/default/generated_videos/L40S_reference_videos
|
|
||||||
```
|
|
||||||
|
|
||||||
(The `--generated-dir` points at the device-folder root inside the
|
|
||||||
downloaded tree; `copy-local` walks all `<model>/<backend>/*.{mp4,pt}`
|
|
||||||
underneath it. Since the Modal run was scoped to a single test file via
|
|
||||||
`--test-files`, only that test's model(s) are present — so the copy is
|
|
||||||
implicitly per-test.)
|
|
||||||
|
|
||||||
Result for pixel: `fastvideo/tests/ssim/reference_videos/default/L40S_reference_videos/<model_id>/<backend>/<prompt>.mp4`.
|
|
||||||
Result for latent: same path with `.pt` extension.
|
|
||||||
|
|
||||||
### 6. Upload to HF — scoped per model_id, with overwrite guard
|
|
||||||
|
|
||||||
For each `<model_id>`:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python fastvideo/tests/ssim/reference_videos_cli.py upload \
|
|
||||||
--quality-tier default \
|
|
||||||
--device-folder L40S_reference_videos \
|
|
||||||
--model-id "<model_id>"
|
|
||||||
```
|
|
||||||
|
|
||||||
The upload command:
|
|
||||||
|
|
||||||
- Uploads **only** `reference_videos/default/L40S_reference_videos/<model_id>/`.
|
|
||||||
- **Refuses** if any file already exists at that path on HF (this is the
|
|
||||||
guard — seeding a new test should never clobber existing refs). To override,
|
|
||||||
the user must re-run with `--force`. If the guard fires, stop and report
|
|
||||||
exactly which files exist; do not silently `--force`.
|
|
||||||
|
|
||||||
Reads the HF token from `HF_API_KEY` / `HUGGINGFACE_HUB_TOKEN` / `HF_TOKEN`.
|
|
||||||
|
|
||||||
### 7. Report success
|
|
||||||
|
|
||||||
List what was uploaded (paths in repo) and remind the user to push any
|
|
||||||
related code changes. Do **not** auto-verify by re-running Modal — the user
|
|
||||||
can run `pytest fastvideo/tests/ssim/<test_file>` later to confirm end-to-end;
|
|
||||||
it will auto-download the refs they just uploaded.
|
|
||||||
|
|
||||||
## Failure modes and how to handle them
|
|
||||||
|
|
||||||
- **`HF_API_KEY` unset.** Stop before step 2. The Modal run needs it (passed
|
|
||||||
via `--hf-api-key`), and step 6 needs it for upload. If the user
|
|
||||||
ran `hf auth login` instead of exporting an env var, read the cached
|
|
||||||
token via `huggingface_hub.get_token()` and forward it to Modal as
|
|
||||||
`--hf-api-key="$CACHED_TOKEN"`.
|
|
||||||
- **Modal run fails before generation.** No artefacts on the volume — nothing
|
|
||||||
to download. Fix the test locally (`pytest fastvideo/tests/ssim/<test_file>`)
|
|
||||||
and retry from step 2.
|
|
||||||
- **`./generated_videos_modal/default/L40S_reference_videos/` missing after
|
|
||||||
`modal volume get`.** The run didn't produce artefacts (most likely the
|
|
||||||
test crashed before writing, or `REQUIRED_GPUS` exceeded the partition
|
|
||||||
capacity — see Modal logs).
|
|
||||||
- **Latent test crashed with FSDP / inference_mode error
|
|
||||||
(`RuntimeError: Inference tensors do not track version counter`).** The
|
|
||||||
test must pass `init_kwargs_override={"use_fsdp_inference": False}` when
|
|
||||||
`sp_size == 1` — see `test_stable_audio_similarity.py` for the pattern.
|
|
||||||
Fix in the test, push, retry.
|
|
||||||
- **Upload guard fires (files already exist).** The test name / model id
|
|
||||||
collides with something already on HF. Verify the user actually wants to
|
|
||||||
replace existing refs; if so, re-run the upload with `--force`. If not,
|
|
||||||
rename the model id in `*_MODEL_TO_PARAMS` and re-seed.
|
|
||||||
- **Quality looks wrong in step 4.** Abort. The artefacts stay on disk for
|
|
||||||
inspection. The fix is usually in the test's params (resolution, steps,
|
|
||||||
seed) — edit the test, then re-run the skill.
|
|
||||||
- For latent: also check `slice_spec.kind` matches the latent rank
|
|
||||||
(`corner_3x3_first_frame` requires 5-D, `audio_first_8_timesteps`
|
|
||||||
requires 3-D); a rank/kind mismatch raises in `_extract_expected_slice`.
|
|
||||||
|
|
||||||
## Design notes (for future skill maintainers)
|
|
||||||
|
|
||||||
- The skill deliberately runs on Modal, **not** locally, because the CI
|
|
||||||
runner is L40S. Seeding from a different GPU SKU produces refs that CI's
|
|
||||||
L40S runs can't match (pixel SSIM drifts across SKUs; latent cosine has
|
|
||||||
tighter cross-SKU bf16 drift but the configured tolerances assume
|
|
||||||
same-SKU seed → same-SKU verify).
|
|
||||||
- The skill is default-tier only. `full_quality` refs are seeded by a
|
|
||||||
separate, deliberate operation — they double runtime and aren't what CI
|
|
||||||
gates on.
|
|
||||||
- The overwrite guard in `reference_videos_cli.py upload` is default-on
|
|
||||||
specifically because this skill exists. Re-seeding is a distinct operation
|
|
||||||
that requires explicit `--force`.
|
|
||||||
- Both artefact types share the same Modal flow: the orchestrator sets
|
|
||||||
`--skip-reference-download` + `--no-fail-fast`, runs pytest, the test's
|
|
||||||
helper writes the artefact (`.mp4` via `imageio` for pixel,
|
|
||||||
`save_latent_reference` → `torch.save` for latent) BEFORE the
|
|
||||||
missing-reference assertion raises. `_sync_generated_videos_to_volume` in
|
|
||||||
`ssim_test.py` does a `shutil.copytree` of the whole `generated_videos/`
|
|
||||||
tree, picking up `.mp4`, `.pt`, and the `*_ssim.json` / `*_latent.json`
|
|
||||||
metric files alongside.
|
|
||||||
|
|
||||||
## References
|
|
||||||
|
|
||||||
- `fastvideo/tests/modal/ssim_test.py` — Modal orchestrator; see
|
|
||||||
`--sync-generated-to-volume`, `--generated-volume-subdir`,
|
|
||||||
`--skip-reference-download`, `--no-fail-fast`.
|
|
||||||
- `fastvideo/tests/ssim/reference_videos_cli.py` — `copy-local`, `upload`
|
|
||||||
(with `--model-id`, `--force`), `download`, `ensure` subcommands.
|
|
||||||
Extension allowlist is `REFERENCE_EXTENSIONS = VIDEO_EXTENSIONS +
|
|
||||||
LATENT_EXTENSIONS` (`.pt`).
|
|
||||||
- `fastvideo/tests/ssim/README.md` — reference layout, HF repo conventions.
|
|
||||||
- `fastvideo/tests/ssim/inference_similarity_utils.py` — pixel helpers
|
|
||||||
(`run_text_to_video_similarity_test`,
|
|
||||||
`run_image_to_video_similarity_test`, `build_init_kwargs`).
|
|
||||||
- `fastvideo/tests/ssim/latent_similarity_utils.py` — latent helper
|
|
||||||
(`run_text_to_latent_similarity_test`), slice spec dispatch
|
|
||||||
(`_extract_expected_slice`), reference schema
|
|
||||||
(`save_latent_reference` / `load_latent_reference`),
|
|
||||||
`LATENT_REFERENCE_FORMAT_VERSION`.
|
|
||||||
|
|
||||||
## Changelog
|
|
||||||
|
|
||||||
| Date | Change |
|
|
||||||
|------|--------|
|
|
||||||
| 2026-04-17 | Initial version (Modal sync-to-volume flow). |
|
|
||||||
| 2026-04-21 | Rewrite: single-test scope, explicit user-review pause, per-`model_id` upload, HF overwrite guard. Dropped `scripts/seed_ssim.sh`. |
|
|
||||||
| 2026-04-21 | Post-first-run fixes: `modal volume get` needs `--force` when parent exists; download tree has an extra `generated_videos/` level so `--generated-dir` must reflect it. |
|
|
||||||
| 2026-05-01 | Latent (`*.pt`) artefact support: artefact-type detection in step 1, type-aware verification (visual eyeball for mp4, numerics dump for pt) in step 4, FSDP+inference_mode failure-mode added, design notes for the unified Modal flow. Triggered by PR #1253 (LTX-2 latent migration + Stable Audio latent test). |
|
|
||||||
@@ -29,8 +29,8 @@
|
|||||||
"Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of a bustling street food market. The scene captures a mix of humor and authenticity. Mid-shot framing, vibrant lighting."
|
"Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of a bustling street food market. The scene captures a mix of humor and authenticity. Mid-shot framing, vibrant lighting."
|
||||||
],
|
],
|
||||||
"run_config": {
|
"run_config": {
|
||||||
"num_warmup_runs": 2,
|
"num_warmup_runs": 1,
|
||||||
"num_measurement_runs": 5,
|
"num_measurement_runs": 3,
|
||||||
"required_gpus": 2
|
"required_gpus": 2
|
||||||
},
|
},
|
||||||
"thresholds": {
|
"thresholds": {
|
||||||
|
|||||||
+2
-186
@@ -9,183 +9,11 @@ notify:
|
|||||||
- github_commit_status:
|
- github_commit_status:
|
||||||
context: "full-suite-passed"
|
context: "full-suite-passed"
|
||||||
if: build.env("TEST_SCOPE") == "full"
|
if: build.env("TEST_SCOPE") == "full"
|
||||||
- github_commit_status:
|
|
||||||
context: "direct-test-completed"
|
|
||||||
if: build.env("TEST_SCOPE") == "direct"
|
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
# ============================================================
|
# ============================================================
|
||||||
# Direct test: triggered by /test <name> slash command.
|
- label: ":dart: Direct Test (${TEST_TYPE})"
|
||||||
# Labels match fastcheck/full-suite counterparts so the GitHub
|
if: build.env("TEST_SCOPE") == "direct"
|
||||||
# check status overwrites the original failed check.
|
|
||||||
# Only ONE step executes per build (gated by TEST_TYPE).
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
# --- Fastcheck-scope direct tests ---
|
|
||||||
- label: ":microscope: Encoder Tests"
|
|
||||||
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "encoder"
|
|
||||||
command: "timeout 90m .buildkite/scripts/pr_test.sh"
|
|
||||||
retry:
|
|
||||||
automatic:
|
|
||||||
- exit_status: 128
|
|
||||||
limit: 3
|
|
||||||
- exit_status: -1
|
|
||||||
limit: 2
|
|
||||||
agents:
|
|
||||||
queue: "default"
|
|
||||||
- label: ":microscope: VAE Tests"
|
|
||||||
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "vae"
|
|
||||||
command: "timeout 90m .buildkite/scripts/pr_test.sh"
|
|
||||||
retry:
|
|
||||||
automatic:
|
|
||||||
- exit_status: 128
|
|
||||||
limit: 3
|
|
||||||
- exit_status: -1
|
|
||||||
limit: 2
|
|
||||||
agents:
|
|
||||||
queue: "default"
|
|
||||||
- label: ":microscope: Transformer Tests"
|
|
||||||
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "transformer"
|
|
||||||
command: "timeout 90m .buildkite/scripts/pr_test.sh"
|
|
||||||
retry:
|
|
||||||
automatic:
|
|
||||||
- exit_status: 128
|
|
||||||
limit: 3
|
|
||||||
- exit_status: -1
|
|
||||||
limit: 2
|
|
||||||
agents:
|
|
||||||
queue: "default"
|
|
||||||
- label: ":microscope: Kernel Tests"
|
|
||||||
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "kernel_tests"
|
|
||||||
command: "timeout 90m .buildkite/scripts/pr_test.sh"
|
|
||||||
retry:
|
|
||||||
automatic:
|
|
||||||
- exit_status: 128
|
|
||||||
limit: 3
|
|
||||||
- exit_status: -1
|
|
||||||
limit: 2
|
|
||||||
agents:
|
|
||||||
queue: "default"
|
|
||||||
- label: ":microscope: Unit Tests"
|
|
||||||
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "unit_test"
|
|
||||||
command: "timeout 90m .buildkite/scripts/pr_test.sh"
|
|
||||||
retry:
|
|
||||||
automatic:
|
|
||||||
- exit_status: 128
|
|
||||||
limit: 3
|
|
||||||
- exit_status: -1
|
|
||||||
limit: 2
|
|
||||||
agents:
|
|
||||||
queue: "default"
|
|
||||||
|
|
||||||
# --- Full-suite-scope direct tests ---
|
|
||||||
- label: ":bar_chart: SSIM Tests"
|
|
||||||
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "ssim"
|
|
||||||
command: "timeout 90m .buildkite/scripts/pr_test.sh"
|
|
||||||
retry:
|
|
||||||
automatic:
|
|
||||||
- exit_status: 128
|
|
||||||
limit: 3
|
|
||||||
- exit_status: -1
|
|
||||||
limit: 2
|
|
||||||
- exit_status: 1
|
|
||||||
limit: 2
|
|
||||||
agents:
|
|
||||||
queue: "default"
|
|
||||||
- label: ":test_tube: LoRA Inference Tests"
|
|
||||||
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "inference_lora"
|
|
||||||
command: "timeout 90m .buildkite/scripts/pr_test.sh"
|
|
||||||
retry:
|
|
||||||
automatic:
|
|
||||||
- exit_status: 128
|
|
||||||
limit: 3
|
|
||||||
- exit_status: -1
|
|
||||||
limit: 2
|
|
||||||
agents:
|
|
||||||
queue: "default"
|
|
||||||
- label: ":test_tube: Training Tests"
|
|
||||||
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "training"
|
|
||||||
command: "timeout 90m .buildkite/scripts/pr_test.sh"
|
|
||||||
retry:
|
|
||||||
automatic:
|
|
||||||
- exit_status: 128
|
|
||||||
limit: 3
|
|
||||||
- exit_status: -1
|
|
||||||
limit: 2
|
|
||||||
agents:
|
|
||||||
queue: "default"
|
|
||||||
- label: ":test_tube: Distillation DMD Tests"
|
|
||||||
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "distillation_dmd"
|
|
||||||
command: "timeout 90m .buildkite/scripts/pr_test.sh"
|
|
||||||
retry:
|
|
||||||
automatic:
|
|
||||||
- exit_status: 128
|
|
||||||
limit: 3
|
|
||||||
- exit_status: -1
|
|
||||||
limit: 2
|
|
||||||
agents:
|
|
||||||
queue: "default"
|
|
||||||
- label: ":test_tube: Self-Forcing Tests"
|
|
||||||
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "self_forcing"
|
|
||||||
command: "timeout 90m .buildkite/scripts/pr_test.sh"
|
|
||||||
retry:
|
|
||||||
automatic:
|
|
||||||
- exit_status: 128
|
|
||||||
limit: 3
|
|
||||||
- exit_status: -1
|
|
||||||
limit: 2
|
|
||||||
agents:
|
|
||||||
queue: "default"
|
|
||||||
- label: ":test_tube: LoRA Training Tests"
|
|
||||||
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "training_lora"
|
|
||||||
command: "timeout 90m .buildkite/scripts/pr_test.sh"
|
|
||||||
retry:
|
|
||||||
automatic:
|
|
||||||
- exit_status: 128
|
|
||||||
limit: 3
|
|
||||||
- exit_status: -1
|
|
||||||
limit: 2
|
|
||||||
- exit_status: 1
|
|
||||||
limit: 2
|
|
||||||
agents:
|
|
||||||
queue: "default"
|
|
||||||
- label: ":test_tube: Training Tests VSA"
|
|
||||||
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "training_vsa"
|
|
||||||
command: "timeout 90m .buildkite/scripts/pr_test.sh"
|
|
||||||
retry:
|
|
||||||
automatic:
|
|
||||||
- exit_status: 128
|
|
||||||
limit: 3
|
|
||||||
- exit_status: -1
|
|
||||||
limit: 2
|
|
||||||
- exit_status: 1
|
|
||||||
limit: 2
|
|
||||||
agents:
|
|
||||||
queue: "default"
|
|
||||||
- label: ":test_tube: Inference Tests VMoBA"
|
|
||||||
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "inference_vmoba"
|
|
||||||
command: "timeout 90m .buildkite/scripts/pr_test.sh"
|
|
||||||
retry:
|
|
||||||
automatic:
|
|
||||||
- exit_status: 128
|
|
||||||
limit: 3
|
|
||||||
- exit_status: -1
|
|
||||||
limit: 2
|
|
||||||
agents:
|
|
||||||
queue: "default"
|
|
||||||
- label: ":test_tube: Performance Tests"
|
|
||||||
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "performance"
|
|
||||||
command: "timeout 90m .buildkite/scripts/pr_test.sh"
|
|
||||||
retry:
|
|
||||||
automatic:
|
|
||||||
- exit_status: 128
|
|
||||||
limit: 3
|
|
||||||
- exit_status: -1
|
|
||||||
limit: 2
|
|
||||||
agents:
|
|
||||||
queue: "default"
|
|
||||||
- label: ":test_tube: API Server Tests"
|
|
||||||
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "api_server"
|
|
||||||
command: "timeout 90m .buildkite/scripts/pr_test.sh"
|
command: "timeout 90m .buildkite/scripts/pr_test.sh"
|
||||||
retry:
|
retry:
|
||||||
automatic:
|
automatic:
|
||||||
@@ -307,10 +135,6 @@ steps:
|
|||||||
label: ":bar_chart: SSIM Tests"
|
label: ":bar_chart: SSIM Tests"
|
||||||
env:
|
env:
|
||||||
- TEST_TYPE=ssim
|
- TEST_TYPE=ssim
|
||||||
retry:
|
|
||||||
automatic:
|
|
||||||
- exit_status: 1
|
|
||||||
limit: 2
|
|
||||||
agents:
|
agents:
|
||||||
queue: "default"
|
queue: "default"
|
||||||
- path:
|
- path:
|
||||||
@@ -371,10 +195,6 @@ steps:
|
|||||||
label: ":test_tube: LoRA Training Tests"
|
label: ":test_tube: LoRA Training Tests"
|
||||||
env:
|
env:
|
||||||
- TEST_TYPE=training_lora
|
- TEST_TYPE=training_lora
|
||||||
retry:
|
|
||||||
automatic:
|
|
||||||
- exit_status: 1
|
|
||||||
limit: 2
|
|
||||||
agents:
|
agents:
|
||||||
queue: "default"
|
queue: "default"
|
||||||
- path:
|
- path:
|
||||||
@@ -387,10 +207,6 @@ steps:
|
|||||||
label: ":test_tube: Training Tests VSA"
|
label: ":test_tube: Training Tests VSA"
|
||||||
env:
|
env:
|
||||||
- TEST_TYPE=training_vsa
|
- TEST_TYPE=training_vsa
|
||||||
retry:
|
|
||||||
automatic:
|
|
||||||
- exit_status: 1
|
|
||||||
limit: 2
|
|
||||||
agents:
|
agents:
|
||||||
queue: "default"
|
queue: "default"
|
||||||
- path:
|
- path:
|
||||||
|
|||||||
@@ -15,21 +15,8 @@ log "Project root: $PROJECT_ROOT"
|
|||||||
# Install Modal if not available
|
# Install Modal if not available
|
||||||
if ! python3 -m modal --version &> /dev/null; then
|
if ! python3 -m modal --version &> /dev/null; then
|
||||||
log "Modal not found, installing..."
|
log "Modal not found, installing..."
|
||||||
if ! command -v uv &> /dev/null; then
|
python3 -m pip install modal
|
||||||
log "uv not found, bootstrapping..."
|
|
||||||
if ! curl -LsSf https://astral.sh/uv/install.sh | sh; then
|
|
||||||
log "Error: Failed to bootstrap uv via astral.sh installer."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
export PATH="$HOME/.local/bin:$PATH"
|
|
||||||
if ! command -v uv &> /dev/null; then
|
|
||||||
log "Error: uv still not on PATH after bootstrap."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
# --break-system-packages preserves prior `pip install --user` semantics on PEP 668 agents.
|
|
||||||
uv pip install --system --break-system-packages modal
|
|
||||||
|
|
||||||
# Verify installation
|
# Verify installation
|
||||||
if ! python3 -m modal --version &> /dev/null; then
|
if ! python3 -m modal --version &> /dev/null; then
|
||||||
log "Error: Failed to install modal. Please install it manually."
|
log "Error: Failed to install modal. Please install it manually."
|
||||||
@@ -76,72 +63,7 @@ EFFECTIVE_PR=${BUILDKITE_PULL_REQUEST:-false}
|
|||||||
if [ "$EFFECTIVE_PR" = "false" ] && [ -n "${PR_NUMBER:-}" ]; then
|
if [ "$EFFECTIVE_PR" = "false" ] && [ -n "${PR_NUMBER:-}" ]; then
|
||||||
EFFECTIVE_PR=$PR_NUMBER
|
EFFECTIVE_PR=$PR_NUMBER
|
||||||
fi
|
fi
|
||||||
MODAL_ENV="BUILDKITE_REPO=$BUILDKITE_REPO BUILDKITE_COMMIT=$BUILDKITE_COMMIT BUILDKITE_PULL_REQUEST=$EFFECTIVE_PR BUILDKITE_BRANCH=${BUILDKITE_BRANCH:-} TEST_SCOPE=${TEST_SCOPE:-} IMAGE_VERSION=$IMAGE_VERSION"
|
MODAL_ENV="BUILDKITE_REPO=$BUILDKITE_REPO BUILDKITE_COMMIT=$BUILDKITE_COMMIT BUILDKITE_PULL_REQUEST=$EFFECTIVE_PR IMAGE_VERSION=$IMAGE_VERSION"
|
||||||
|
|
||||||
POST_RUN_HOOK=""
|
|
||||||
|
|
||||||
upload_performance_artifacts() {
|
|
||||||
SHORT_SHA=${BUILDKITE_COMMIT:0:7}
|
|
||||||
LOCAL_DIR="downloaded_reports"
|
|
||||||
|
|
||||||
_download_reports() {
|
|
||||||
log "Downloading perf_reports/ from Modal Volume..."
|
|
||||||
mkdir -p "$LOCAL_DIR"
|
|
||||||
if ! modal volume get hf-model-weights "perf_reports/" "$LOCAL_DIR"; then
|
|
||||||
log "Error: Failed to download perf_reports/ from Modal Volume."
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
_upload_dashboard() {
|
|
||||||
local target
|
|
||||||
target=$(find "$LOCAL_DIR" -name "dashboard_${SHORT_SHA}_*" | head -n 1)
|
|
||||||
log "TARGET dashboard: '$target'"
|
|
||||||
|
|
||||||
if [ -n "$target" ]; then
|
|
||||||
log "Found dashboard: $target. Uploading to Buildkite..."
|
|
||||||
buildkite-agent artifact upload "$target"
|
|
||||||
buildkite-agent annotate --style info --context "perf-dashboard" < "$target"
|
|
||||||
else
|
|
||||||
log "Warning: Could not find a dashboard file matching $SHORT_SHA"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
_upload_perf_summary() {
|
|
||||||
local target
|
|
||||||
target=$(find "$LOCAL_DIR" -name "perf_${SHORT_SHA}_*" | head -n 1)
|
|
||||||
log "TARGET perf summary: '$target'"
|
|
||||||
|
|
||||||
if [ -n "$target" ]; then
|
|
||||||
log "Found perf summary: $target. Uploading to Buildkite..."
|
|
||||||
buildkite-agent artifact upload "$target"
|
|
||||||
buildkite-agent annotate --style info --context "perf-summary" < "$target"
|
|
||||||
else
|
|
||||||
log "Warning: Could not find a perf summary file matching $SHORT_SHA"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
_cleanup_modal_volume() {
|
|
||||||
log "Cleaning up perf_reports/ from Modal Volume..."
|
|
||||||
if modal volume rm hf-model-weights "perf_reports/" --recursive; then
|
|
||||||
log "Successfully deleted perf_reports/ from Modal Volume."
|
|
||||||
else
|
|
||||||
log "Warning: Failed to delete perf_reports/ from Modal Volume. Manual cleanup may be required."
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
_cleanup_local() {
|
|
||||||
log "Cleaning up local download directory..."
|
|
||||||
rm -rf "$LOCAL_DIR"
|
|
||||||
}
|
|
||||||
|
|
||||||
# --- Main flow ---
|
|
||||||
_download_reports || { _cleanup_local; return 1; }
|
|
||||||
_upload_dashboard
|
|
||||||
_upload_perf_summary
|
|
||||||
_cleanup_modal_volume
|
|
||||||
_cleanup_local
|
|
||||||
}
|
|
||||||
|
|
||||||
case "$TEST_TYPE" in
|
case "$TEST_TYPE" in
|
||||||
"encoder")
|
"encoder")
|
||||||
@@ -202,9 +124,8 @@ case "$TEST_TYPE" in
|
|||||||
MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_lora_extraction_tests"
|
MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_lora_extraction_tests"
|
||||||
;;
|
;;
|
||||||
"performance")
|
"performance")
|
||||||
log "Running performance tests on Modal..."
|
log "Running performance tests..."
|
||||||
MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_performance_tests"
|
MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_performance_tests"
|
||||||
POST_RUN_HOOK="upload_performance_artifacts"
|
|
||||||
;;
|
;;
|
||||||
"api_server")
|
"api_server")
|
||||||
log "Running API server integration tests..."
|
log "Running API server integration tests..."
|
||||||
@@ -226,10 +147,5 @@ else
|
|||||||
log "Error: Modal test failed with exit code: $TEST_EXIT_CODE"
|
log "Error: Modal test failed with exit code: $TEST_EXIT_CODE"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ -n "$POST_RUN_HOOK" ]; then
|
|
||||||
log "Executing post-run hook: $POST_RUN_HOOK"
|
|
||||||
"$POST_RUN_HOOK"
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "=== Test execution completed with exit code: $TEST_EXIT_CODE ==="
|
log "=== Test execution completed with exit code: $TEST_EXIT_CODE ==="
|
||||||
exit $TEST_EXIT_CODE
|
exit $TEST_EXIT_CODE
|
||||||
|
|||||||
@@ -13,21 +13,8 @@ log "Project root: $PROJECT_ROOT"
|
|||||||
|
|
||||||
if ! python3 -m pre_commit --version &> /dev/null; then
|
if ! python3 -m pre_commit --version &> /dev/null; then
|
||||||
log "pre-commit not found, installing..."
|
log "pre-commit not found, installing..."
|
||||||
if ! command -v uv &> /dev/null; then
|
python3 -m pip install --user pre-commit==4.0.1
|
||||||
log "uv not found, bootstrapping..."
|
|
||||||
if ! curl -LsSf https://astral.sh/uv/install.sh | sh; then
|
|
||||||
log "Error: Failed to bootstrap uv via astral.sh installer."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
export PATH="$HOME/.local/bin:$PATH"
|
|
||||||
if ! command -v uv &> /dev/null; then
|
|
||||||
log "Error: uv still not on PATH after bootstrap."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
# --break-system-packages preserves prior `pip install --user` semantics on PEP 668 agents.
|
|
||||||
uv pip install --system --break-system-packages pre-commit==4.0.1
|
|
||||||
|
|
||||||
if ! python3 -m pre_commit --version &> /dev/null; then
|
if ! python3 -m pre_commit --version &> /dev/null; then
|
||||||
log "Error: Failed to install pre-commit."
|
log "Error: Failed to install pre-commit."
|
||||||
exit 1
|
exit 1
|
||||||
|
|||||||
+12
-7
@@ -4,10 +4,8 @@ merge_protections:
|
|||||||
- base = main
|
- base = main
|
||||||
success_conditions:
|
success_conditions:
|
||||||
- "title~=(?i)^\\[(feat|feature|bugfix|fix|refactor|perf|ci|doc|docs|misc|chore|kernel|new.?model)\\]"
|
- "title~=(?i)^\\[(feat|feature|bugfix|fix|refactor|perf|ci|doc|docs|misc|chore|kernel|new.?model)\\]"
|
||||||
- "#approved-reviews-by>=1"
|
|
||||||
- check-success~=pre-commit
|
- check-success~=pre-commit
|
||||||
- check-success=fastcheck-passed
|
- check-success=fastcheck-passed
|
||||||
- check-success=full-suite-passed
|
|
||||||
|
|
||||||
pull_request_rules:
|
pull_request_rules:
|
||||||
|
|
||||||
@@ -105,7 +103,7 @@ pull_request_rules:
|
|||||||
- files~=^fastvideo/pipelines/samplers/
|
- files~=^fastvideo/pipelines/samplers/
|
||||||
- files~=^fastvideo/entrypoints/
|
- files~=^fastvideo/entrypoints/
|
||||||
- files~=^fastvideo/worker/
|
- files~=^fastvideo/worker/
|
||||||
- files~=^fastvideo/api/sampling_param
|
- files~=^fastvideo/configs/sample/
|
||||||
- files~=^fastvideo/configs/pipelines/
|
- files~=^fastvideo/configs/pipelines/
|
||||||
- files~=^examples/inference/
|
- files~=^examples/inference/
|
||||||
- -closed
|
- -closed
|
||||||
@@ -274,15 +272,24 @@ pull_request_rules:
|
|||||||
merge:
|
merge:
|
||||||
method: squash
|
method: squash
|
||||||
|
|
||||||
- name: auto-update when ready
|
- name: auto-rebase when ready and Full Suite passed
|
||||||
conditions:
|
conditions:
|
||||||
- label=ready
|
- label=ready
|
||||||
- "#approved-reviews-by>=1"
|
- "#approved-reviews-by>=1"
|
||||||
|
- check-success=full-suite-passed
|
||||||
- -conflict
|
- -conflict
|
||||||
- -closed
|
- -closed
|
||||||
- -draft
|
- -draft
|
||||||
actions:
|
actions:
|
||||||
update: {}
|
rebase: {}
|
||||||
|
|
||||||
|
- name: remove ready label on Full Suite failure
|
||||||
|
conditions:
|
||||||
|
- label=ready
|
||||||
|
- check-failure=full-suite-passed
|
||||||
|
actions:
|
||||||
|
label:
|
||||||
|
remove: [ready]
|
||||||
|
|
||||||
# ============================================================
|
# ============================================================
|
||||||
# PR title format help
|
# PR title format help
|
||||||
@@ -312,5 +319,3 @@ pull_request_rules:
|
|||||||
|
|
||||||
Please update your PR title and the merge protection check will pass automatically.
|
Please update your PR title and the merge protection check will pass automatically.
|
||||||
|
|
||||||
merge_protections_settings:
|
|
||||||
reporting_method: check-runs
|
|
||||||
|
|||||||
@@ -1,80 +0,0 @@
|
|||||||
name: Aggregate Test Status
|
|
||||||
|
|
||||||
on:
|
|
||||||
status:
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
statuses: write
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
aggregate:
|
|
||||||
if: >-
|
|
||||||
github.event.context == 'direct-test-completed'
|
|
||||||
&& github.event.state == 'success'
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Check and update aggregate status
|
|
||||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
|
||||||
with:
|
|
||||||
script: |
|
|
||||||
const sha = context.payload.sha;
|
|
||||||
|
|
||||||
const { data } = await github.rest.repos.getCombinedStatusForRef({
|
|
||||||
owner: context.repo.owner,
|
|
||||||
repo: context.repo.repo,
|
|
||||||
ref: sha,
|
|
||||||
per_page: 100,
|
|
||||||
});
|
|
||||||
|
|
||||||
const bkStatuses = data.statuses.filter(
|
|
||||||
s => s.context.startsWith('buildkite/ci/')
|
|
||||||
);
|
|
||||||
|
|
||||||
const FASTCHECK_PREFIX = 'buildkite/ci/microscope-';
|
|
||||||
const FULL_SUITE_PREFIXES = [
|
|
||||||
'buildkite/ci/test-tube-',
|
|
||||||
'buildkite/ci/bar-chart-',
|
|
||||||
];
|
|
||||||
|
|
||||||
const fastcheck = bkStatuses.filter(
|
|
||||||
s => s.context.startsWith(FASTCHECK_PREFIX)
|
|
||||||
);
|
|
||||||
const fullSuite = bkStatuses.filter(
|
|
||||||
s => FULL_SUITE_PREFIXES.some(p => s.context.startsWith(p))
|
|
||||||
);
|
|
||||||
|
|
||||||
if (
|
|
||||||
fastcheck.length > 0
|
|
||||||
&& fastcheck.every(s => s.state === 'success')
|
|
||||||
) {
|
|
||||||
core.info(
|
|
||||||
`All ${fastcheck.length} fastcheck tests passed — updating fastcheck-passed`
|
|
||||||
);
|
|
||||||
await github.rest.repos.createCommitStatus({
|
|
||||||
owner: context.repo.owner,
|
|
||||||
repo: context.repo.repo,
|
|
||||||
sha,
|
|
||||||
state: 'success',
|
|
||||||
context: 'fastcheck-passed',
|
|
||||||
description:
|
|
||||||
`All ${fastcheck.length} fastcheck tests passed`,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
|
||||||
fullSuite.length > 0
|
|
||||||
&& fullSuite.every(s => s.state === 'success')
|
|
||||||
) {
|
|
||||||
core.info(
|
|
||||||
`All ${fullSuite.length} full suite tests passed — updating full-suite-passed`
|
|
||||||
);
|
|
||||||
await github.rest.repos.createCommitStatus({
|
|
||||||
owner: context.repo.owner,
|
|
||||||
repo: context.repo.repo,
|
|
||||||
sha,
|
|
||||||
state: 'success',
|
|
||||||
context: 'full-suite-passed',
|
|
||||||
description:
|
|
||||||
`All ${fullSuite.length} full suite tests passed`,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
@@ -4,11 +4,10 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
branches: [main]
|
branches: [main]
|
||||||
workflow_call:
|
workflow_call:
|
||||||
inputs:
|
|
||||||
ref:
|
concurrency:
|
||||||
description: 'Git ref to checkout (defaults to github.ref)'
|
group: pre-commit-${{ github.ref }}
|
||||||
required: false
|
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||||
type: string
|
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
@@ -19,8 +18,6 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
with:
|
|
||||||
ref: ${{ inputs.ref || '' }}
|
|
||||||
- uses: actions/setup-python@v5
|
- uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: "3.12"
|
python-version: "3.12"
|
||||||
|
|||||||
@@ -33,7 +33,6 @@ jobs:
|
|||||||
core.setOutput('has_write', String(hasWrite));
|
core.setOutput('has_write', String(hasWrite));
|
||||||
|
|
||||||
- name: Add ready label and react
|
- name: Add ready label and react
|
||||||
id: label
|
|
||||||
if: steps.perm.outputs.has_write == 'true'
|
if: steps.perm.outputs.has_write == 'true'
|
||||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||||
with:
|
with:
|
||||||
@@ -41,6 +40,7 @@ jobs:
|
|||||||
const owner = context.repo.owner;
|
const owner = context.repo.owner;
|
||||||
const repo = context.repo.repo;
|
const repo = context.repo.repo;
|
||||||
const prNumber = context.payload.issue.number;
|
const prNumber = context.payload.issue.number;
|
||||||
|
// Remove ready first to allow re-trigger (labeled event fires on add, not if already present)
|
||||||
try { await github.rest.issues.removeLabel({ owner, repo, issue_number: prNumber, name: 'ready' }); } catch {}
|
try { await github.rest.issues.removeLabel({ owner, repo, issue_number: prNumber, name: 'ready' }); } catch {}
|
||||||
await github.rest.issues.addLabels({ owner, repo, issue_number: prNumber, labels: ['ready'] });
|
await github.rest.issues.addLabels({ owner, repo, issue_number: prNumber, labels: ['ready'] });
|
||||||
await github.rest.reactions.createForIssueComment({
|
await github.rest.reactions.createForIssueComment({
|
||||||
@@ -48,44 +48,6 @@ jobs:
|
|||||||
comment_id: context.payload.comment.id,
|
comment_id: context.payload.comment.id,
|
||||||
content: 'rocket',
|
content: 'rocket',
|
||||||
});
|
});
|
||||||
const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: prNumber });
|
|
||||||
core.setOutput('pr_sha', pr.head.sha);
|
|
||||||
core.setOutput('pr_branch', pr.head.ref);
|
|
||||||
core.setOutput('pr_number', String(prNumber));
|
|
||||||
|
|
||||||
- name: Trigger Full Suite
|
|
||||||
if: steps.perm.outputs.has_write == 'true'
|
|
||||||
env:
|
|
||||||
BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
|
|
||||||
PR_SHA: ${{ steps.label.outputs.pr_sha }}
|
|
||||||
PR_BRANCH: ${{ steps.label.outputs.pr_branch }}
|
|
||||||
PR_NUMBER: ${{ steps.label.outputs.pr_number }}
|
|
||||||
BK_ORG: ${{ vars.BUILDKITE_ORG_SLUG }}
|
|
||||||
BK_PIPELINE: ${{ vars.BUILDKITE_PIPELINE_SLUG }}
|
|
||||||
run: |
|
|
||||||
curl -sS --fail-with-body -X POST \
|
|
||||||
"https://api.buildkite.com/v2/organizations/${BK_ORG}/pipelines/${BK_PIPELINE}/builds" \
|
|
||||||
-H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
--data-raw "$(jq -n \
|
|
||||||
--arg commit "$PR_SHA" \
|
|
||||||
--arg branch "$PR_BRANCH" \
|
|
||||||
--arg message "Full Suite for PR #${PR_NUMBER} (via /merge)" \
|
|
||||||
--argjson pr_id "$PR_NUMBER" \
|
|
||||||
'{
|
|
||||||
commit: $commit,
|
|
||||||
branch: $branch,
|
|
||||||
message: $message,
|
|
||||||
ignore_pipeline_branch_filters: true,
|
|
||||||
pull_request_id: $pr_id,
|
|
||||||
pull_request_base_branch: "main",
|
|
||||||
env: {
|
|
||||||
TEST_SCOPE: "full",
|
|
||||||
FULL_SUITE: "true",
|
|
||||||
PR_NUMBER: ($pr_id | tostring)
|
|
||||||
}
|
|
||||||
}')"
|
|
||||||
|
|
||||||
parse-command:
|
parse-command:
|
||||||
if: >-
|
if: >-
|
||||||
github.event.issue.pull_request != null
|
github.event.issue.pull_request != null
|
||||||
@@ -181,26 +143,12 @@ jobs:
|
|||||||
core.setOutput('sha', pr.head.sha);
|
core.setOutput('sha', pr.head.sha);
|
||||||
core.setOutput('branch', pr.head.ref);
|
core.setOutput('branch', pr.head.ref);
|
||||||
|
|
||||||
- name: React to comment
|
|
||||||
if: steps.perm.outputs.has_write == 'true'
|
|
||||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
|
||||||
with:
|
|
||||||
script: |
|
|
||||||
await github.rest.reactions.createForIssueComment({
|
|
||||||
owner: context.repo.owner,
|
|
||||||
repo: context.repo.repo,
|
|
||||||
comment_id: context.payload.comment.id,
|
|
||||||
content: 'rocket',
|
|
||||||
});
|
|
||||||
|
|
||||||
pre-commit:
|
pre-commit:
|
||||||
needs: parse-command
|
needs: parse-command
|
||||||
if: >-
|
if: >-
|
||||||
needs.parse-command.outputs.has_write == 'true'
|
needs.parse-command.outputs.has_write == 'true'
|
||||||
&& needs.parse-command.outputs.test_scope == 'precommit'
|
&& needs.parse-command.outputs.test_scope == 'precommit'
|
||||||
uses: ./.github/workflows/ci-precommit.yml
|
uses: ./.github/workflows/ci-precommit.yml
|
||||||
with:
|
|
||||||
ref: refs/pull/${{ github.event.issue.number }}/merge
|
|
||||||
|
|
||||||
post-precommit-status:
|
post-precommit-status:
|
||||||
needs: [parse-command, pre-commit]
|
needs: [parse-command, pre-commit]
|
||||||
@@ -230,6 +178,17 @@ jobs:
|
|||||||
&& needs.parse-command.outputs.test_type != ''
|
&& needs.parse-command.outputs.test_type != ''
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
|
- name: React to comment
|
||||||
|
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||||
|
with:
|
||||||
|
script: |
|
||||||
|
await github.rest.reactions.createForIssueComment({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
comment_id: context.payload.comment.id,
|
||||||
|
content: 'rocket',
|
||||||
|
});
|
||||||
|
|
||||||
- name: Trigger Buildkite
|
- name: Trigger Buildkite
|
||||||
env:
|
env:
|
||||||
BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
|
BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
name: Trigger Full Suite
|
name: Trigger Full Suite
|
||||||
|
|
||||||
on:
|
on:
|
||||||
pull_request_target:
|
pull_request:
|
||||||
types: [labeled, synchronize]
|
types: [labeled, synchronize]
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
@@ -10,7 +10,7 @@ permissions:
|
|||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: full-suite-${{ github.event.pull_request.number }}
|
group: full-suite-${{ github.event.pull_request.number }}
|
||||||
cancel-in-progress: false
|
cancel-in-progress: true
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
trigger:
|
trigger:
|
||||||
@@ -42,7 +42,7 @@ jobs:
|
|||||||
# Find running builds for this branch with TEST_SCOPE=full and cancel them
|
# Find running builds for this branch with TEST_SCOPE=full and cancel them
|
||||||
builds=$(curl -sS -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
|
builds=$(curl -sS -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
|
||||||
"https://api.buildkite.com/v2/organizations/${{ vars.BUILDKITE_ORG_SLUG }}/pipelines/${{ vars.BUILDKITE_PIPELINE_SLUG }}/builds?branch=${PR_BRANCH}&state=running,scheduled" \
|
"https://api.buildkite.com/v2/organizations/${{ vars.BUILDKITE_ORG_SLUG }}/pipelines/${{ vars.BUILDKITE_PIPELINE_SLUG }}/builds?branch=${PR_BRANCH}&state=running,scheduled" \
|
||||||
| jq -r '.[] | select(try (.env.TEST_SCOPE == "full") catch false) | .number')
|
| jq -r '.[] | select(.env.TEST_SCOPE == "full") | .number')
|
||||||
for build_num in $builds; do
|
for build_num in $builds; do
|
||||||
echo "Cancelling Buildkite build #$build_num"
|
echo "Cancelling Buildkite build #$build_num"
|
||||||
curl -sS -X PUT -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
|
curl -sS -X PUT -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
|
||||||
|
|||||||
@@ -37,11 +37,10 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
python-version: '3.12'
|
python-version: '3.12'
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
uses: astral-sh/setup-uv@v3
|
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: uv pip install --system -r requirements-mkdocs.txt
|
run: |
|
||||||
|
python -m pip install --upgrade pip
|
||||||
|
pip install -r requirements-mkdocs.txt
|
||||||
|
|
||||||
- name: Setup Pages
|
- name: Setup Pages
|
||||||
uses: actions/configure-pages@v4
|
uses: actions/configure-pages@v4
|
||||||
|
|||||||
@@ -56,11 +56,10 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
python-version: '3.10'
|
python-version: '3.10'
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
uses: astral-sh/setup-uv@v3
|
|
||||||
|
|
||||||
- name: Install build dependencies
|
- name: Install build dependencies
|
||||||
run: uv pip install --system build twine wheel
|
run: |
|
||||||
|
python -m pip install --upgrade pip
|
||||||
|
pip install build twine wheel
|
||||||
|
|
||||||
- name: Build package
|
- name: Build package
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
@@ -131,13 +131,11 @@ jobs:
|
|||||||
clang-11 --version
|
clang-11 --version
|
||||||
nvcc --version
|
nvcc --version
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
uses: astral-sh/setup-uv@v3
|
|
||||||
|
|
||||||
- name: Install PyTorch ${{ matrix.torch-cuda.torch-version }}+cu${{ matrix.torch-cuda.cuda-version }}
|
- name: Install PyTorch ${{ matrix.torch-cuda.torch-version }}+cu${{ matrix.torch-cuda.cuda-version }}
|
||||||
run: |
|
run: |
|
||||||
uv pip install --system typing-extensions==4.12.2
|
pip install --upgrade pip
|
||||||
uv pip install --system --no-cache-dir torch==${{ matrix.torch-cuda.torch-version }} --index-url https://download.pytorch.org/whl/${{matrix.torch-cuda.torch-cuda-short}}
|
pip install typing-extensions==4.12.2
|
||||||
|
pip install --no-cache-dir torch==${{ matrix.torch-cuda.torch-version }} --index-url https://download.pytorch.org/whl/${{matrix.torch-cuda.torch-cuda-short}}
|
||||||
nvcc --version
|
nvcc --version
|
||||||
python --version
|
python --version
|
||||||
python -c "import torch; print('PyTorch:', torch.__version__)"
|
python -c "import torch; print('PyTorch:', torch.__version__)"
|
||||||
@@ -147,20 +145,20 @@ jobs:
|
|||||||
- name: Build wheel
|
- name: Build wheel
|
||||||
run: |
|
run: |
|
||||||
export PYTHONPATH=$GITHUB_WORKSPACE:$PYTHONPATH
|
export PYTHONPATH=$GITHUB_WORKSPACE:$PYTHONPATH
|
||||||
|
|
||||||
uv pip install --system setuptools ninja packaging wheel triton scikit-build-core cmake build
|
pip install setuptools ninja packaging wheel triton scikit-build-core cmake build
|
||||||
|
|
||||||
cd fastvideo-kernel
|
cd fastvideo-kernel
|
||||||
git submodule update --init --recursive # Ensure ThunderKittens submodule is initialized
|
git submodule update --init --recursive # Ensure ThunderKittens submodule is initialized
|
||||||
# Release builds are produced on GPU-less runners, so force-enable TK and target Hopper.
|
# Release builds are produced on GPU-less runners, so force-enable TK and target Hopper.
|
||||||
export TORCH_CUDA_ARCH_LIST="9.0a"
|
export TORCH_CUDA_ARCH_LIST="9.0a"
|
||||||
export CMAKE_ARGS="${CMAKE_ARGS:-} -DFASTVIDEO_KERNEL_BUILD_TK=ON -DCMAKE_CUDA_ARCHITECTURES=90a"
|
export CMAKE_ARGS="${CMAKE_ARGS:-} -DFASTVIDEO_KERNEL_BUILD_TK=ON -DCMAKE_CUDA_ARCHITECTURES=90a"
|
||||||
|
|
||||||
# Build standard wheel (no local version suffix) for PyPI
|
# Build standard wheel (no local version suffix) for PyPI
|
||||||
python -m build --wheel --outdir dist
|
python -m build --wheel --outdir dist
|
||||||
|
|
||||||
# Fix the wheel to be manylinux compliant
|
# Fix the wheel to be manylinux compliant
|
||||||
uv pip install --system auditwheel
|
pip install auditwheel
|
||||||
# Point auditwheel at torch libs, but do not vendor them into the wheel.
|
# Point auditwheel at torch libs, but do not vendor them into the wheel.
|
||||||
TORCH_LIB_DIR=$(python - <<'PY'
|
TORCH_LIB_DIR=$(python - <<'PY'
|
||||||
import os
|
import os
|
||||||
@@ -213,13 +211,10 @@ jobs:
|
|||||||
pattern: 'fastvideo_kernel-py*'
|
pattern: 'fastvideo_kernel-py*'
|
||||||
merge-multiple: true
|
merge-multiple: true
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
uses: astral-sh/setup-uv@v3
|
|
||||||
|
|
||||||
- name: Build source distribution
|
- name: Build source distribution
|
||||||
run: |
|
run: |
|
||||||
uv pip install --system build scikit-build-core cmake ninja
|
pip install build scikit-build-core cmake ninja
|
||||||
|
|
||||||
cd fastvideo-kernel
|
cd fastvideo-kernel
|
||||||
# We don't need full CUDA/Torch to just package the source (sdist)
|
# We don't need full CUDA/Torch to just package the source (sdist)
|
||||||
python -m build --sdist --outdir dist
|
python -m build --sdist --outdir dist
|
||||||
|
|||||||
@@ -85,18 +85,9 @@ docs/distillation/examples/
|
|||||||
dmd_t2v_output/
|
dmd_t2v_output/
|
||||||
preprocess_output_text/
|
preprocess_output_text/
|
||||||
|
|
||||||
# Next.js / Node artifacts under ui/: see ui/.gitignore
|
|
||||||
|
|
||||||
.claude/
|
.claude/
|
||||||
.codex/
|
.codex/
|
||||||
.sisyphus/
|
.sisyphus/
|
||||||
openspec/
|
openspec/
|
||||||
fastvideo/tests/ssim/reference_videos/**
|
fastvideo/tests/ssim/reference_videos/**
|
||||||
|
|
||||||
# Local clones of upstream repos used only for parity testing.
|
|
||||||
/stable-audio-tools/
|
|
||||||
/daVinci-MagiHuman/
|
|
||||||
|
|
||||||
# Converted model weights (produced by scripts/checkpoint_conversion/*).
|
|
||||||
# Tens of GB; should live on HF, not in git.
|
|
||||||
/converted_weights/
|
|
||||||
|
|||||||
@@ -1 +0,0 @@
|
|||||||
WRN 2026-03-26T13:46:33.469 ?.19646 server_start:193: Failed to start server: operation not permitted: /var/folders/z_/h_6myyk14d1b7z87z3vy4mjh0000gn/T/nvim.dsynkd/iSe0el/nvim.19646.0
|
|
||||||
@@ -7,13 +7,20 @@ exclude: |
|
|||||||
fastvideo-kernel/.*|
|
fastvideo-kernel/.*|
|
||||||
assets/.*|
|
assets/.*|
|
||||||
tests/.*|
|
tests/.*|
|
||||||
|
demo/.*|
|
||||||
|
predict\.py|
|
||||||
scripts/.*|
|
scripts/.*|
|
||||||
|
assets/prompts/.*|
|
||||||
|
fastvideo/data_preprocess/.*|
|
||||||
fastvideo/dataset/.*|
|
fastvideo/dataset/.*|
|
||||||
fastvideo/models/.*|
|
fastvideo/models/.*|
|
||||||
|
fastvideo/sample/.*|
|
||||||
|
fastvideo/train\.py|
|
||||||
|
fastvideo/utils/.*|
|
||||||
examples/.*|
|
examples/.*|
|
||||||
\.agents/.*|
|
|
||||||
.github/workflows/publish-fastvideo.yml|
|
.github/workflows/publish-fastvideo.yml|
|
||||||
.github/workflows/_template-build-image.yml
|
.github/workflows/_template-build-image.yml|
|
||||||
|
docs/source/inference/support_matrix.md
|
||||||
)
|
)
|
||||||
repos:
|
repos:
|
||||||
- repo: https://github.com/google/yapf
|
- repo: https://github.com/google/yapf
|
||||||
|
|||||||
@@ -1 +0,0 @@
|
|||||||
3.12
|
|
||||||
@@ -11,7 +11,7 @@
|
|||||||
- Static assets: `assets/` (including `assets/images/`, `assets/videos/`, and `assets/prompts/`) and `comfyui/assets/`.
|
- Static assets: `assets/` (including `assets/images/`, `assets/videos/`, and `assets/prompts/`) and `comfyui/assets/`.
|
||||||
|
|
||||||
## Build, Test, and Development Commands
|
## Build, Test, and Development Commands
|
||||||
- `uv pip install -e ".[dev]"`: editable install with lint/test extras.
|
- `uv pip install -e .[dev]`: editable install with lint/test extras.
|
||||||
- `pre-commit install --hook-type pre-commit --hook-type commit-msg`: enable local hooks.
|
- `pre-commit install --hook-type pre-commit --hook-type commit-msg`: enable local hooks.
|
||||||
- `pre-commit run --all-files`: run formatter/lint/type/spelling checks.
|
- `pre-commit run --all-files`: run formatter/lint/type/spelling checks.
|
||||||
- `pytest tests/`: run top-level test suite.
|
- `pytest tests/`: run top-level test suite.
|
||||||
@@ -23,8 +23,7 @@
|
|||||||
- Python 3.10+; 4-space indentation; keep code and imports readable and explicit.
|
- Python 3.10+; 4-space indentation; keep code and imports readable and explicit.
|
||||||
- Style tools are configured in `pyproject.toml` and `.pre-commit-config.yaml`:
|
- Style tools are configured in `pyproject.toml` and `.pre-commit-config.yaml`:
|
||||||
- `yapf` (format), `ruff` (lint, auto-fix), `mypy` (typing), `codespell`.
|
- `yapf` (format), `ruff` (lint, auto-fix), `mypy` (typing), `codespell`.
|
||||||
- Lint via `pre-commit run --files <changed paths>` (or `pre-commit run --all-files` for a full sweep) before committing. Do not shell out to `yapf`/`ruff`/`codespell`/`mypy` directly — pre-commit chains them with the project's config and respects the `.pre-commit-config.yaml` excludes (e.g. `fastvideo/tests/` is intentionally skipped). If pre-commit reports `(no files to check)` for your paths, that exclude is deliberate — don't bypass it.
|
- Target line length is 80.
|
||||||
- Target line length is 120 (configured in `pyproject.toml` for ruff, yapf, and isort).
|
|
||||||
- Naming: `snake_case` for functions/files, `PascalCase` for classes, `UPPER_SNAKE_CASE` for constants.
|
- Naming: `snake_case` for functions/files, `PascalCase` for classes, `UPPER_SNAKE_CASE` for constants.
|
||||||
|
|
||||||
## Testing Guidelines
|
## Testing Guidelines
|
||||||
@@ -55,31 +54,3 @@ This repository is agent-friendly. Before doing any work, read:
|
|||||||
If you are exploring a new procedure that has no existing SOP, document your
|
If you are exploring a new procedure that has no existing SOP, document your
|
||||||
progress in `.agents/exploration/` and flag it for review at the end of your
|
progress in `.agents/exploration/` and flag it for review at the end of your
|
||||||
session.
|
session.
|
||||||
|
|
||||||
## Per-Directory AGENTS.md
|
|
||||||
|
|
||||||
Local guidance lives next to the code. Read the in-scope file before editing:
|
|
||||||
|
|
||||||
| Directory | What it covers |
|
|
||||||
|-----------|----------------|
|
|
||||||
| `fastvideo/AGENTS.md` | Core package map, public API, registry-driven model dispatch |
|
|
||||||
| `fastvideo/configs/AGENTS.md` | Arch + pipeline config dataclasses, `param_names_mapping` |
|
|
||||||
| `fastvideo/models/AGENTS.md` | DiT / VAE / encoder / scheduler / loader layout (pre-commit excluded) |
|
|
||||||
| `fastvideo/layers/AGENTS.md` | Tensor-parallel linear/attention layer rules for ports |
|
|
||||||
| `fastvideo/attention/AGENTS.md` | Backend registry + env-var override |
|
|
||||||
| `fastvideo/pipelines/AGENTS.md` | Stage ABC, `basic/<model>/`, `preprocess/`, presets |
|
|
||||||
| `fastvideo/training/AGENTS.md` | Legacy monolithic pipelines (frozen for existing models) |
|
|
||||||
| `fastvideo/train/AGENTS.md` | New modular trainer (methods × models × callbacks, YAML) |
|
|
||||||
| `fastvideo/tests/AGENTS.md` | Test taxonomy, conftest, pre-commit-excluded path |
|
|
||||||
| `fastvideo/tests/ssim/AGENTS.md` | GPU SSIM regression authoring + reference video sync |
|
|
||||||
| `scripts/checkpoint_conversion/AGENTS.md` | Adding a converter for a new HF/official checkpoint |
|
|
||||||
|
|
||||||
## Critical: Two Training Stacks Coexist
|
|
||||||
|
|
||||||
- `fastvideo/training/` — legacy, monolithic per-model `*_training_pipeline.py` and
|
|
||||||
`*_distillation_pipeline.py`. Still authoritative for shipped models.
|
|
||||||
- `fastvideo/train/` — new modular framework (composable methods × models × callbacks
|
|
||||||
driven by YAML). Preferred for new training work.
|
|
||||||
|
|
||||||
Pick the matching stack before editing. Do not migrate a pipeline between them
|
|
||||||
without an explicit ask — the conventions and config surfaces differ.
|
|
||||||
|
|||||||
@@ -128,7 +128,7 @@ class CLIPFeatureExtractor(BaseFeatureExtractor):
|
|||||||
|
|
||||||
def __init__(self, device: str = 'cuda', model_name: str = "openai/clip-vit-base-patch32"):
|
def __init__(self, device: str = 'cuda', model_name: str = "openai/clip-vit-base-patch32"):
|
||||||
if not TRANSFORMERS_AVAILABLE:
|
if not TRANSFORMERS_AVAILABLE:
|
||||||
raise ImportError("Please install transformers: uv pip install transformers")
|
raise ImportError("Please install transformers: pip install transformers")
|
||||||
super().__init__(device)
|
super().__init__(device)
|
||||||
self.processor = CLIPProcessor.from_pretrained(model_name)
|
self.processor = CLIPProcessor.from_pretrained(model_name)
|
||||||
self.model = CLIPModel.from_pretrained(model_name).to(self.device)
|
self.model = CLIPModel.from_pretrained(model_name).to(self.device)
|
||||||
@@ -171,7 +171,7 @@ class VideoMAEFeatureExtractor(BaseFeatureExtractor):
|
|||||||
|
|
||||||
def __init__(self, device: str = 'cuda', model_name: str = "MCG-NJU/videomae-base"):
|
def __init__(self, device: str = 'cuda', model_name: str = "MCG-NJU/videomae-base"):
|
||||||
if not TRANSFORMERS_AVAILABLE:
|
if not TRANSFORMERS_AVAILABLE:
|
||||||
raise ImportError("Please install transformers: uv pip install transformers")
|
raise ImportError("Please install transformers: pip install transformers")
|
||||||
super().__init__(device)
|
super().__init__(device)
|
||||||
self.model = VideoMAEModel.from_pretrained(model_name).to(self.device)
|
self.model = VideoMAEModel.from_pretrained(model_name).to(self.device)
|
||||||
self.model.eval()
|
self.model.eval()
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ class I3DFeatureExtractor(nn.Module):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise RuntimeError(f"Failed to load I3D model from Hugging Face Hub. Error: {e}\n"
|
raise RuntimeError(f"Failed to load I3D model from Hugging Face Hub. Error: {e}\n"
|
||||||
f"Ensure you have internet connection and huggingface_hub installed:\n"
|
f"Ensure you have internet connection and huggingface_hub installed:\n"
|
||||||
f"uv pip install huggingface_hub") from e
|
f"pip install huggingface_hub") from e
|
||||||
|
|
||||||
def preprocess(self, videos: torch.Tensor) -> torch.Tensor:
|
def preprocess(self, videos: torch.Tensor) -> torch.Tensor:
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
# 1. Install missing dependency
|
# 1. Install missing dependency
|
||||||
uv pip install -q opencv-python-headless transformers huggingface_hub
|
pip install -q opencv-python-headless transformers huggingface_hub
|
||||||
|
|
||||||
# 2. Run FVD script
|
# 2. Run FVD script
|
||||||
python benchmarks/fvd/run_fvd.py
|
python benchmarks/fvd/run_fvd.py
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
# 1. Install missing dependency
|
# 1. Install missing dependency
|
||||||
uv pip install -q opencv-python-headless
|
pip install -q opencv-python-headless
|
||||||
+2
-2
@@ -38,10 +38,10 @@ cp -r /path/to/FastVideo/comfyui /path/to/ComfyUI/custom_nodes/FastVideo
|
|||||||
|
|
||||||
#### Install dependencies:
|
#### Install dependencies:
|
||||||
|
|
||||||
Currently, the only dependency is `fastvideo`, which can be installed with `uv`.
|
Currently, the only dependency is `fastvideo`, which can be installed using pip.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv pip install fastvideo
|
pip install fastvideo
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Install missing custom nodes:
|
#### Install missing custom nodes:
|
||||||
|
|||||||
@@ -42,15 +42,15 @@ RUN source $HOME/.local/bin/env && \
|
|||||||
uv venv --python 3.10 --seed /opt/venv && \
|
uv venv --python 3.10 --seed /opt/venv && \
|
||||||
source /opt/venv/bin/activate && \
|
source /opt/venv/bin/activate && \
|
||||||
uv pip install --no-cache-dir --upgrade pip && \
|
uv pip install --no-cache-dir --upgrade pip && \
|
||||||
uv pip install --no-cache-dir ".[dev]" && \
|
uv pip install --no-cache-dir .[dev] && \
|
||||||
uv pip install --no-cache-dir https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.4/flash_attn-2.8.3+cu128torch2.11-cp310-cp310-linux_x86_64.whl
|
uv pip install --no-cache-dir https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.7.16/flash_attn-2.8.3+cu128torch2.10-cp310-cp310-linux_x86_64.whl
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Install dependencies using uv and set up shell configuration
|
# Install dependencies using uv and set up shell configuration
|
||||||
RUN source $HOME/.local/bin/env && \
|
RUN source $HOME/.local/bin/env && \
|
||||||
source /opt/venv/bin/activate && \
|
source /opt/venv/bin/activate && \
|
||||||
uv pip install --no-cache-dir -e ".[dev]" && \
|
uv pip install --no-cache-dir -e .[dev] && \
|
||||||
git config --unset-all http.https://github.com/.extraheader || true && \
|
git config --unset-all http.https://github.com/.extraheader || true && \
|
||||||
echo 'source /opt/venv/bin/activate' >> /root/.bashrc && \
|
echo 'source /opt/venv/bin/activate' >> /root/.bashrc && \
|
||||||
echo 'if [ -n "$ZSH_VERSION" ] && [ -f ~/.zshrc ]; then . ~/.zshrc; elif [ -f ~/.bashrc ]; then . ~/.bashrc; fi' > /root/.profile
|
echo 'if [ -n "$ZSH_VERSION" ] && [ -f ~/.zshrc ]; then . ~/.zshrc; elif [ -f ~/.bashrc ]; then . ~/.bashrc; fi' > /root/.profile
|
||||||
|
|||||||
@@ -42,15 +42,15 @@ RUN source $HOME/.local/bin/env && \
|
|||||||
uv venv --python 3.11 --seed /opt/venv && \
|
uv venv --python 3.11 --seed /opt/venv && \
|
||||||
source /opt/venv/bin/activate && \
|
source /opt/venv/bin/activate && \
|
||||||
uv pip install --no-cache-dir --upgrade pip && \
|
uv pip install --no-cache-dir --upgrade pip && \
|
||||||
uv pip install --no-cache-dir ".[dev]" && \
|
uv pip install --no-cache-dir .[dev] && \
|
||||||
uv pip install --no-cache-dir https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.4/flash_attn-2.8.3+cu128torch2.11-cp311-cp311-linux_x86_64.whl
|
uv pip install --no-cache-dir https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.7.16/flash_attn-2.8.3+cu128torch2.10-cp311-cp311-linux_x86_64.whl
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Install dependencies using uv and set up shell configuration
|
# Install dependencies using uv and set up shell configuration
|
||||||
RUN source $HOME/.local/bin/env && \
|
RUN source $HOME/.local/bin/env && \
|
||||||
source /opt/venv/bin/activate && \
|
source /opt/venv/bin/activate && \
|
||||||
uv pip install --no-cache-dir -e ".[dev]" && \
|
uv pip install --no-cache-dir -e .[dev] && \
|
||||||
git config --unset-all http.https://github.com/.extraheader || true && \
|
git config --unset-all http.https://github.com/.extraheader || true && \
|
||||||
echo 'source /opt/venv/bin/activate' >> /root/.bashrc && \
|
echo 'source /opt/venv/bin/activate' >> /root/.bashrc && \
|
||||||
echo 'if [ -n "$ZSH_VERSION" ] && [ -f ~/.zshrc ]; then . ~/.zshrc; elif [ -f ~/.bashrc ]; then . ~/.bashrc; fi' > /root/.profile
|
echo 'if [ -n "$ZSH_VERSION" ] && [ -f ~/.zshrc ]; then . ~/.zshrc; elif [ -f ~/.bashrc ]; then . ~/.bashrc; fi' > /root/.profile
|
||||||
|
|||||||
@@ -42,15 +42,15 @@ RUN source $HOME/.local/bin/env && \
|
|||||||
uv venv --python 3.12 --seed /opt/venv && \
|
uv venv --python 3.12 --seed /opt/venv && \
|
||||||
source /opt/venv/bin/activate && \
|
source /opt/venv/bin/activate && \
|
||||||
uv pip install --no-cache-dir --upgrade pip && \
|
uv pip install --no-cache-dir --upgrade pip && \
|
||||||
uv pip install --no-cache-dir ".[dev]" && \
|
uv pip install --no-cache-dir .[dev] && \
|
||||||
uv pip install --no-cache-dir https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.4/flash_attn-2.8.3+cu128torch2.11-cp312-cp312-linux_x86_64.whl
|
uv pip install --no-cache-dir https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.7.16/flash_attn-2.8.3+cu128torch2.10-cp312-cp312-linux_x86_64.whl
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Install dependencies using uv and set up shell configuration
|
# Install dependencies using uv and set up shell configuration
|
||||||
RUN source $HOME/.local/bin/env && \
|
RUN source $HOME/.local/bin/env && \
|
||||||
source /opt/venv/bin/activate && \
|
source /opt/venv/bin/activate && \
|
||||||
uv pip install --no-cache-dir -e ".[dev]" && \
|
uv pip install --no-cache-dir -e .[dev] && \
|
||||||
git config --unset-all http.https://github.com/.extraheader || true && \
|
git config --unset-all http.https://github.com/.extraheader || true && \
|
||||||
echo 'source /opt/venv/bin/activate' >> /root/.bashrc && \
|
echo 'source /opt/venv/bin/activate' >> /root/.bashrc && \
|
||||||
echo 'if [ -n "$ZSH_VERSION" ] && [ -f ~/.zshrc ]; then . ~/.zshrc; elif [ -f ~/.bashrc ]; then . ~/.bashrc; fi' > /root/.profile
|
echo 'if [ -n "$ZSH_VERSION" ] && [ -f ~/.zshrc ]; then . ~/.zshrc; elif [ -f ~/.bashrc ]; then . ~/.bashrc; fi' > /root/.profile
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ RUN source $HOME/.local/bin/env && \
|
|||||||
uv venv --python 3.12 --seed /opt/venv && \
|
uv venv --python 3.12 --seed /opt/venv && \
|
||||||
source /opt/venv/bin/activate && \
|
source /opt/venv/bin/activate && \
|
||||||
uv pip install --no-cache-dir --upgrade pip && \
|
uv pip install --no-cache-dir --upgrade pip && \
|
||||||
uv pip install --no-cache-dir ".[dev]" && \
|
uv pip install --no-cache-dir .[dev] && \
|
||||||
uv pip install --no-cache-dir flash-attn==2.8.3 --no-build-isolation
|
uv pip install --no-cache-dir flash-attn==2.8.3 --no-build-isolation
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
@@ -50,7 +50,7 @@ COPY . .
|
|||||||
# Install dependencies using uv and set up shell configuration
|
# Install dependencies using uv and set up shell configuration
|
||||||
RUN source $HOME/.local/bin/env && \
|
RUN source $HOME/.local/bin/env && \
|
||||||
source /opt/venv/bin/activate && \
|
source /opt/venv/bin/activate && \
|
||||||
uv pip install --no-cache-dir -e ".[dev]" && \
|
uv pip install --no-cache-dir -e .[dev] && \
|
||||||
git config --unset-all http.https://github.com/.extraheader || true && \
|
git config --unset-all http.https://github.com/.extraheader || true && \
|
||||||
echo 'source /opt/venv/bin/activate' >> /root/.bashrc && \
|
echo 'source /opt/venv/bin/activate' >> /root/.bashrc && \
|
||||||
echo 'if [ -n "$ZSH_VERSION" ] && [ -f ~/.zshrc ]; then . ~/.zshrc; elif [ -f ~/.bashrc ]; then . ~/.bashrc; fi' > /root/.profile
|
echo 'if [ -n "$ZSH_VERSION" ] && [ -f ~/.zshrc ]; then . ~/.zshrc; elif [ -f ~/.bashrc ]; then . ~/.bashrc; fi' > /root/.profile
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ COPY . .
|
|||||||
# Install dependencies using uv and set up shell configuration
|
# Install dependencies using uv and set up shell configuration
|
||||||
RUN source $HOME/.local/bin/env && \
|
RUN source $HOME/.local/bin/env && \
|
||||||
source /opt/venv/bin/activate && \
|
source /opt/venv/bin/activate && \
|
||||||
uv pip install --no-cache-dir -e ".[rocm]" && \
|
uv pip install --no-cache-dir -e .[rocm] && \
|
||||||
git config --unset-all http.https://github.com/.extraheader || true && \
|
git config --unset-all http.https://github.com/.extraheader || true && \
|
||||||
echo 'source /opt/venv/bin/activate' >> /root/.bashrc && \
|
echo 'source /opt/venv/bin/activate' >> /root/.bashrc && \
|
||||||
echo 'if [ -n "$ZSH_VERSION" ] && [ -f ~/.zshrc ]; then . ~/.zshrc; elif [ -f ~/.bashrc ]; then . ~/.bashrc; fi' > /root/.profile
|
echo 'if [ -n "$ZSH_VERSION" ] && [ -f ~/.zshrc ]; then . ~/.zshrc; elif [ -f ~/.bashrc ]; then . ~/.bashrc; fi' > /root/.profile
|
||||||
|
|||||||
+1
-1
@@ -6,7 +6,7 @@ This directory contains the FastVideo documentation built with MkDocs.
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Install dependencies
|
# Install dependencies
|
||||||
uv pip install -r requirements-mkdocs.txt
|
pip install -r requirements-mkdocs.txt
|
||||||
|
|
||||||
# Serve docs with live reload (recommended for development)
|
# Serve docs with live reload (recommended for development)
|
||||||
mkdocs serve
|
mkdocs serve
|
||||||
|
|||||||
@@ -62,9 +62,9 @@ This page contains the complete API reference for the FastVideo library.
|
|||||||
show_root_toc_entry: true
|
show_root_toc_entry: true
|
||||||
heading_level: 4
|
heading_level: 4
|
||||||
|
|
||||||
#### fastvideo.api.sampling_param
|
#### fastvideo.configs.sample
|
||||||
|
|
||||||
::: fastvideo.api.sampling_param
|
::: fastvideo.configs.sample
|
||||||
options:
|
options:
|
||||||
show_source: true
|
show_source: true
|
||||||
show_root_heading: true
|
show_root_heading: true
|
||||||
|
|||||||
@@ -1,210 +0,0 @@
|
|||||||
# Activation Trace Mode
|
|
||||||
|
|
||||||
!!! note
|
|
||||||
This page covers Extension 0 (module forward hooks), which is the implemented
|
|
||||||
tracing mechanism. Extensions 1-3 are design sketches for future work and are
|
|
||||||
**not yet implemented**.
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
Activation trace mode is a zero-overhead-when-off, env-gated mechanism for
|
|
||||||
dumping per-layer activation statistics during FastVideo inference. Its primary
|
|
||||||
use case is **parity debugging across model ports**: enable tracing on both
|
|
||||||
FastVideo and the upstream reference implementation, then `diff` the resulting
|
|
||||||
JSONL files to find the first divergent layer.
|
|
||||||
|
|
||||||
The mechanism is intentionally narrow. It doesn't replace general logging,
|
|
||||||
profiling, or function tracing. It answers one question: "at which layer do
|
|
||||||
FastVideo and the reference model first produce different numbers?"
|
|
||||||
|
|
||||||
## When to use
|
|
||||||
|
|
||||||
- Investigating numerical drift between FastVideo and an upstream reference.
|
|
||||||
- Debugging mid-pipeline divergence (e.g., one block produces wrong output while earlier blocks match).
|
|
||||||
- Validating that a refactor preserves bf16 noise-floor behavior across many layers.
|
|
||||||
|
|
||||||
## When NOT to use
|
|
||||||
|
|
||||||
| Goal | Use instead |
|
|
||||||
|---|---|
|
|
||||||
| General logging | `init_logger(__name__)` |
|
|
||||||
| Per-stage timing | `FASTVIDEO_STAGE_LOGGING` |
|
|
||||||
| Profiling kernel timings | `FASTVIDEO_TORCH_PROFILER_DIR` (see [Profiling](profiling.md)) |
|
|
||||||
| Function-call tracing | `FASTVIDEO_TRACE_FUNCTION` (heavy) |
|
|
||||||
|
|
||||||
## Quickstart
|
|
||||||
|
|
||||||
```bash
|
|
||||||
FASTVIDEO_TRACE_ACTIVATIONS=1 \
|
|
||||||
FASTVIDEO_TRACE_LAYERS="^block\.layers\.[0-9]+$" \
|
|
||||||
FASTVIDEO_TRACE_STATS="abs_mean,sum,max,shape" \
|
|
||||||
FASTVIDEO_TRACE_OUTPUT="/tmp/fv_trace.jsonl" \
|
|
||||||
python examples/inference/basic/basic_magi_human.py
|
|
||||||
```
|
|
||||||
|
|
||||||
Each line in `/tmp/fv_trace.jsonl` is a JSON record:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{"module": "block.layers.0", "tensor": "out", "step": 0, "abs_mean": 1.234, "sum": -5.678, "max": 9.012, "shape": [1, 4096, 5120]}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
| Env var | Default | Description |
|
|
||||||
|---|---|---|
|
|
||||||
| `FASTVIDEO_TRACE_ACTIVATIONS` | `False` | Master toggle. When unset or false, **zero overhead** in the production hot path. |
|
|
||||||
| `FASTVIDEO_TRACE_LAYERS` | `""` (all) | Python regex filter applied to `model.named_modules()` names. Empty string matches all modules. |
|
|
||||||
| `FASTVIDEO_TRACE_STATS` | `"abs_mean,sum"` | Comma-separated stats to compute. Available: `abs_mean`, `sum`, `min`, `max`, `mean`, `std`, `shape`, `dtype`. |
|
|
||||||
| `FASTVIDEO_TRACE_OUTPUT` | `"/tmp/fv_trace_<pid>.jsonl"` | Output file path. `<pid>` is replaced with the process ID at runtime. |
|
|
||||||
| `FASTVIDEO_TRACE_STEPS` | `""` (all) | Comma-separated denoising step indices to capture. Empty string captures all steps. |
|
|
||||||
|
|
||||||
## Workflow: parity-debug a model port
|
|
||||||
|
|
||||||
1. Set up a tightly-controlled comparison: a parity test or a small standalone
|
|
||||||
script that loads both the FastVideo model and the upstream reference with
|
|
||||||
identical inputs and seeds.
|
|
||||||
|
|
||||||
2. Run the FastVideo side with tracing on:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
FASTVIDEO_TRACE_ACTIVATIONS=1 \
|
|
||||||
FASTVIDEO_TRACE_LAYERS="<your regex>" \
|
|
||||||
FASTVIDEO_TRACE_OUTPUT="/tmp/fv_trace_fv.jsonl" \
|
|
||||||
python <fv_runner.py>
|
|
||||||
```
|
|
||||||
|
|
||||||
3. Run the upstream side. The upstream repo needs separate instrumentation. See
|
|
||||||
"Hooking the upstream side" below.
|
|
||||||
|
|
||||||
4. Sort both files by `(module, step)` if needed, then diff:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
diff /tmp/fv_trace_fv.jsonl /tmp/fv_trace_upstream.jsonl
|
|
||||||
```
|
|
||||||
|
|
||||||
5. The first divergent line identifies the first layer where FastVideo and the
|
|
||||||
upstream produce different outputs. Start debugging there.
|
|
||||||
|
|
||||||
## Architecture (Extension 0: module forward hooks)
|
|
||||||
|
|
||||||
At pipeline initialization, `attach_activation_trace()` reads the env vars once.
|
|
||||||
If `FASTVIDEO_TRACE_ACTIVATIONS` is unset or false, the function returns
|
|
||||||
immediately and no hooks are registered. If tracing is on, it walks
|
|
||||||
`model.named_modules()`, filters by the layer regex, and registers an
|
|
||||||
`ActivationStatHook` on each matching module.
|
|
||||||
|
|
||||||
During the forward pass, each hook fires after its module completes, computes
|
|
||||||
the requested stats on the output tensor, and appends a JSON record to the
|
|
||||||
output file.
|
|
||||||
|
|
||||||
```
|
|
||||||
ComposedPipelineBase
|
|
||||||
└─ attach_activation_trace()
|
|
||||||
├─ reads env vars (once at startup)
|
|
||||||
├─ if off: returns None immediately
|
|
||||||
└─ if on: walks named_modules()
|
|
||||||
└─ registers ActivationStatHook on matching modules
|
|
||||||
└─ on each forward: compute stats → append JSONL
|
|
||||||
```
|
|
||||||
|
|
||||||
### Zero-overhead-when-off guarantee
|
|
||||||
|
|
||||||
- The env var check happens **once at startup** inside `attach_activation_trace()`.
|
|
||||||
- If the env var is unset or false, the function returns `None` immediately.
|
|
||||||
- No hooks are registered. No branches are added to the production forward path.
|
|
||||||
- The only cost when tracing is off is one env var lookup at pipeline
|
|
||||||
initialization, which takes under a microsecond.
|
|
||||||
|
|
||||||
### Hooking the upstream side
|
|
||||||
|
|
||||||
The upstream reference repo isn't part of FastVideo, so it can't read FastVideo
|
|
||||||
env vars directly. Two options:
|
|
||||||
|
|
||||||
**Option 1: Inline patch** in your local clone of the upstream repo. Add
|
|
||||||
`register_forward_hook` calls in the same shape as `ActivationStatHook`. Clean
|
|
||||||
up afterward with `git stash` or `git checkout HEAD -- <file>`.
|
|
||||||
|
|
||||||
**Option 2: Wrapper script**. Write a small Python harness that imports the
|
|
||||||
upstream model, walks its `named_modules()`, and attaches hooks externally.
|
|
||||||
This is the same pattern used in
|
|
||||||
`tests/local_tests/transformers/_debug_magi_human_block_parity.py`.
|
|
||||||
|
|
||||||
The `add-model-trace` skill at `~/.config/opencode/skill/add-model-trace/`
|
|
||||||
provides a script template for this purpose.
|
|
||||||
|
|
||||||
## Future extensions (design only, not yet implemented)
|
|
||||||
|
|
||||||
### Extension 1: FX/Dynamo backend graph rewrite
|
|
||||||
|
|
||||||
**Granularity**: per-FX-node (every matmul, every add).
|
|
||||||
|
|
||||||
**Mechanism**: a `torch.compile` backend that takes the captured `GraphModule`
|
|
||||||
and inserts logger nodes after each op. Compiles into a separate artifact from
|
|
||||||
the production graph.
|
|
||||||
|
|
||||||
**Off semantics**: zero overhead. The production compile path is untouched.
|
|
||||||
|
|
||||||
**When to add**: if you need to trace inside a `torch.compile`'d graph and
|
|
||||||
Extension 0 is too coarse.
|
|
||||||
|
|
||||||
**Build cost**: roughly 1-2 days. Reference:
|
|
||||||
`torchao.quantization.pt2e._numeric_debugger`.
|
|
||||||
|
|
||||||
### Extension 2: AST source injection at import time
|
|
||||||
|
|
||||||
**Granularity**: per-line (between any two Python statements).
|
|
||||||
|
|
||||||
**Mechanism**: an importlib loader hook rewrites Python source AST at module
|
|
||||||
import time, inserting `if TRACE: dump(...)` statements. The decision is made
|
|
||||||
once at import.
|
|
||||||
|
|
||||||
**Off semantics**: zero overhead. If the env var is off at import time, source
|
|
||||||
is loaded as-is.
|
|
||||||
|
|
||||||
**When to add**: if you need per-line granularity that even FX-node-level can't
|
|
||||||
provide. This is almost never the right choice.
|
|
||||||
|
|
||||||
**Build cost**: roughly 1 week. Brittle and hard to debug.
|
|
||||||
|
|
||||||
### Extension 3: `__torch_dispatch__` / `TorchDispatchMode`
|
|
||||||
|
|
||||||
**Granularity**: per-op (every dispatcher call: matmul, add, view, etc.).
|
|
||||||
|
|
||||||
**Mechanism**: a `TorchDispatchMode` context manager that intercepts all ops at
|
|
||||||
the dispatcher level.
|
|
||||||
|
|
||||||
**Off semantics**: zero overhead. PyTorch's dispatcher only invokes mode hooks
|
|
||||||
when a mode is active.
|
|
||||||
|
|
||||||
**When on**: significant overhead. Every op pays a Python callback cost. Triton
|
|
||||||
kernels bypass it.
|
|
||||||
|
|
||||||
**When to add**: useful for quantization or dtype debugging where module-level
|
|
||||||
granularity isn't enough.
|
|
||||||
|
|
||||||
**Build cost**: roughly 1 day. Reference:
|
|
||||||
`torch.utils._python_dispatch.TorchDispatchMode`.
|
|
||||||
|
|
||||||
## Comparison with similar tools
|
|
||||||
|
|
||||||
| Tool | Pattern | FastVideo equivalent |
|
|
||||||
|---|---|---|
|
|
||||||
| SGLang `--debug-tensor-dump-output-folder` | env-gated forward hooks at startup | Extension 0 (this) |
|
|
||||||
| TransformerEngine `DumpTensors` | config-driven selective dumps | Extension 0 (env-driven) |
|
|
||||||
| HuggingFace `output_hidden_states=True` | source-level boolean gating | Not used; Extension 0 avoids model code edits |
|
|
||||||
| torchao numeric debugger | FX pass + node-level loggers | Extension 1 (future) |
|
|
||||||
| W&B `wandb.watch()` | runtime forward hooks (always on once registered) | Extension 0 has a similar mechanism, but gated off by default |
|
|
||||||
|
|
||||||
## Implementation references
|
|
||||||
|
|
||||||
- Module: `fastvideo/hooks/activation_trace.py`
|
|
||||||
- Env vars: `fastvideo/envs.py` (`FASTVIDEO_TRACE_ACTIVATIONS` and friends)
|
|
||||||
- Pipeline integration: `fastvideo/pipelines/composed_pipeline_base.py`
|
|
||||||
- Tests: `fastvideo/tests/hooks/test_activation_trace.py`
|
|
||||||
- Companion skill (for ad-hoc port investigations): `~/.config/opencode/skill/add-model-trace/`
|
|
||||||
|
|
||||||
## Changelog
|
|
||||||
|
|
||||||
| Date | Change |
|
|
||||||
|---|---|
|
|
||||||
| 2026-05-01 | Initial Extension 0 (module forward hooks) implementation. Extensions 1-3 designed but not implemented. |
|
|
||||||
@@ -24,7 +24,7 @@ PR push
|
|||||||
Runs on the PR branch directly
|
Runs on the PR branch directly
|
||||||
│
|
│
|
||||||
pass ──► Mergify auto-squash-merges to main, branch deleted
|
pass ──► Mergify auto-squash-merges to main, branch deleted
|
||||||
fail ──► fix the regression, push, and /merge again
|
fail ──► Mergify removes 'ready' label; fix and /merge again
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -102,8 +102,8 @@ failing test's output.
|
|||||||
| Performance Tests | `performance` | 30 min |
|
| Performance Tests | `performance` | 30 min |
|
||||||
| API Server Tests | `api_server` | 30 min |
|
| API Server Tests | `api_server` | 30 min |
|
||||||
|
|
||||||
If a Full Suite test fails, check the Buildkite build log for the failing step's output.
|
A Full Suite failure removes the `ready` label automatically. A Mergify comment links to
|
||||||
Fix the regression, push, and comment `/merge` again to re-trigger.
|
the Buildkite build. Fix the regression, push, and comment `/merge` again.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -129,8 +129,8 @@ Suite passing directly on the PR branch.
|
|||||||
- No merge conflicts
|
- No merge conflicts
|
||||||
5. If all conditions pass, Mergify squash-merges to `main` automatically. The branch is
|
5. If all conditions pass, Mergify squash-merges to `main` automatically. The branch is
|
||||||
deleted after merge.
|
deleted after merge.
|
||||||
6. If the Full Suite fails, the developer fixes the issue, pushes, and comments `/merge`
|
6. If the Full Suite fails, Mergify removes the `ready` label and posts a comment linking to
|
||||||
again to re-trigger.
|
the Buildkite build. The developer fixes the issue, pushes, and comments `/merge` again.
|
||||||
|
|
||||||
**Merge conditions summary:**
|
**Merge conditions summary:**
|
||||||
|
|
||||||
@@ -173,7 +173,7 @@ Applied by Mergify based on which paths you modified. Multiple scope labels can
|
|||||||
| Label | File paths that trigger it |
|
| Label | File paths that trigger it |
|
||||||
|-------|---------------------------|
|
|-------|---------------------------|
|
||||||
| `scope: training` | `fastvideo/train/`, `fastvideo/training/`, `fastvideo/distillation/`, `examples/train/`, `examples/training/`, `examples/distill/` |
|
| `scope: training` | `fastvideo/train/`, `fastvideo/training/`, `fastvideo/distillation/`, `examples/train/`, `examples/training/`, `examples/distill/` |
|
||||||
| `scope: inference` | `fastvideo/pipelines/basic/`, `fastvideo/pipelines/stages/`, `fastvideo/pipelines/samplers/`, `fastvideo/entrypoints/`, `fastvideo/worker/`, `fastvideo/api/sampling_param.py`, `fastvideo/configs/pipelines/`, `examples/inference/` |
|
| `scope: inference` | `fastvideo/pipelines/basic/`, `fastvideo/pipelines/stages/`, `fastvideo/pipelines/samplers/`, `fastvideo/entrypoints/`, `fastvideo/worker/`, `fastvideo/configs/sample/`, `fastvideo/configs/pipelines/`, `examples/inference/` |
|
||||||
| `scope: attention` | `fastvideo/attention/` |
|
| `scope: attention` | `fastvideo/attention/` |
|
||||||
| `scope: kernel` | `fastvideo-kernel/`, `csrc/` |
|
| `scope: kernel` | `fastvideo-kernel/`, `csrc/` |
|
||||||
| `scope: data` | `fastvideo/dataset/`, `fastvideo/pipelines/preprocess/`, `examples/preprocessing/` |
|
| `scope: data` | `fastvideo/dataset/`, `fastvideo/pipelines/preprocess/`, `examples/preprocessing/` |
|
||||||
@@ -279,30 +279,6 @@ Triggers a specific Buildkite test or suite on the current PR branch.
|
|||||||
| `/test api` | API server integration tests | `api_server` |
|
| `/test api` | API server integration tests | `api_server` |
|
||||||
| `/test full` | Entire Full Suite | all (with `TEST_SCOPE=full`) |
|
| `/test full` | Entire Full Suite | all (with `TEST_SCOPE=full`) |
|
||||||
| `/test fastcheck` | Entire Fastcheck suite | fastcheck (with `TEST_SCOPE=fastcheck`) |
|
| `/test fastcheck` | Entire Fastcheck suite | fastcheck (with `TEST_SCOPE=fastcheck`) |
|
||||||
| `/test pre-commit` | Pre-commit checks on PR code | — (runs `ci-precommit.yml` via `workflow_call`) |
|
|
||||||
|
|
||||||
**Re-running failed tests:** When you use `/test <name>` to re-run a specific failed test,
|
|
||||||
the resulting Buildkite check uses the same name as the original (e.g., `/test encoder`
|
|
||||||
creates `buildkite/ci/microscope-encoder-tests`). This overwrites the failed check status.
|
|
||||||
Once all tests in a tier pass, the aggregate status (`fastcheck-passed` or
|
|
||||||
`full-suite-passed`) is automatically updated to `success` by the `ci-aggregate-status.yml`
|
|
||||||
workflow.
|
|
||||||
|
|
||||||
**How aggregate status refresh works:**
|
|
||||||
|
|
||||||
1. `/test <name>` triggers a Buildkite build with `TEST_SCOPE=direct`. The test step uses
|
|
||||||
the same label as its fastcheck/full-suite counterpart, so the resulting GitHub check
|
|
||||||
overwrites the original.
|
|
||||||
2. When the build completes, Buildkite's `notify` posts a `direct-test-completed` commit
|
|
||||||
status. This is the only signal that triggers the aggregate workflow — intermediate step
|
|
||||||
status updates do not trigger it.
|
|
||||||
3. `ci-aggregate-status.yml` fires, calls `getCombinedStatusForRef` to fetch the latest
|
|
||||||
status for every context on that commit (each context returns only its most recent
|
|
||||||
state), groups them by prefix (`microscope-*` → fastcheck, `test-tube-*`/`bar-chart-*`
|
|
||||||
→ full suite), and posts `fastcheck-passed: success` or `full-suite-passed: success` if
|
|
||||||
all entries in the group are `success`.
|
|
||||||
4. Tests that were never triggered (skipped by monorepo-diff) have no status entry and do
|
|
||||||
not block the aggregate.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -320,7 +296,6 @@ Protected branches (`main`, `master`, `release/*`) are never deleted.
|
|||||||
| `ci-precommit.yml` | Every push / PR against `main` | Runs pre-commit hooks (yapf, ruff, mypy, codespell, pymarkdown, actionlint, check-filenames) |
|
| `ci-precommit.yml` | Every push / PR against `main` | Runs pre-commit hooks (yapf, ruff, mypy, codespell, pymarkdown, actionlint, check-filenames) |
|
||||||
| `ci-trigger-full-suite.yml` | `ready` label added to a PR | Calls Buildkite API to run Full Suite on the PR branch |
|
| `ci-trigger-full-suite.yml` | `ready` label added to a PR | Calls Buildkite API to run Full Suite on the PR branch |
|
||||||
| `ci-slash-commands.yml` | PR comment starting with `/merge` or `/test` | Handles slash commands; adds `ready` label or triggers Buildkite |
|
| `ci-slash-commands.yml` | PR comment starting with `/merge` or `/test` | Handles slash commands; adds `ready` label or triggers Buildkite |
|
||||||
| `ci-aggregate-status.yml` | Any Buildkite commit status update | Checks if all tests in a tier passed; updates `fastcheck-passed` or `full-suite-passed` |
|
|
||||||
| `community-issue-labeler.yml` | Issue opened or edited | Auto-labels issues by keyword matching against title and body |
|
| `community-issue-labeler.yml` | Issue opened or edited | Auto-labels issues by keyword matching against title and body |
|
||||||
| `community-welcome.yml` | First contribution | Posts a welcome comment for first-time contributors |
|
| `community-welcome.yml` | First contribution | Posts a welcome comment for first-time contributors |
|
||||||
| `community-stale.yml` | Scheduled | Marks and closes stale issues and PRs |
|
| `community-stale.yml` | Scheduled | Marks and closes stale issues and PRs |
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ FastVideo maps a Diffusers-style repo into a pipeline like:
|
|||||||
- `fastvideo/configs/models/*`: arch configs and `param_names_mapping` for
|
- `fastvideo/configs/models/*`: arch configs and `param_names_mapping` for
|
||||||
weight name translation.
|
weight name translation.
|
||||||
- `fastvideo/configs/pipelines/*`: pipeline wiring (component classes + names).
|
- `fastvideo/configs/pipelines/*`: pipeline wiring (component classes + names).
|
||||||
- `fastvideo/api/sampling_param.py`: runtime sampling parameters.
|
- `fastvideo/configs/sample/*`: default runtime sampling parameters.
|
||||||
- `fastvideo/pipelines/basic/*`: end-to-end pipeline logic built from stages.
|
- `fastvideo/pipelines/basic/*`: end-to-end pipeline logic built from stages.
|
||||||
- `model_index.json`: the HF repo entrypoint that maps component names to
|
- `model_index.json`: the HF repo entrypoint that maps component names to
|
||||||
classes and weight files.
|
classes and weight files.
|
||||||
@@ -55,7 +55,7 @@ Minimal usage example (based on `examples/inference/basic/basic.py`):
|
|||||||
|
|
||||||
```python
|
```python
|
||||||
from fastvideo import VideoGenerator
|
from fastvideo import VideoGenerator
|
||||||
from fastvideo.api.sampling_param import SamplingParam
|
from fastvideo.configs.sample import SamplingParam
|
||||||
|
|
||||||
model_id = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" # or official_weights/<model_name>/
|
model_id = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" # or official_weights/<model_name>/
|
||||||
generator = VideoGenerator.from_pretrained(model_id, num_gpus=1)
|
generator = VideoGenerator.from_pretrained(model_id, num_gpus=1)
|
||||||
@@ -296,10 +296,8 @@ Action:
|
|||||||
|
|
||||||
- Add or reuse a numerical parity test that loads the official model and the
|
- Add or reuse a numerical parity test that loads the official model and the
|
||||||
FastVideo model and compares outputs.
|
FastVideo model and compares outputs.
|
||||||
- See examples in `tests/local_tests/` organized by model family
|
- See examples in `tests/local_tests/` (e.g., `tests/local_tests/upsamplers/`)
|
||||||
(e.g., `tests/local_tests/sd35/`, `tests/local_tests/ltx2/`,
|
and the commands in `tests/local_tests/README.md`.
|
||||||
`tests/local_tests/stable_audio/`) and the navigation index in
|
|
||||||
`tests/local_tests/README.md`.
|
|
||||||
- If there are discrepancies, add opt‑in logging to both models and compare
|
- If there are discrepancies, add opt‑in logging to both models and compare
|
||||||
activation summaries (layer output sums, per‑stage logs).
|
activation summaries (layer output sums, per‑stage logs).
|
||||||
- First align the loaded weights (validate `param_names_mapping`).
|
- First align the loaded weights (validate `param_names_mapping`).
|
||||||
@@ -321,8 +319,7 @@ Purpose:
|
|||||||
|
|
||||||
- `fastvideo/configs/pipelines/` describes pipeline wiring and model module
|
- `fastvideo/configs/pipelines/` describes pipeline wiring and model module
|
||||||
names.
|
names.
|
||||||
- `fastvideo/api/sampling_param.py` defines runtime sampling parameters.
|
- `fastvideo/configs/sample/` defines default runtime parameters.
|
||||||
Defaults come from profiles in `fastvideo/pipelines/basic/<family>/profiles.py`.
|
|
||||||
|
|
||||||
Action:
|
Action:
|
||||||
|
|
||||||
@@ -350,8 +347,7 @@ Purpose:
|
|||||||
|
|
||||||
Action:
|
Action:
|
||||||
|
|
||||||
- Add a pipeline parity test under `tests/local_tests/<family>/`
|
- Add a pipeline parity test under `tests/local_tests/pipelines/`.
|
||||||
(e.g., `tests/local_tests/<family>/test_<family>_pipeline_parity.py`).
|
|
||||||
- See the [Testing Guide](testing.md) for test conventions.
|
- See the [Testing Guide](testing.md) for test conventions.
|
||||||
|
|
||||||
### 7) Add user‑facing examples
|
### 7) Add user‑facing examples
|
||||||
@@ -478,7 +474,7 @@ FastVideo integration.
|
|||||||
3. Pipeline wiring.
|
3. Pipeline wiring.
|
||||||
- Pipeline: `fastvideo/pipelines/basic/wan/wan_pipeline.py`
|
- Pipeline: `fastvideo/pipelines/basic/wan/wan_pipeline.py`
|
||||||
- Pipeline config: `fastvideo/configs/pipelines/wan.py`
|
- Pipeline config: `fastvideo/configs/pipelines/wan.py`
|
||||||
- Sampling defaults: `fastvideo/pipelines/basic/wan/profiles.py`
|
- Sampling defaults: `fastvideo/configs/sample/wan.py`
|
||||||
|
|
||||||
4. Minimal example.
|
4. Minimal example.
|
||||||
- Script: `examples/inference/basic/basic.py`
|
- Script: `examples/inference/basic/basic.py`
|
||||||
|
|||||||
@@ -99,7 +99,7 @@ cd /FastVideo
|
|||||||
**Install the package**
|
**Install the package**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv pip install -e ".[dev]"
|
uv pip install -e .[dev]
|
||||||
```
|
```
|
||||||
|
|
||||||
The Docker image already includes Flash Attention and most heavy dependencies, so this is fast.
|
The Docker image already includes Flash Attention and most heavy dependencies, so this is fast.
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ git clone https://github.com/hao-ai-lab/FastVideo.git && cd FastVideo
|
|||||||
Install FastVideo in editable mode and set up hooks:
|
Install FastVideo in editable mode and set up hooks:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv pip install -e ".[dev]"
|
uv pip install -e .[dev]
|
||||||
|
|
||||||
# Optional: FlashAttention (builds native kernels)
|
# Optional: FlashAttention (builds native kernels)
|
||||||
uv pip install flash-attn --no-build-isolation -v
|
uv pip install flash-attn --no-build-isolation -v
|
||||||
|
|||||||
@@ -104,9 +104,8 @@ distillation, self-forcing, VSA, VMoBA, performance benchmarks, and API server t
|
|||||||
8. If all Full Suite tests pass and all merge conditions are met (approval, valid title,
|
8. If all Full Suite tests pass and all merge conditions are met (approval, valid title,
|
||||||
pre-commit green, fastcheck green, no draft, no conflicts), Mergify squash-merges to
|
pre-commit green, fastcheck green, no draft, no conflicts), Mergify squash-merges to
|
||||||
`main` automatically. Your branch is deleted.
|
`main` automatically. Your branch is deleted.
|
||||||
9. If a Full Suite test fails, check the Buildkite build log for the failing step. Fix the
|
9. If a Full Suite test fails, Mergify removes the `ready` label and posts a comment with a
|
||||||
issue, push, and comment `/merge` again. You can also re-run individual failed tests
|
link to the Buildkite build. Fix the issue, push, and comment `/merge` again.
|
||||||
with `/test <name>` — see below.
|
|
||||||
|
|
||||||
!!! note
|
!!! note
|
||||||
Only contributors with write permission to the repository can trigger slash commands.
|
Only contributors with write permission to the repository can trigger slash commands.
|
||||||
@@ -150,15 +149,10 @@ Comment on your PR to trigger specific tests independently of the auto-merge flo
|
|||||||
/test vmoba # VMoBA inference tests
|
/test vmoba # VMoBA inference tests
|
||||||
/test performance # Performance benchmarks
|
/test performance # Performance benchmarks
|
||||||
/test api # API server integration tests
|
/test api # API server integration tests
|
||||||
/test pre-commit # Pre-commit checks on PR code
|
|
||||||
```
|
```
|
||||||
|
|
||||||
The workflow reacts with a 🚀 emoji to confirm the command was received.
|
The workflow reacts with a 🚀 emoji to confirm the command was received.
|
||||||
|
|
||||||
When you re-run an individual test with `/test <name>`, the new result overwrites the
|
|
||||||
original failed check (same Buildkite check name). Once all tests in a tier pass, the
|
|
||||||
`fastcheck-passed` or `full-suite-passed` status is automatically updated.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Troubleshooting
|
## Troubleshooting
|
||||||
@@ -205,8 +199,9 @@ Mergify removes the `needs-rebase` label automatically once conflicts are resolv
|
|||||||
|
|
||||||
### Full Suite failed after `/merge`
|
### Full Suite failed after `/merge`
|
||||||
|
|
||||||
The Full Suite found a regression. Check the failing Buildkite step's output for assertion
|
The Full Suite found a regression. Mergify removes the `ready` label and posts a comment
|
||||||
errors or tracebacks.
|
linking to the Buildkite build. Check the failing step's output for assertion errors or
|
||||||
|
tracebacks.
|
||||||
|
|
||||||
Common causes:
|
Common causes:
|
||||||
|
|
||||||
|
|||||||
@@ -1,480 +0,0 @@
|
|||||||
status_definitions:
|
|
||||||
kept: "Public field remains on a public adapter surface with the same meaning."
|
|
||||||
moved: "Public field remains supported but normalizes into a different nested path."
|
|
||||||
preset_owned: "Public field remains supported only through a model/preset-specific surface."
|
|
||||||
compatibility_only: "Legacy public field remains adapter-only during migration and is not part of the canonical typed schema."
|
|
||||||
private_only: "Field should only be handled by private adapters and is not a public FastVideo compatibility promise."
|
|
||||||
internal_only: "Field is runtime/config plumbing and should not be part of the new public typed inference API."
|
|
||||||
|
|
||||||
surfaces:
|
|
||||||
fastvideo_args:
|
|
||||||
moved:
|
|
||||||
model_path: generator.model_path
|
|
||||||
workload_type: generator.pipeline.workload_type
|
|
||||||
distributed_executor_backend: generator.engine.execution_backend
|
|
||||||
trust_remote_code: generator.trust_remote_code
|
|
||||||
revision: generator.revision
|
|
||||||
num_gpus: generator.engine.num_gpus
|
|
||||||
tp_size: generator.engine.parallelism.tp_size
|
|
||||||
sp_size: generator.engine.parallelism.sp_size
|
|
||||||
hsdp_replicate_dim: generator.engine.parallelism.hsdp_replicate_dim
|
|
||||||
hsdp_shard_dim: generator.engine.parallelism.hsdp_shard_dim
|
|
||||||
dist_timeout: generator.engine.parallelism.dist_timeout
|
|
||||||
lora_path: generator.pipeline.components.lora_path
|
|
||||||
dit_cpu_offload: generator.engine.offload.dit
|
|
||||||
use_fsdp_inference: generator.engine.use_fsdp_inference
|
|
||||||
dit_layerwise_offload: generator.engine.offload.dit_layerwise
|
|
||||||
text_encoder_cpu_offload: generator.engine.offload.text_encoder
|
|
||||||
image_encoder_cpu_offload: generator.engine.offload.image_encoder
|
|
||||||
vae_cpu_offload: generator.engine.offload.vae
|
|
||||||
pin_cpu_memory: generator.engine.offload.pin_cpu_memory
|
|
||||||
enable_torch_compile: generator.engine.compile.enabled
|
|
||||||
torch_compile_kwargs: generator.engine.compile.backend,fullgraph,mode,dynamic,extras
|
|
||||||
disable_autocast: generator.engine.disable_autocast
|
|
||||||
enable_stage_verification: generator.engine.enable_stage_verification
|
|
||||||
prompt_txt: request.inputs.prompt_path
|
|
||||||
override_text_encoder_safetensors: generator.pipeline.components.text_encoder_weights
|
|
||||||
override_text_encoder_quant: generator.engine.quantization.text_encoder_quant
|
|
||||||
override_transformer_cls_name: generator.pipeline.components.override_transformer_cls_name
|
|
||||||
init_weights_from_safetensors: generator.pipeline.components.transformer_weights
|
|
||||||
init_weights_from_safetensors_2: generator.pipeline.components.transformer_2_weights
|
|
||||||
override_pipeline_cls_name: generator.pipeline.components.override_pipeline_cls_name
|
|
||||||
boundary_ratio: request.sampling.boundary_ratio
|
|
||||||
ltx2_vae_tiling: generator.pipeline.vae_tiling
|
|
||||||
preset_owned:
|
|
||||||
ltx2_vae_spatial_tile_size_in_pixels: generator.pipeline.preset_overrides.ltx2.vae.spatial_tile_size_in_pixels
|
|
||||||
ltx2_vae_spatial_tile_overlap_in_pixels: generator.pipeline.preset_overrides.ltx2.vae.spatial_tile_overlap_in_pixels
|
|
||||||
ltx2_vae_temporal_tile_size_in_frames: generator.pipeline.preset_overrides.ltx2.vae.temporal_tile_size_in_frames
|
|
||||||
ltx2_vae_temporal_tile_overlap_in_frames: generator.pipeline.preset_overrides.ltx2.vae.temporal_tile_overlap_in_frames
|
|
||||||
ltx2_initial_latent_path: request.extensions.ltx2.initial_latent_path
|
|
||||||
compatibility_only:
|
|
||||||
mode: "Legacy multi-mode FastVideoArgs switch; typed inference config should not expose execution mode."
|
|
||||||
inference_mode: "Legacy boolean mirror of mode; kept only through adapters while FastVideoArgs remains."
|
|
||||||
lora_nickname: "Legacy adapter-selection surface pending LoRA API cleanup."
|
|
||||||
lora_target_modules: "Legacy LoRA configuration surface pending dedicated component API."
|
|
||||||
output_type: "Legacy output formatting surface pending GenerationResult cleanup."
|
|
||||||
VSA_sparsity: "Model-specific inference optimization not yet represented in the typed public schema."
|
|
||||||
moba_config_path: "Model-specific MoBA optimization surface not yet represented in the typed public schema."
|
|
||||||
master_port: "Executor/bootstrap compatibility field; not part of the canonical inference schema."
|
|
||||||
private_only:
|
|
||||||
ray_placement_group: "Ray deployment-only field."
|
|
||||||
ray_runtime_env: "Ray deployment-only field."
|
|
||||||
internal_only:
|
|
||||||
pipeline_config: "Legacy internal carrier object."
|
|
||||||
preprocess_config: "Legacy preprocess carrier object."
|
|
||||||
moba_config: "Derived runtime config loaded from moba_config_path."
|
|
||||||
model_paths: "Runtime bookkeeping."
|
|
||||||
model_loaded: "Runtime bookkeeping."
|
|
||||||
|
|
||||||
pipeline_config_base:
|
|
||||||
moved:
|
|
||||||
pipeline_config_path: generator.pipeline.components.pipeline_config_path
|
|
||||||
preset_owned:
|
|
||||||
embedded_cfg_scale: generator.pipeline.preset_overrides.embedded_cfg_scale
|
|
||||||
flow_shift: generator.pipeline.preset_overrides.flow_shift
|
|
||||||
flow_shift_sr: generator.pipeline.preset_overrides.flow_shift_sr
|
|
||||||
is_causal: generator.pipeline.preset_overrides.is_causal
|
|
||||||
vae_tiling: generator.pipeline.preset_overrides.vae_tiling
|
|
||||||
vae_sp: generator.pipeline.preset_overrides.vae_sp
|
|
||||||
dmd_denoising_steps: generator.pipeline.preset_overrides.dmd_denoising_steps
|
|
||||||
ti2v_task: generator.pipeline.preset_overrides.ti2v_task
|
|
||||||
boundary_ratio: generator.pipeline.preset_overrides.boundary_ratio
|
|
||||||
compatibility_only:
|
|
||||||
model_path: "Redundant with generator.model_path."
|
|
||||||
disable_autocast: "Duplicated by generator.engine.disable_autocast during migration."
|
|
||||||
dit_precision: "Precision override pending dedicated typed component precision design."
|
|
||||||
upsampler_precision: "Precision override pending dedicated typed component precision design."
|
|
||||||
vae_precision: "Precision override pending dedicated typed component precision design."
|
|
||||||
image_encoder_precision: "Precision override pending dedicated typed component precision design."
|
|
||||||
text_encoder_precisions: "Precision override pending dedicated typed component precision design."
|
|
||||||
internal_only:
|
|
||||||
dit_config: "Legacy internal component config object."
|
|
||||||
upsampler_config: "Legacy internal component config object."
|
|
||||||
vae_config: "Legacy internal component config object."
|
|
||||||
image_encoder_config: "Legacy internal component config object."
|
|
||||||
text_encoder_configs: "Legacy internal component config object."
|
|
||||||
preprocess_text_funcs: "Internal text preprocessing hooks."
|
|
||||||
postprocess_text_funcs: "Internal text postprocessing hooks."
|
|
||||||
|
|
||||||
pipeline_config_extensions:
|
|
||||||
preset_owned:
|
|
||||||
conditioning_strategy:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.cosmos.CosmosConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
max_num_conditional_frames:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.cosmos.CosmosConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
min_num_conditional_frames:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.cosmos.CosmosConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
sigma_conditional:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.cosmos.CosmosConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
sigma_data:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.cosmos.CosmosConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
state_ch:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.cosmos.CosmosConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
state_t:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.cosmos.CosmosConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
text_encoder_class:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.cosmos.CosmosConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
autoregressive_chunk_frames:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
autoregressive_overlap_frames:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
cfg_behavior:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
default_camera_rotation:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
default_movement_distance:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
default_negative_prompt:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
default_trajectory_type:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
filter_points_threshold:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
fps:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
frame_buffer_max:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
moge_model_name:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
noise_aug_strength:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
num_frames:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
offload_moge_after_depth:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
use_moge_depth:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
video_resolution:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CConfig
|
|
||||||
- fastvideo.configs.pipelines.gen3c.Gen3CInferenceConfig
|
|
||||||
text_encoder_crop_start:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15I2V480PStepDistilledConfig
|
|
||||||
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15I2V720PConfig
|
|
||||||
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15SR1080PConfig
|
|
||||||
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15T2V480PConfig
|
|
||||||
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15T2V720PConfig
|
|
||||||
- fastvideo.configs.pipelines.hyworld.HYWorldConfig
|
|
||||||
- fastvideo.configs.pipelines.hyworld.Hunyuan15T2V480PConfig
|
|
||||||
text_encoder_max_lengths:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15I2V480PStepDistilledConfig
|
|
||||||
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15I2V720PConfig
|
|
||||||
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15SR1080PConfig
|
|
||||||
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15T2V480PConfig
|
|
||||||
- fastvideo.configs.pipelines.hunyuan15.Hunyuan15T2V720PConfig
|
|
||||||
- fastvideo.configs.pipelines.hyworld.HYWorldConfig
|
|
||||||
- fastvideo.configs.pipelines.hyworld.Hunyuan15T2V480PConfig
|
|
||||||
precision:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.lingbotworld.LingBotWorldI2V480PConfig
|
|
||||||
- fastvideo.configs.pipelines.lingbotworld.Wan2_2_I2V_A14B_Config
|
|
||||||
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionI2VConfig
|
|
||||||
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionI2V_A14B_Config
|
|
||||||
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionT2VConfig
|
|
||||||
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionT2V_14B_Config
|
|
||||||
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionT2V_1_3B_Config
|
|
||||||
- fastvideo.configs.pipelines.wan.FastWan2_1_T2V_480P_Config
|
|
||||||
- fastvideo.configs.pipelines.wan.FastWan2_2_TI2V_5B_Config
|
|
||||||
- fastvideo.configs.pipelines.wan.MatrixGameBaseI2V480PConfig
|
|
||||||
- fastvideo.configs.pipelines.wan.MatrixGameI2V480PConfig
|
|
||||||
- fastvideo.configs.pipelines.wan.SelfForcingWan2_2_T2V480PConfig
|
|
||||||
- fastvideo.configs.pipelines.wan.SelfForcingWanT2V480PConfig
|
|
||||||
- fastvideo.configs.pipelines.wan.WANV2VConfig
|
|
||||||
- fastvideo.configs.pipelines.wan.Wan2_2_I2V_A14B_Config
|
|
||||||
- fastvideo.configs.pipelines.wan.Wan2_2_T2V_A14B_Config
|
|
||||||
- fastvideo.configs.pipelines.wan.Wan2_2_TI2V_5B_Config
|
|
||||||
- fastvideo.configs.pipelines.wan.WanI2V480PConfig
|
|
||||||
- fastvideo.configs.pipelines.wan.WanI2V720PConfig
|
|
||||||
- fastvideo.configs.pipelines.wan.WanT2V480PConfig
|
|
||||||
- fastvideo.configs.pipelines.wan.WanT2V720PConfig
|
|
||||||
warp_denoising_step:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.lingbotworld.LingBotWorldI2V480PConfig
|
|
||||||
- fastvideo.configs.pipelines.lingbotworld.Wan2_2_I2V_A14B_Config
|
|
||||||
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionI2VConfig
|
|
||||||
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionI2V_A14B_Config
|
|
||||||
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionT2VConfig
|
|
||||||
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionT2V_14B_Config
|
|
||||||
- fastvideo.configs.pipelines.turbodiffusion.TurboDiffusionT2V_1_3B_Config
|
|
||||||
- fastvideo.configs.pipelines.wan.FastWan2_1_T2V_480P_Config
|
|
||||||
- fastvideo.configs.pipelines.wan.FastWan2_2_TI2V_5B_Config
|
|
||||||
- fastvideo.configs.pipelines.wan.MatrixGameBaseI2V480PConfig
|
|
||||||
- fastvideo.configs.pipelines.wan.MatrixGameI2V480PConfig
|
|
||||||
- fastvideo.configs.pipelines.wan.SelfForcingWan2_2_T2V480PConfig
|
|
||||||
- fastvideo.configs.pipelines.wan.SelfForcingWanT2V480PConfig
|
|
||||||
- fastvideo.configs.pipelines.wan.WANV2VConfig
|
|
||||||
- fastvideo.configs.pipelines.wan.Wan2_2_I2V_A14B_Config
|
|
||||||
- fastvideo.configs.pipelines.wan.Wan2_2_T2V_A14B_Config
|
|
||||||
- fastvideo.configs.pipelines.wan.Wan2_2_TI2V_5B_Config
|
|
||||||
- fastvideo.configs.pipelines.wan.WanI2V480PConfig
|
|
||||||
- fastvideo.configs.pipelines.wan.WanI2V720PConfig
|
|
||||||
- fastvideo.configs.pipelines.wan.WanT2V480PConfig
|
|
||||||
- fastvideo.configs.pipelines.wan.WanT2V720PConfig
|
|
||||||
bsa_cdf_threshold:
|
|
||||||
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
|
|
||||||
bsa_chunk_k:
|
|
||||||
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
|
|
||||||
bsa_chunk_q:
|
|
||||||
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
|
|
||||||
bsa_params:
|
|
||||||
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
|
|
||||||
bsa_sparsity:
|
|
||||||
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
|
|
||||||
enable_bsa:
|
|
||||||
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
|
|
||||||
enable_kv_cache:
|
|
||||||
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
|
|
||||||
enhance_hf:
|
|
||||||
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
|
|
||||||
offload_kv_cache:
|
|
||||||
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
|
|
||||||
t_thresh:
|
|
||||||
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
|
|
||||||
use_distill:
|
|
||||||
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
|
|
||||||
scheduler_arch:
|
|
||||||
sources: [fastvideo.configs.pipelines.sd35.SD35Config]
|
|
||||||
text_encoder_archs:
|
|
||||||
sources: [fastvideo.configs.pipelines.sd35.SD35Config]
|
|
||||||
tokenizer_archs:
|
|
||||||
sources: [fastvideo.configs.pipelines.sd35.SD35Config]
|
|
||||||
transformer_arch:
|
|
||||||
sources: [fastvideo.configs.pipelines.sd35.SD35Config]
|
|
||||||
vae_arch:
|
|
||||||
sources: [fastvideo.configs.pipelines.sd35.SD35Config]
|
|
||||||
expand_timesteps:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.wan.FastWan2_2_TI2V_5B_Config
|
|
||||||
- fastvideo.configs.pipelines.wan.Wan2_2_TI2V_5B_Config
|
|
||||||
context_noise:
|
|
||||||
sources: [fastvideo.configs.pipelines.wan.MatrixGameI2V480PConfig]
|
|
||||||
num_frames_per_block:
|
|
||||||
sources: [fastvideo.configs.pipelines.wan.MatrixGameI2V480PConfig]
|
|
||||||
audio_channels:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.stable_audio.StableAudioT2AConfig
|
|
||||||
- fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig
|
|
||||||
audio_end_in_s:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.stable_audio.StableAudioT2AConfig
|
|
||||||
- fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig
|
|
||||||
audio_start_in_s:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.stable_audio.StableAudioT2AConfig
|
|
||||||
- fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig
|
|
||||||
max_audio_duration_s:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.stable_audio.StableAudioT2AConfig
|
|
||||||
- fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig
|
|
||||||
sample_size:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.stable_audio.StableAudioT2AConfig
|
|
||||||
- fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig
|
|
||||||
sampling_rate:
|
|
||||||
sources:
|
|
||||||
- fastvideo.configs.pipelines.stable_audio.StableAudioT2AConfig
|
|
||||||
- fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig
|
|
||||||
compatibility_only:
|
|
||||||
batch_size: "Gen3C inference-only tuning field pending typed batching design."
|
|
||||||
gradient_checkpointing: "Gen3C inference-only compatibility field pending typed batching design."
|
|
||||||
guidance_scale: "Gen3C pipeline-level default pending preset/default-request cleanup."
|
|
||||||
num_inference_steps: "Gen3C pipeline-level default pending preset/default-request cleanup."
|
|
||||||
internal_only:
|
|
||||||
audio_decoder_config: "Legacy internal component config object."
|
|
||||||
audio_decoder_precision: "Precision override pending dedicated component precision design."
|
|
||||||
vocoder_config: "Legacy internal component config object."
|
|
||||||
vocoder_precision: "Precision override pending dedicated component precision design."
|
|
||||||
|
|
||||||
sampling_param_base:
|
|
||||||
moved:
|
|
||||||
image_path: request.inputs.image_path
|
|
||||||
pil_image: request.inputs.pil_image
|
|
||||||
video_path: request.inputs.video_path
|
|
||||||
mouse_cond: request.inputs.mouse_cond
|
|
||||||
keyboard_cond: request.inputs.keyboard_cond
|
|
||||||
grid_sizes: request.inputs.grid_sizes
|
|
||||||
pose: request.inputs.pose
|
|
||||||
c2ws_plucker_emb: request.inputs.c2ws_plucker_emb
|
|
||||||
refine_from: request.inputs.refine_from
|
|
||||||
stage1_video: request.inputs.stage1_video
|
|
||||||
prompt: request.prompt
|
|
||||||
negative_prompt: request.negative_prompt
|
|
||||||
prompt_path: request.inputs.prompt_path
|
|
||||||
output_path: request.output.output_path
|
|
||||||
output_video_name: request.output.output_video_name
|
|
||||||
num_videos_per_prompt: request.sampling.num_videos_per_prompt
|
|
||||||
seed: request.sampling.seed
|
|
||||||
num_frames: request.sampling.num_frames
|
|
||||||
height: request.sampling.height
|
|
||||||
width: request.sampling.width
|
|
||||||
height_sr: request.sampling.height_sr
|
|
||||||
width_sr: request.sampling.width_sr
|
|
||||||
fps: request.sampling.fps
|
|
||||||
num_inference_steps: request.sampling.num_inference_steps
|
|
||||||
num_inference_steps_sr: request.sampling.num_inference_steps_sr
|
|
||||||
guidance_scale: request.sampling.guidance_scale
|
|
||||||
guidance_scale_2: request.sampling.guidance_scale_2
|
|
||||||
guidance_rescale: request.sampling.guidance_rescale
|
|
||||||
boundary_ratio: request.sampling.boundary_ratio
|
|
||||||
sigmas: request.sampling.sigmas
|
|
||||||
enable_teacache: request.runtime.enable_teacache
|
|
||||||
save_video: request.output.save_video
|
|
||||||
return_frames: request.output.return_frames
|
|
||||||
return_trajectory_latents: request.runtime.return_trajectory_latents
|
|
||||||
return_trajectory_decoded: request.runtime.return_trajectory_decoded
|
|
||||||
continuation_state: request.state
|
|
||||||
return_continuation_state: request.output.return_state
|
|
||||||
preset_owned:
|
|
||||||
t_thresh: request.stage_overrides.refine.t_thresh
|
|
||||||
spatial_refine_only: request.stage_overrides.refine.spatial_refine_only
|
|
||||||
num_cond_frames: request.stage_overrides.refine.num_cond_frames
|
|
||||||
trajectory_type: request.extensions.gen3c.trajectory_type
|
|
||||||
movement_distance: request.extensions.gen3c.movement_distance
|
|
||||||
camera_rotation: request.extensions.gen3c.camera_rotation
|
|
||||||
prompt_attention_mask: request.extensions.hyworld.prompt_attention_mask
|
|
||||||
negative_attention_mask: request.extensions.hyworld.negative_attention_mask
|
|
||||||
camera_states: request.extensions.hunyuangamecraft.camera_states
|
|
||||||
camera_trajectory: request.extensions.hunyuangamecraft.camera_trajectory
|
|
||||||
action_list: request.extensions.hunyuangamecraft.action_list
|
|
||||||
action_speed_list: request.extensions.hunyuangamecraft.action_speed_list
|
|
||||||
gt_latents: request.extensions.hunyuangamecraft.gt_latents
|
|
||||||
conditioning_mask: request.extensions.hunyuangamecraft.conditioning_mask
|
|
||||||
ltx2_cfg_scale_video: request.extensions.ltx2.cfg_scale_video
|
|
||||||
ltx2_cfg_scale_audio: request.extensions.ltx2.cfg_scale_audio
|
|
||||||
ltx2_modality_scale_video: request.extensions.ltx2.modality_scale_video
|
|
||||||
ltx2_modality_scale_audio: request.extensions.ltx2.modality_scale_audio
|
|
||||||
ltx2_rescale_scale: request.extensions.ltx2.rescale_scale
|
|
||||||
ltx2_stg_scale_video: request.extensions.ltx2.stg_scale_video
|
|
||||||
ltx2_stg_scale_audio: request.extensions.ltx2.stg_scale_audio
|
|
||||||
ltx2_stg_blocks_video: request.extensions.ltx2.stg_blocks_video
|
|
||||||
ltx2_stg_blocks_audio: request.extensions.ltx2.stg_blocks_audio
|
|
||||||
audio_start_in_s: request.extensions.stable_audio.audio_start_in_s
|
|
||||||
audio_end_in_s: request.extensions.stable_audio.audio_end_in_s
|
|
||||||
init_audio: request.extensions.stable_audio.init_audio
|
|
||||||
init_audio_strength: request.extensions.stable_audio.init_audio_strength
|
|
||||||
init_noise_level: request.extensions.stable_audio.init_noise_level
|
|
||||||
inpaint_audio: request.extensions.stable_audio.inpaint_audio
|
|
||||||
inpaint_mask: request.extensions.stable_audio.inpaint_mask
|
|
||||||
internal_only:
|
|
||||||
data_type: "Derived from the request shape and not a public input."
|
|
||||||
|
|
||||||
sampling_param_extensions: {}
|
|
||||||
|
|
||||||
openai_image_request:
|
|
||||||
kept:
|
|
||||||
model: "HTTP adapter model-routing field."
|
|
||||||
response_format: "HTTP adapter response formatting field."
|
|
||||||
output_format: "HTTP adapter output-format field."
|
|
||||||
background: "HTTP adapter output-format field."
|
|
||||||
quality: "Compatibility field currently accepted by the adapter."
|
|
||||||
style: "Compatibility field currently accepted by the adapter."
|
|
||||||
user: "Compatibility field currently accepted by the adapter."
|
|
||||||
moved:
|
|
||||||
prompt: request.prompt
|
|
||||||
n: request.sampling.num_videos_per_prompt
|
|
||||||
size:
|
|
||||||
target: request.sampling.width,height
|
|
||||||
note: "Adapter parses OpenAI size strings as WIDTHxHEIGHT and forwards width then height."
|
|
||||||
num_inference_steps: request.sampling.num_inference_steps
|
|
||||||
guidance_scale: request.sampling.guidance_scale
|
|
||||||
true_cfg_scale: request.sampling.true_cfg_scale
|
|
||||||
seed: request.sampling.seed
|
|
||||||
negative_prompt: request.negative_prompt
|
|
||||||
enable_teacache: request.runtime.enable_teacache
|
|
||||||
|
|
||||||
openai_video_request:
|
|
||||||
kept:
|
|
||||||
model: "HTTP adapter model-routing field."
|
|
||||||
moved:
|
|
||||||
prompt: request.prompt
|
|
||||||
input_reference: request.inputs.image_path
|
|
||||||
reference_url: request.inputs.image_path
|
|
||||||
size:
|
|
||||||
target: request.sampling.width,height
|
|
||||||
note: "Adapter parses OpenAI size strings as WIDTHxHEIGHT and forwards width then height."
|
|
||||||
fps: request.sampling.fps
|
|
||||||
num_frames: request.sampling.num_frames
|
|
||||||
seed: request.sampling.seed
|
|
||||||
num_inference_steps: request.sampling.num_inference_steps
|
|
||||||
guidance_scale: request.sampling.guidance_scale
|
|
||||||
guidance_scale_2: request.sampling.guidance_scale_2
|
|
||||||
true_cfg_scale: request.sampling.true_cfg_scale
|
|
||||||
negative_prompt: request.negative_prompt
|
|
||||||
enable_teacache: request.runtime.enable_teacache
|
|
||||||
output_path: request.output.output_path
|
|
||||||
compatibility_only:
|
|
||||||
seconds:
|
|
||||||
target: request.sampling.num_frames
|
|
||||||
note: "HTTP adapter duration convenience field. If num_frames is omitted, the adapter computes num_frames = fps * seconds."
|
|
||||||
|
|
||||||
cli:
|
|
||||||
notes:
|
|
||||||
- "CLI parity is checked against the actual generate/serve parser dest sets."
|
|
||||||
- "The inventory tracks parser dest names, excluding argparse's implicit help action."
|
|
||||||
- "The refactored inference CLI is config-only: subcommands expose only --config, and any additional CLI input must use dotted override paths."
|
|
||||||
generate:
|
|
||||||
explicit_local_fields:
|
|
||||||
- config
|
|
||||||
expected_dests:
|
|
||||||
- config
|
|
||||||
serve:
|
|
||||||
explicit_local_fields:
|
|
||||||
- config
|
|
||||||
expected_dests:
|
|
||||||
- config
|
|
||||||
@@ -12,7 +12,7 @@ FastVideo maps a Diffusers-style repo into a pipeline like this:
|
|||||||
- `fastvideo/configs/models/*`: arch configs and `param_names_mapping` for
|
- `fastvideo/configs/models/*`: arch configs and `param_names_mapping` for
|
||||||
weight name translation.
|
weight name translation.
|
||||||
- `fastvideo/configs/pipelines/*`: pipeline wiring (component classes + names).
|
- `fastvideo/configs/pipelines/*`: pipeline wiring (component classes + names).
|
||||||
- `fastvideo/api/sampling_param.py`: runtime sampling parameters.
|
- `fastvideo/configs/sample/*`: default runtime sampling parameters.
|
||||||
- `fastvideo/pipelines/basic/*`: end-to-end pipelines.
|
- `fastvideo/pipelines/basic/*`: end-to-end pipelines.
|
||||||
- `fastvideo/pipelines/stages/*`: reusable pipeline stages.
|
- `fastvideo/pipelines/stages/*`: reusable pipeline stages.
|
||||||
- `fastvideo/models/loader/*`: component loaders for Diffusers-style repos.
|
- `fastvideo/models/loader/*`: component loaders for Diffusers-style repos.
|
||||||
@@ -26,7 +26,7 @@ Minimal usage (from `examples/inference/basic/basic.py`):
|
|||||||
|
|
||||||
```python
|
```python
|
||||||
from fastvideo import VideoGenerator
|
from fastvideo import VideoGenerator
|
||||||
from fastvideo.api.sampling_param import SamplingParam
|
from fastvideo.configs.sample import SamplingParam
|
||||||
|
|
||||||
model_id = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" # or official_weights/<model_name>/
|
model_id = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" # or official_weights/<model_name>/
|
||||||
generator = VideoGenerator.from_pretrained(model_id, num_gpus=1)
|
generator = VideoGenerator.from_pretrained(model_id, num_gpus=1)
|
||||||
@@ -49,9 +49,8 @@ runtime parameters consistent:
|
|||||||
- `fastvideo/configs/models/`: architecture definitions, layer shapes, and
|
- `fastvideo/configs/models/`: architecture definitions, layer shapes, and
|
||||||
`param_names_mapping` rules for key renaming.
|
`param_names_mapping` rules for key renaming.
|
||||||
- `fastvideo/configs/pipelines/`: pipeline wiring and required components.
|
- `fastvideo/configs/pipelines/`: pipeline wiring and required components.
|
||||||
- `fastvideo/api/sampling_param.py`: sampling parameters (steps, frames,
|
- `fastvideo/configs/sample/`: default sampling parameters (steps, frames,
|
||||||
guidance scale, resolution, fps). Defaults come from profiles in
|
guidance scale, resolution, fps).
|
||||||
`fastvideo/pipelines/basic/<family>/profiles.py`.
|
|
||||||
- `fastvideo/registry.py`: unified registry for pipeline config + sampling
|
- `fastvideo/registry.py`: unified registry for pipeline config + sampling
|
||||||
defaults and model metadata resolution, defined via explicit
|
defaults and model metadata resolution, defined via explicit
|
||||||
`register_configs(...)` blocks (no separate dict registries).
|
`register_configs(...)` blocks (no separate dict registries).
|
||||||
@@ -143,7 +142,7 @@ How this maps to FastVideo:
|
|||||||
- `T5TokenizerFast` -> loaded via HF in `fastvideo/models/loader/`
|
- `T5TokenizerFast` -> loaded via HF in `fastvideo/models/loader/`
|
||||||
- `UniPCMultistepScheduler` -> loaded via Diffusers scheduler utilities
|
- `UniPCMultistepScheduler` -> loaded via Diffusers scheduler utilities
|
||||||
- Pipeline defaults -> `fastvideo/configs/pipelines/wan.py`
|
- Pipeline defaults -> `fastvideo/configs/pipelines/wan.py`
|
||||||
- Sampling defaults -> `fastvideo/pipelines/basic/wan/profiles.py`
|
- Sampling defaults -> `fastvideo/configs/sample/wan.py`
|
||||||
|
|
||||||
## Pipeline system
|
## Pipeline system
|
||||||
|
|
||||||
|
|||||||
@@ -1,177 +0,0 @@
|
|||||||
# Streaming WebSocket Server Contract
|
|
||||||
|
|
||||||
The streaming server (`fastvideo/entrypoints/streaming/server.py`) speaks
|
|
||||||
a JSON-over-WebSocket protocol with binary fMP4 chunks for media. This
|
|
||||||
document is the authoritative spec for the message catalogue and the
|
|
||||||
session state machine. Any change to either must update this document
|
|
||||||
in the same PR that touches `protocol.py` or `session.py`.
|
|
||||||
|
|
||||||
## Endpoint
|
|
||||||
|
|
||||||
| Path | Protocol | Purpose |
|
|
||||||
|---|---|---|
|
|
||||||
| `WS /v1/stream` | WebSocket (JSON + binary) | Per-session realtime streaming |
|
|
||||||
| `GET /health` | HTTP | Liveness probe (`status`, `stream_mode`, active `sessions`) |
|
|
||||||
|
|
||||||
The server is launched by `fastvideo serve --config <serve.yaml>` when
|
|
||||||
the config carries a `streaming:` block. Without that block the same CLI
|
|
||||||
launches the OpenAI stateless HTTP server instead.
|
|
||||||
|
|
||||||
## Connection lifecycle
|
|
||||||
|
|
||||||
Every WebSocket connection holds exactly one `Session`. Sessions move
|
|
||||||
through the states in `SessionState` (`fastvideo/entrypoints/streaming/session.py`).
|
|
||||||
|
|
||||||
```
|
|
||||||
┌──────────────┐
|
|
||||||
│ INITIALIZING │ ← WebSocket accepted, before init frame
|
|
||||||
└──────┬───────┘
|
|
||||||
│ session_init_v2 received
|
|
||||||
┌──────────────┼──────────────┐
|
|
||||||
▼ ▼ ▼
|
|
||||||
QUEUED GPU_BINDING REJECTED
|
|
||||||
│ │ ↑
|
|
||||||
│ slot ready │ │ max-sessions hit
|
|
||||||
▼ ▼ │ or invalid init
|
|
||||||
┌────────┐ │
|
|
||||||
│ ACTIVE │ ────────┘
|
|
||||||
└────┬───┘
|
|
||||||
segment loop │
|
|
||||||
│
|
|
||||||
┌───────────┼───────────┐
|
|
||||||
▼ ▼ ▼
|
|
||||||
COMPLETE ERROR TIMEOUT
|
|
||||||
(clean leave) (any failure) (idle / segment_cap reached)
|
|
||||||
```
|
|
||||||
|
|
||||||
Terminal states (`COMPLETE`, `ERROR`, `TIMEOUT`, `REJECTED`) are sinks —
|
|
||||||
no transitions out. The transition matrix is enforced in
|
|
||||||
`session.py::_VALID_TRANSITIONS`; bad transitions raise.
|
|
||||||
|
|
||||||
`SessionManager` enforces the per-process budgets pulled from
|
|
||||||
`StreamingConfig`:
|
|
||||||
|
|
||||||
- `session_timeout_seconds` — idle reaper drops sessions that haven't
|
|
||||||
advanced; non-terminal sessions transition to `TIMEOUT`.
|
|
||||||
- `generation_segment_cap` — a session that hits the cap transitions to
|
|
||||||
`COMPLETE` after the last segment ships.
|
|
||||||
|
|
||||||
## Message catalogue
|
|
||||||
|
|
||||||
Every JSON frame carries `{"type": <str>, ...}`. Pydantic models in
|
|
||||||
`protocol.py` are the source of truth; this table is the human-readable
|
|
||||||
view.
|
|
||||||
|
|
||||||
### Client → server
|
|
||||||
|
|
||||||
| `type` | Required fields | Purpose |
|
|
||||||
|---|---|---|
|
|
||||||
| `session_init_v2` | — | Opening frame. Carries preset, curated prompts, optional initial image, feature toggles, optional `continuation_state` to resume from a snapshot. |
|
|
||||||
| `segment_prompt_source` | `prompt` | Request the next segment using the supplied prompt; optional sampling overrides (`seed`, `num_inference_steps`, `guidance_scale`, `negative_prompt`). |
|
|
||||||
| `seed_prompts_updated` | `seed_prompts` | Replace the session's seed-prompt list; takes effect on the next segment. |
|
|
||||||
| `enhancement_updated` | `enabled` | Toggle prompt enhancement for subsequent segments. |
|
|
||||||
| `auto_extension_updated` | `enabled` | Toggle automatic per-segment prompt extension. |
|
|
||||||
| `loop_generation_updated` | `enabled` | Toggle loop-generation mode. |
|
|
||||||
| `generation_paused_updated` | `paused` | Pause/resume segment generation; queued requests defer. |
|
|
||||||
| `snapshot_state` | — | Request the current `ContinuationState` for export; server replies with `continuation_state_snapshot`. |
|
|
||||||
|
|
||||||
The opening frame must be `session_init_v2`. Any other first frame is
|
|
||||||
rejected with an `error` (code `invalid_message`) and the WebSocket is
|
|
||||||
closed.
|
|
||||||
|
|
||||||
### Server → client
|
|
||||||
|
|
||||||
| `type` | Carries | When emitted |
|
|
||||||
|---|---|---|
|
|
||||||
| `queue_status` | `position`, `queue_depth` | After `session_init_v2` accepted, before GPU binding. |
|
|
||||||
| `gpu_assigned` | GPU id, model id | Once a generator slot is bound. |
|
|
||||||
| `ltx2_stream_start` | session-level metadata | Once the session enters `ACTIVE`. |
|
|
||||||
| `ltx2_segment_start` | `segment_idx`, `prompt`, prompt source | When a `segment_prompt_source` request begins generation. |
|
|
||||||
| `step_complete` | `segment_idx`, denoise timings | After the segment's denoising loop finishes (before media emission). |
|
|
||||||
| `media_init` | `segment_idx`, mime, stream id | First frame of fMP4 output for the segment. |
|
|
||||||
| binary frame | fMP4 fragment bytes | Subsequent media chunks; the protocol enforces that `media_init` precedes any binary frames. |
|
|
||||||
| `media_segment_complete` | `segment_idx`, chunk count, byte count | Last media chunk for the segment. |
|
|
||||||
| `ltx2_segment_complete` | `segment_idx`, segment summary | Segment fully shipped; ready for the next `segment_prompt_source`. |
|
|
||||||
| `ltx2_stream_complete` | session summary | Session reached `generation_segment_cap` or client requested clean shutdown. |
|
|
||||||
| `session_timeout` | reason | Session hit `session_timeout_seconds`; immediately followed by close. |
|
|
||||||
| `continuation_state_snapshot` | `kind`, `payload` | Reply to `snapshot_state`. The payload is the same shape produced by `LTX2ContinuationState.to_continuation_state(...)`. |
|
|
||||||
| `error` | `code`, `message` | Any validation/runtime error. Non-fatal errors keep the connection open; fatal errors precede a `close`. |
|
|
||||||
|
|
||||||
## Continuation state
|
|
||||||
|
|
||||||
The session optionally accepts a `continuation_state` dict inside the
|
|
||||||
opening `session_init_v2` frame. When present, the server hydrates it
|
|
||||||
into a `ContinuationState(kind, payload)` envelope and feeds it as the
|
|
||||||
`request.state` on the first segment's `GenerationRequest` — letting a
|
|
||||||
client resume after a disconnect, migrate sessions across processes,
|
|
||||||
or replay a prior session.
|
|
||||||
|
|
||||||
After every segment, if the runtime returns a fresh state, the server
|
|
||||||
persists it to the `SessionStore` so a `snapshot_state` request can
|
|
||||||
export it. The store and serialization contracts live with the model
|
|
||||||
family (e.g. `fastvideo/pipelines/basic/ltx2/continuation.py` for LTX-2).
|
|
||||||
|
|
||||||
## Example flow
|
|
||||||
|
|
||||||
```
|
|
||||||
client server
|
|
||||||
────── ──────
|
|
||||||
WS /v1/stream ─────── connect ─────────────────────────►
|
|
||||||
◄────── (accept)
|
|
||||||
|
|
||||||
{"type": "session_init_v2",
|
|
||||||
"preset": "ltx2_two_stage",
|
|
||||||
"curated_prompts": ["a fox in snow", "the fox jumps"],
|
|
||||||
"initial_image": {...},
|
|
||||||
"stream_mode": "av_fmp4"} ─────────────────────────────►
|
|
||||||
|
|
||||||
(validate, queue, bind)
|
|
||||||
◄──── {"type": "queue_status",
|
|
||||||
"position": 0, "queue_depth": 0}
|
|
||||||
◄──── {"type": "gpu_assigned",
|
|
||||||
"gpu_id": 0, "model_id": "..."}
|
|
||||||
◄──── {"type": "ltx2_stream_start", ...}
|
|
||||||
|
|
||||||
{"type": "segment_prompt_source",
|
|
||||||
"prompt": "a fox in snow",
|
|
||||||
"source": "curated"} ───────────────────────────────────►
|
|
||||||
(run pipeline)
|
|
||||||
◄──── {"type": "ltx2_segment_start",
|
|
||||||
"segment_idx": 1, ...}
|
|
||||||
◄──── {"type": "step_complete",
|
|
||||||
"segment_idx": 1, "timings": {...}}
|
|
||||||
◄──── {"type": "media_init",
|
|
||||||
"segment_idx": 1,
|
|
||||||
"mime": "video/mp4", ...}
|
|
||||||
◄──── <binary fMP4 init segment>
|
|
||||||
◄──── <binary fMP4 fragment>
|
|
||||||
◄──── <binary fMP4 fragment>
|
|
||||||
◄──── {"type": "media_segment_complete",
|
|
||||||
"segment_idx": 1, "chunks": 12}
|
|
||||||
◄──── {"type": "ltx2_segment_complete",
|
|
||||||
"segment_idx": 1, ...}
|
|
||||||
|
|
||||||
{"type": "segment_prompt_source",
|
|
||||||
"prompt": "the fox jumps"} ─────────────────────────────►
|
|
||||||
(segment 2 …)
|
|
||||||
|
|
||||||
{"type": "snapshot_state"} ──────────────────────────────►
|
|
||||||
◄──── {"type": "continuation_state_snapshot",
|
|
||||||
"kind": "ltx2.v1",
|
|
||||||
"payload": {"schema_version": 1, ...}}
|
|
||||||
|
|
||||||
(close) ──────────────────────────────────────────────────►
|
|
||||||
(session → COMPLETE)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Backward / forward compatibility
|
|
||||||
|
|
||||||
- Adding a new client message: append a Pydantic model to `protocol.py`
|
|
||||||
with a unique `type`; add the discriminator entry to `ClientMessage`;
|
|
||||||
add a row to the table above. Old clients that don't send the new
|
|
||||||
message remain compatible.
|
|
||||||
- Adding a new server message: emit only when a new feature flag is
|
|
||||||
enabled (or always emit, since clients ignore unknown types).
|
|
||||||
- Changing an existing message: bump the `type` (e.g. `session_init_v2`
|
|
||||||
→ `session_init_v3`) and accept both for one release cycle. Never
|
|
||||||
silently change field semantics under the same `type`.
|
|
||||||
@@ -243,7 +243,7 @@ for step in range(start_step, max_steps):
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Install
|
# Install
|
||||||
uv pip install -e ".[dev]"
|
uv pip install -e .[dev]
|
||||||
|
|
||||||
# Run DMD2 distillation on Wan 2.1
|
# Run DMD2 distillation on Wan 2.1
|
||||||
torchrun --nproc_per_node=8 -m fastvideo.train.entrypoint.train \
|
torchrun --nproc_per_node=8 -m fastvideo.train.entrypoint.train \
|
||||||
|
|||||||
@@ -16,8 +16,7 @@ Both models are trained on **61×448×832** resolution but support generating vi
|
|||||||
First install [VSA](../attention/vsa/index.md). Set `MODEL_BASE` to your own model path and run:
|
First install [VSA](../attention/vsa/index.md). Set `MODEL_BASE` to your own model path and run:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN \
|
bash scripts/inference/v1_inference_wan_dmd.sh
|
||||||
fastvideo generate --config scripts/inference/inference_wan_VSA_DMD_1_3B.yaml
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## 🗂️ Dataset
|
## 🗂️ Dataset
|
||||||
@@ -86,25 +85,3 @@ sbatch examples/distill/Wan2.2-TI2V-5B-Diffusers/Data-free/distill_dmd_t2v_5B.sh
|
|||||||
- Learning rate: 2e-5
|
- Learning rate: 2e-5
|
||||||
- Training steps: 3000 (~12 hours)
|
- Training steps: 3000 (~12 hours)
|
||||||
- HSDP shard dim: 1
|
- HSDP shard dim: 1
|
||||||
|
|
||||||
## 🧭 Note on `real_score_guidance_scale`
|
|
||||||
|
|
||||||
The teacher CFG used inside the DMD loss follows the DMD2 reference
|
|
||||||
implementation and uses the parameterization
|
|
||||||
|
|
||||||
```
|
|
||||||
x = x_cond + w * (x_cond - x_uncond)
|
|
||||||
```
|
|
||||||
|
|
||||||
rather than the Ho & Salimans form `x_uncond + w * (x_cond - x_uncond)`. The
|
|
||||||
two are mathematically equivalent up to a constant offset:
|
|
||||||
|
|
||||||
| `real_score_guidance_scale` (`w`) | Equivalent standard CFG (`w + 1`) | Output |
|
|
||||||
|-----------------------------------|-----------------------------------|-----------------------|
|
|
||||||
| `-1` | `0` | unconditional |
|
|
||||||
| `0` | `1` | conditional |
|
|
||||||
| `3.5` (default) | `4.5` | strong guidance |
|
|
||||||
|
|
||||||
So `real_score_guidance_scale` should be read as the **extra** guidance
|
|
||||||
strength added on top of the conditional prediction. When porting values
|
|
||||||
from a paper that uses the Ho & Salimans form, subtract 1.
|
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ uv pip install fastvideo
|
|||||||
conda create -n fastvideo python=3.12 -y
|
conda create -n fastvideo python=3.12 -y
|
||||||
conda activate fastvideo
|
conda activate fastvideo
|
||||||
|
|
||||||
uv pip install fastvideo
|
pip install fastvideo
|
||||||
```
|
```
|
||||||
|
|
||||||
### From source
|
### From source
|
||||||
@@ -41,11 +41,11 @@ uv pip install -e .
|
|||||||
uv pip install flash-attn --no-build-isolation -v
|
uv pip install flash-attn --no-build-isolation -v
|
||||||
```
|
```
|
||||||
|
|
||||||
Alternative with Conda environment (still drives installs through `uv`):
|
Alternative with Conda environment:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv pip install -e .
|
pip install -e .
|
||||||
uv pip install flash-attn --no-build-isolation -v
|
pip install flash-attn --no-build-isolation -v
|
||||||
```
|
```
|
||||||
|
|
||||||
## Hardware Requirements
|
## Hardware Requirements
|
||||||
|
|||||||
@@ -58,16 +58,14 @@ uv pip install flash-attn --no-build-isolation -v
|
|||||||
|
|
||||||
#### With Conda environment (alternative)
|
#### With Conda environment (alternative)
|
||||||
|
|
||||||
`uv` works inside an active conda env too, so prefer `uv pip` for the actual install:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv pip install fastvideo
|
pip install fastvideo
|
||||||
```
|
```
|
||||||
|
|
||||||
Also optionally install FlashAttention:
|
Also optionally install FlashAttention:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv pip install flash-attn --no-build-isolation -v
|
pip install flash-attn --no-build-isolation -v
|
||||||
```
|
```
|
||||||
|
|
||||||
### Installation from Source
|
### Installation from Source
|
||||||
@@ -89,7 +87,7 @@ uv pip install -e .
|
|||||||
Alternative with Conda environment:
|
Alternative with Conda environment:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv pip install -e .
|
pip install -e .
|
||||||
```
|
```
|
||||||
|
|
||||||
### Optional Dependencies
|
### Optional Dependencies
|
||||||
@@ -103,7 +101,7 @@ uv pip install flash-attn --no-build-isolation -v
|
|||||||
Alternative with Conda environment:
|
Alternative with Conda environment:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv pip install flash-attn --no-build-isolation -v
|
pip install flash-attn --no-build-isolation -v
|
||||||
```
|
```
|
||||||
|
|
||||||
## Set up using Docker
|
## Set up using Docker
|
||||||
|
|||||||
@@ -57,10 +57,8 @@ uv pip install fastvideo
|
|||||||
|
|
||||||
#### With Conda environment (alternative)
|
#### With Conda environment (alternative)
|
||||||
|
|
||||||
`uv` works inside an active conda env too, so prefer `uv pip` for the actual install:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv pip install fastvideo
|
pip install fastvideo
|
||||||
```
|
```
|
||||||
|
|
||||||
### Installation from Source
|
### Installation from Source
|
||||||
@@ -82,7 +80,7 @@ uv pip install -e .
|
|||||||
Alternative with Conda environment:
|
Alternative with Conda environment:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv pip install -e .
|
pip install -e .
|
||||||
```
|
```
|
||||||
|
|
||||||
## Development Environment Setup
|
## Development Environment Setup
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ The following two classes `PipelineConfig` and `SamplingParam` are used to confi
|
|||||||
|
|
||||||
### SamplingParam
|
### SamplingParam
|
||||||
|
|
||||||
::: fastvideo.api.sampling_param.SamplingParam
|
::: fastvideo.configs.sample.base.SamplingParam
|
||||||
options:
|
options:
|
||||||
show_root_heading: true
|
show_root_heading: true
|
||||||
show_source: false
|
show_source: false
|
||||||
|
|||||||
@@ -128,14 +128,19 @@ Concrete hierarchy: `DiTConfig` → `DiTArchConfig`, `VAEConfig` →
|
|||||||
- `dump_to_json()` / `load_from_json()` — JSON persistence. Callable
|
- `dump_to_json()` / `load_from_json()` — JSON persistence. Callable
|
||||||
fields and `arch_config` are excluded from dumps.
|
fields and `arch_config` are excluded from dumps.
|
||||||
|
|
||||||
### SamplingParam (`fastvideo/api/sampling_param.py`)
|
### SamplingParam (`fastvideo/configs/sample/`)
|
||||||
|
|
||||||
Generation parameters separate from pipeline config. Each model family
|
Generation parameters separate from pipeline config. Each model family
|
||||||
provides defaults via a profile (see `fastvideo/pipelines/basic/<family>/profiles.py`):
|
provides defaults:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
sp = SamplingParam.from_pretrained("Wan-AI/Wan2.1-T2V-1.3B-Diffusers")
|
@dataclass
|
||||||
# sp.height == 480, sp.width == 832, sp.num_frames == 81, etc.
|
class WanT2V_1_3B_SamplingParam(SamplingParam):
|
||||||
|
height: int = 480
|
||||||
|
width: int = 832
|
||||||
|
num_frames: int = 81
|
||||||
|
guidance_scale: float = 3.0
|
||||||
|
num_inference_steps: int = 50
|
||||||
```
|
```
|
||||||
|
|
||||||
## Component Loading
|
## Component Loading
|
||||||
@@ -425,9 +430,9 @@ User: generator.generate_video(prompt, ...)
|
|||||||
`fastvideo/configs/pipelines/<model>.py`. Set DiT/VAE/encoder configs,
|
`fastvideo/configs/pipelines/<model>.py`. Set DiT/VAE/encoder configs,
|
||||||
flow_shift, precision defaults.
|
flow_shift, precision defaults.
|
||||||
|
|
||||||
2. **Sampling param profile** — Create a profile in
|
2. **Sampling param** — Create a `SamplingParam` subclass in
|
||||||
`fastvideo/pipelines/basic/<model>/profiles.py` with default height,
|
`fastvideo/configs/sample/<model>.py`. Set default height, width,
|
||||||
width, num_frames, guidance_scale, num_inference_steps.
|
num_frames, guidance_scale, num_inference_steps.
|
||||||
|
|
||||||
3. **Register configs** — In `fastvideo/registry.py`, add a
|
3. **Register configs** — In `fastvideo/registry.py`, add a
|
||||||
`register_configs()` call inside `_register_configs()` with
|
`register_configs()` call inside `_register_configs()` with
|
||||||
@@ -450,6 +455,6 @@ User: generator.generate_video(prompt, ...)
|
|||||||
`fastvideo/pipelines/stages/`, implement `forward()`, optionally
|
`fastvideo/pipelines/stages/`, implement `forward()`, optionally
|
||||||
implement `verify_input()`/`verify_output()`.
|
implement `verify_input()`/`verify_output()`.
|
||||||
|
|
||||||
7. **Verify** — Run `fastvideo generate --config <config.yaml>` with a
|
7. **Verify** — Run `fastvideo generate --model-path <path> --prompt
|
||||||
minimal nested config to confirm the pipeline loads and generates
|
"test" --num-inference-steps 2` to confirm the pipeline loads and
|
||||||
output.
|
generates output.
|
||||||
|
|||||||
+81
-42
@@ -1,29 +1,71 @@
|
|||||||
# FastVideo CLI Inference
|
# FastVideo CLI Inference
|
||||||
|
|
||||||
The FastVideo CLI is config-first. Inference runs are driven by a nested JSON or
|
The FastVideo CLI exposes the same core inference controls as the Python API.
|
||||||
YAML config, with optional dotted-path overrides on the command line. The
|
|
||||||
contract matches training: use an explicit subcommand plus `--config`, then add
|
|
||||||
any dotted overrides you need.
|
|
||||||
|
|
||||||
## Basic Usage
|
## Basic Usage
|
||||||
|
|
||||||
|
Use either:
|
||||||
|
|
||||||
|
1. `--model-path` + `--prompt`
|
||||||
|
2. `--model-path` + `--prompt-txt` (batch prompts, one line per prompt)
|
||||||
|
3. `--config` (JSON/YAML)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
fastvideo generate --config config.yaml
|
fastvideo generate --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \
|
||||||
fastvideo serve --config serve.yaml
|
--prompt "A cat playing with a ball of yarn"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
fastvideo generate --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \
|
||||||
|
--prompt-txt prompts.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
You cannot provide both `--prompt` and `--prompt-txt` in the same run.
|
||||||
|
|
||||||
## View All Arguments
|
## View All Arguments
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
fastvideo generate --help
|
fastvideo generate --help
|
||||||
```
|
```
|
||||||
|
|
||||||
The subcommands intentionally expose only `--config`. Any per-run CLI changes
|
Arguments come from:
|
||||||
must use dotted override paths such as:
|
|
||||||
|
|
||||||
- `--generator.engine.num_gpus 2`
|
- FastVideo runtime args (`FastVideoArgs`)
|
||||||
- `--request.sampling.seed 42`
|
- Sampling args (`SamplingParam`)
|
||||||
- `--server.port 9000`
|
- Pipeline config args (`PipelineConfig`)
|
||||||
|
|
||||||
|
## Common Arguments
|
||||||
|
|
||||||
|
### Parallelism
|
||||||
|
|
||||||
|
- `--num-gpus`
|
||||||
|
- `--sp-size`
|
||||||
|
- `--tp-size`
|
||||||
|
|
||||||
|
### Sampling
|
||||||
|
|
||||||
|
- `--num-frames`
|
||||||
|
- `--height` / `--width`
|
||||||
|
- `--num-inference-steps`
|
||||||
|
- `--guidance-scale`
|
||||||
|
- `--seed`
|
||||||
|
- `--negative-prompt`
|
||||||
|
|
||||||
|
### Output
|
||||||
|
|
||||||
|
- `--output-path`
|
||||||
|
- `--save-video` / `--no-save-video`
|
||||||
|
- `--return-frames`
|
||||||
|
|
||||||
|
### Offloading and Performance
|
||||||
|
|
||||||
|
- `--dit-layerwise-offload`
|
||||||
|
- `--use-fsdp-inference`
|
||||||
|
- `--text-encoder-cpu-offload`
|
||||||
|
- `--image-encoder-cpu-offload`
|
||||||
|
- `--vae-cpu-offload`
|
||||||
|
- `--enable-torch-compile`
|
||||||
|
- `--torch-compile-kwargs`
|
||||||
|
|
||||||
## Using Config Files
|
## Using Config Files
|
||||||
|
|
||||||
@@ -31,53 +73,50 @@ must use dotted override paths such as:
|
|||||||
fastvideo generate --config config.yaml
|
fastvideo generate --config config.yaml
|
||||||
```
|
```
|
||||||
|
|
||||||
Config files can be JSON or YAML. Dotted CLI overrides take precedence over
|
Config files can be JSON or YAML. CLI flags override config-file values.
|
||||||
config-file values.
|
|
||||||
|
|
||||||
Example `config.yaml`:
|
Example `config.yaml`:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
generator:
|
model_path: "FastVideo/FastHunyuan-diffusers"
|
||||||
model_path: FastVideo/FastHunyuan-diffusers
|
prompt: "A capybara lounging in a hammock"
|
||||||
engine:
|
output_path: "outputs/"
|
||||||
num_gpus: 2
|
num_gpus: 2
|
||||||
parallelism:
|
sp_size: 2
|
||||||
sp_size: 2
|
tp_size: 1
|
||||||
tp_size: 1
|
num_frames: 45
|
||||||
request:
|
height: 720
|
||||||
prompt: A capybara lounging in a hammock
|
width: 1280
|
||||||
sampling:
|
num_inference_steps: 6
|
||||||
num_frames: 45
|
seed: 1024
|
||||||
height: 720
|
dit_precision: "bf16"
|
||||||
width: 1280
|
vae_precision: "fp16"
|
||||||
num_inference_steps: 6
|
vae_tiling: true
|
||||||
seed: 1024
|
vae_sp: true
|
||||||
output:
|
enable_torch_compile: false
|
||||||
output_path: outputs/
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Notes:
|
Notes:
|
||||||
|
|
||||||
- `generator` and `request` are the top-level keys for generation configs.
|
- Use `dit_precision` / `vae_precision` (not `precision`).
|
||||||
- `serve` configs use `generator`, `server`, and optional `default_request`.
|
- Nested config objects are supported, for example `vae_config` and
|
||||||
- Prompt text files belong under `request.inputs.prompt_path`.
|
`dit_config`.
|
||||||
|
|
||||||
## Examples
|
## Examples
|
||||||
|
|
||||||
Simple generation:
|
Simple generation:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
fastvideo generate --config config.yaml
|
fastvideo generate \
|
||||||
|
--model-path FastVideo/FastHunyuan-diffusers \
|
||||||
|
--prompt "A cat playing with a ball of yarn" \
|
||||||
|
--num-frames 45 --height 720 --width 1280 \
|
||||||
|
--num-inference-steps 6 --seed 1024 \
|
||||||
|
--output-path outputs/
|
||||||
```
|
```
|
||||||
|
|
||||||
Config + dotted override:
|
Config + CLI override:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
fastvideo generate --config config.yaml --request.prompt "A panda skiing at sunset"
|
fastvideo generate --config config.yaml --prompt "A panda skiing at sunset"
|
||||||
```
|
|
||||||
|
|
||||||
Helper wrapper with positional config path:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
bash scripts/inference/run.sh scripts/inference/inference_wan.yaml
|
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -73,40 +73,32 @@ if __name__ == '__main__':
|
|||||||
|
|
||||||
## JSON/YAML Config Files (CLI)
|
## JSON/YAML Config Files (CLI)
|
||||||
|
|
||||||
The inference CLI is config-first. Use an explicit subcommand with `--config`,
|
The CLI supports `--config` with JSON or YAML. Command-line arguments override
|
||||||
then apply optional dotted overrides on top, matching the training CLI style.
|
config file values.
|
||||||
By default, CLI generation uses `return_frames=false` unless you set
|
By default, `fastvideo generate` uses `return_frames=false` unless you set
|
||||||
`request.output.return_frames: true` in config or via a dotted override.
|
`--return-frames` (or `return_frames: true` in config).
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
fastvideo generate --config config.yaml
|
fastvideo generate --config config.yaml
|
||||||
```
|
```
|
||||||
|
|
||||||
Example nested config:
|
Use CLI argument names as keys (underscore or hyphen is accepted). Example:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
generator:
|
model_path: "FastVideo/FastHunyuan-diffusers"
|
||||||
model_path: FastVideo/FastHunyuan-diffusers
|
prompt: "A capybara relaxing in a hammock"
|
||||||
engine:
|
num_gpus: 2
|
||||||
num_gpus: 2
|
sp_size: 2
|
||||||
parallelism:
|
num_frames: 45
|
||||||
sp_size: 2
|
height: 720
|
||||||
request:
|
width: 1280
|
||||||
prompt: A capybara relaxing in a hammock
|
num_inference_steps: 6
|
||||||
sampling:
|
seed: 1024
|
||||||
num_frames: 45
|
dit_precision: "bf16"
|
||||||
height: 720
|
vae_precision: "fp16"
|
||||||
width: 1280
|
vae_tiling: true
|
||||||
num_inference_steps: 6
|
vae_sp: true
|
||||||
seed: 1024
|
enable_torch_compile: false
|
||||||
output:
|
|
||||||
output_path: outputs/
|
|
||||||
```
|
|
||||||
|
|
||||||
Override individual values from the CLI with dotted paths:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
fastvideo generate --config config.yaml --request.sampling.seed 42
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Performance Optimization
|
## Performance Optimization
|
||||||
|
|||||||
@@ -1,129 +0,0 @@
|
|||||||
# GEN3C: 3D-Informed Camera-Controlled Video Generation
|
|
||||||
|
|
||||||
[GEN3C](https://arxiv.org/abs/2503.03751) is NVIDIA's Cosmos-7B-based video model for camera-controlled generation from a single image. The FastVideo integration supports the GEN3C I2V workflow, including 3D cache conditioning and tokenizer-based conditioning latents.
|
|
||||||
|
|
||||||
## Key Features
|
|
||||||
|
|
||||||
- **Camera trajectory control**: `left/right/up/down/zoom_in/zoom_out/clockwise/counterclockwise`
|
|
||||||
- **3D cache conditioning**: depth prediction -> point cloud cache -> forward warping -> latent conditioning
|
|
||||||
- **Single-image to video generation**: 121-frame generation with camera motion
|
|
||||||
- **Official raw checkpoint conversion**: `model.pt` -> Diffusers/FastVideo layout
|
|
||||||
|
|
||||||
## Model Sources
|
|
||||||
|
|
||||||
- Official raw checkpoint (not Diffusers): `nvidia/GEN3C-Cosmos-7B`
|
|
||||||
- Diffusers-format checkpoint: `FastVideo/GEN3C-Cosmos-7B-Diffusers`
|
|
||||||
|
|
||||||
## Prerequisites
|
|
||||||
|
|
||||||
- Install MoGe:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
uv pip install git+https://github.com/microsoft/MoGe.git
|
|
||||||
```
|
|
||||||
|
|
||||||
- If you hit `ImportError: libGL.so.1` (common on Ubuntu/headless nodes), you can try installing OpenCV runtime libs:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
sudo apt-get update
|
|
||||||
sudo apt-get install -y libgl1 libglib2.0-0 libsm6 libxext6 libxrender1
|
|
||||||
```
|
|
||||||
|
|
||||||
## Quick Start
|
|
||||||
|
|
||||||
### Option A: Use Diffusers-format weights directly
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/inference/basic/basic_gen3c.py \
|
|
||||||
--model_path FastVideo/GEN3C-Cosmos-7B-Diffusers \
|
|
||||||
--image_path /path/to/input.png \
|
|
||||||
--prompt "" \
|
|
||||||
--trajectory left \
|
|
||||||
--movement_distance 0.3 \
|
|
||||||
--camera_rotation center_facing \
|
|
||||||
--num_inference_steps 35 \
|
|
||||||
--guidance_scale 1.0 \
|
|
||||||
--output_path outputs_video/gen3c_output.mp4
|
|
||||||
```
|
|
||||||
|
|
||||||
### Option B: Convert official raw checkpoint locally
|
|
||||||
|
|
||||||
1. Download:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
huggingface-cli download nvidia/GEN3C-Cosmos-7B --local-dir official_weights/GEN3C-Cosmos-7B
|
|
||||||
```
|
|
||||||
|
|
||||||
1. Convert:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python scripts/checkpoint_conversion/convert_gen3c_to_fastvideo.py \
|
|
||||||
--source official_weights/GEN3C-Cosmos-7B/model.pt \
|
|
||||||
--output converted_weights/GEN3C-Cosmos-7B
|
|
||||||
```
|
|
||||||
|
|
||||||
1. Run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/inference/basic/basic_gen3c.py \
|
|
||||||
--model_path converted_weights/GEN3C-Cosmos-7B \
|
|
||||||
--image_path /path/to/input.png \
|
|
||||||
--prompt "" \
|
|
||||||
--trajectory left \
|
|
||||||
--movement_distance 0.3 \
|
|
||||||
--camera_rotation center_facing \
|
|
||||||
--num_inference_steps 35 \
|
|
||||||
--guidance_scale 1.0 \
|
|
||||||
--output_path outputs_video/gen3c_output.mp4
|
|
||||||
```
|
|
||||||
|
|
||||||
## FastVideo Defaults
|
|
||||||
|
|
||||||
GEN3C defaults in FastVideo:
|
|
||||||
|
|
||||||
- `height=704`, `width=1280`
|
|
||||||
- `num_frames=121`
|
|
||||||
- `num_inference_steps=35`
|
|
||||||
- `guidance_scale=1.0`
|
|
||||||
- `fps=24`
|
|
||||||
|
|
||||||
These values are defined in:
|
|
||||||
|
|
||||||
- `fastvideo/pipelines/basic/gen3c/profiles.py`
|
|
||||||
- `fastvideo/configs/pipelines/gen3c.py`
|
|
||||||
|
|
||||||
and align with the official GEN3C inference defaults in:
|
|
||||||
|
|
||||||
- `tmp/GEN3C/cosmos_predict1/diffusion/inference/inference_utils.py`
|
|
||||||
|
|
||||||
## Scheduler Note
|
|
||||||
|
|
||||||
The converted GEN3C Diffusers layout may include a FlowMatch scheduler config, but GEN3C denoising uses EDM preconditioning behavior. FastVideo's GEN3C pipeline enforces an EDM scheduler at runtime for parity with official inference behavior.
|
|
||||||
|
|
||||||
Implementation path:
|
|
||||||
|
|
||||||
- `fastvideo/pipelines/basic/gen3c/gen3c_pipeline.py`
|
|
||||||
|
|
||||||
## 3D Cache Conditioning Path
|
|
||||||
|
|
||||||
FastVideo GEN3C conditioning stage performs:
|
|
||||||
|
|
||||||
1. MoGe depth estimation from input image
|
|
||||||
2. 3D cache initialization
|
|
||||||
3. Camera trajectory generation
|
|
||||||
4. Forward rendering of warped frames + masks
|
|
||||||
5. VAE/tokenizer encoding of conditioning buffers
|
|
||||||
6. Denoising with condition mask + condition pose channels
|
|
||||||
|
|
||||||
Main implementation:
|
|
||||||
|
|
||||||
- `fastvideo/pipelines/basic/gen3c/gen3c_pipeline.py`
|
|
||||||
- `fastvideo/pipelines/basic/gen3c/cache_3d.py`
|
|
||||||
- `fastvideo/pipelines/basic/gen3c/depth_estimation.py`
|
|
||||||
- `fastvideo/models/vaes/gen3c_tokenizer_vae.py`
|
|
||||||
|
|
||||||
## References
|
|
||||||
|
|
||||||
- [GEN3C Paper](https://arxiv.org/abs/2503.03751)
|
|
||||||
- [Official Repository](https://github.com/nv-tlabs/GEN3C)
|
|
||||||
- [Official Checkpoint (raw)](https://huggingface.co/nvidia/GEN3C-Cosmos-7B)
|
|
||||||
@@ -54,7 +54,7 @@ FASTVIDEO_ATTENTION_BACKEND=SAGE_ATTN python example.py
|
|||||||
We recommend always installing [Flash Attention 2](https://github.com/Dao-AILab/flash-attention):
|
We recommend always installing [Flash Attention 2](https://github.com/Dao-AILab/flash-attention):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv pip install flash-attn==2.7.4.post1 --no-build-isolation
|
pip install flash-attn==2.7.4.post1 --no-build-isolation
|
||||||
```
|
```
|
||||||
|
|
||||||
And if using a Hopper+ GPU (ie H100), installing [Flash Attention 3](https://github.com/Dao-AILab/flash-attention?tab=readme-ov-file#flashattention-3-beta-release) by compiling it from source (takes about 10 minutes for me):
|
And if using a Hopper+ GPU (ie H100), installing [Flash Attention 3](https://github.com/Dao-AILab/flash-attention?tab=readme-ov-file#flashattention-3-beta-release) by compiling it from source (takes about 10 minutes for me):
|
||||||
@@ -63,7 +63,7 @@ And if using a Hopper+ GPU (ie H100), installing [Flash Attention 3](https://git
|
|||||||
git clone https://github.com/Dao-AILab/flash-attention.git && cd flash-attention
|
git clone https://github.com/Dao-AILab/flash-attention.git && cd flash-attention
|
||||||
|
|
||||||
cd hopper
|
cd hopper
|
||||||
uv pip install ninja
|
pip install ninja
|
||||||
python setup.py install
|
python setup.py install
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -98,7 +98,7 @@ To use [SageAttention](https://github.com/thu-ml/SageAttention) 2.1.1, please co
|
|||||||
```bash
|
```bash
|
||||||
git clone https://github.com/thu-ml/SageAttention.git
|
git clone https://github.com/thu-ml/SageAttention.git
|
||||||
cd sageattention
|
cd sageattention
|
||||||
python setup.py install # or uv pip install -e .
|
python setup.py install # or pip install -e .
|
||||||
```
|
```
|
||||||
|
|
||||||
### Sage Attention 3
|
### Sage Attention 3
|
||||||
|
|||||||
@@ -73,7 +73,6 @@ pipeline initialization and sampling.
|
|||||||
| Matrix Game 2.0 Base | `FastVideo/Matrix-Game-2.0-Base-Diffusers` | 352x640 | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ |
|
| Matrix Game 2.0 Base | `FastVideo/Matrix-Game-2.0-Base-Diffusers` | 352x640 | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ |
|
||||||
| Matrix Game 2.0 GTA | `FastVideo/Matrix-Game-2.0-GTA-Diffusers` | 352x640 | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ |
|
| Matrix Game 2.0 GTA | `FastVideo/Matrix-Game-2.0-GTA-Diffusers` | 352x640 | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ |
|
||||||
| Matrix Game 2.0 TempleRun | `FastVideo/Matrix-Game-2.0-TempleRun-Diffusers` | 352x640 | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ |
|
| Matrix Game 2.0 TempleRun | `FastVideo/Matrix-Game-2.0-TempleRun-Diffusers` | 352x640 | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ |
|
||||||
| GEN3C Cosmos 7B | `FastVideo/GEN3C-Cosmos-7B-Diffusers` | 704px1280p | ❌ | ❌ | ❌ | ⭕ | ⭕ |
|
|
||||||
|
|
||||||
**Note**: Wan2.2 TI2V 5B has some quality issues when performing I2V generation. We are working on fixing this issue.
|
**Note**: Wan2.2 TI2V 5B has some quality issues when performing I2V generation. We are working on fixing this issue.
|
||||||
|
|
||||||
@@ -86,11 +85,6 @@ The authoritative source for model-ID recognition is
|
|||||||
`fastvideo/registry.py`. If a model ID is registered there, FastVideo can
|
`fastvideo/registry.py`. If a model ID is registered there, FastVideo can
|
||||||
resolve default pipeline and sampling configuration for it.
|
resolve default pipeline and sampling configuration for it.
|
||||||
|
|
||||||
**Note (GEN3C)**: The official `nvidia/GEN3C-Cosmos-7B` repo provides a raw
|
|
||||||
`model.pt` checkpoint. Use a Diffusers-format repo (for example,
|
|
||||||
`FastVideo/GEN3C-Cosmos-7B-Diffusers`) or convert locally with
|
|
||||||
`scripts/checkpoint_conversion/convert_gen3c_to_fastvideo.py`.
|
|
||||||
|
|
||||||
## Special requirements
|
## Special requirements
|
||||||
|
|
||||||
### Sliding Tile Attention
|
### Sliding Tile Attention
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ These are end-to-end example scripts for distilling Wan2.1 T2V 1.3B model using
|
|||||||
### 0. Make sure you have installed VSA
|
### 0. Make sure you have installed VSA
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv pip install vsa
|
pip install vsa
|
||||||
```
|
```
|
||||||
|
|
||||||
### 1. Download dataset:
|
### 1. Download dataset:
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ These are end-to-end example scripts for distilling Wan2.2 TI2V 5B model DMD+VSA
|
|||||||
### 0. Make sure you have installed VSA
|
### 0. Make sure you have installed VSA
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv pip install vsa
|
pip install vsa
|
||||||
```
|
```
|
||||||
|
|
||||||
### Data-free Distillation
|
### Data-free Distillation
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ These are end-to-end example scripts for distilling Wan2.2 TI2V 5B model DMD+VSA
|
|||||||
### 0. Make sure you have installed VSA
|
### 0. Make sure you have installed VSA
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv pip install vsa
|
pip install vsa
|
||||||
```
|
```
|
||||||
|
|
||||||
### 1. Download dataset:
|
### 1. Download dataset:
|
||||||
|
|||||||
@@ -28,11 +28,6 @@ For an example running DMD+VSA inference:
|
|||||||
python examples/inference/basic/basic_dmd.py
|
python examples/inference/basic/basic_dmd.py
|
||||||
```
|
```
|
||||||
|
|
||||||
For the typed config/request path added during the inference API refactor:
|
|
||||||
```
|
|
||||||
python examples/inference/basic/basic_dmd_new_api.py
|
|
||||||
```
|
|
||||||
|
|
||||||
## Basic Walkthrough
|
## Basic Walkthrough
|
||||||
|
|
||||||
All you need to generate videos using multi-gpus from state-of-the-art diffusion pipelines is the following few lines!
|
All you need to generate videos using multi-gpus from state-of-the-art diffusion pipelines is the following few lines!
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
from fastvideo import VideoGenerator
|
from fastvideo import VideoGenerator
|
||||||
|
|
||||||
# from fastvideo.api.sampling_param import SamplingParam
|
# from fastvideo.configs.sample import SamplingParam
|
||||||
|
|
||||||
OUTPUT_PATH = "video_samples"
|
OUTPUT_PATH = "video_samples"
|
||||||
def main():
|
def main():
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
from fastvideo import VideoGenerator
|
from fastvideo import VideoGenerator
|
||||||
from fastvideo.api.sampling_param import SamplingParam
|
from fastvideo.configs.sample import SamplingParam
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
from fastvideo import VideoGenerator
|
from fastvideo import VideoGenerator
|
||||||
from fastvideo.api.sampling_param import SamplingParam
|
from fastvideo.configs.sample import SamplingParam
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
from fastvideo import VideoGenerator
|
from fastvideo import VideoGenerator
|
||||||
from fastvideo.api.sampling_param import SamplingParam
|
from fastvideo.configs.sample import SamplingParam
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import os
|
|||||||
import time
|
import time
|
||||||
from fastvideo import VideoGenerator
|
from fastvideo import VideoGenerator
|
||||||
|
|
||||||
from fastvideo.api.sampling_param import SamplingParam
|
from fastvideo.configs.sample import SamplingParam
|
||||||
|
|
||||||
OUTPUT_PATH = "video_samples_dmd2"
|
OUTPUT_PATH = "video_samples_dmd2"
|
||||||
def main():
|
def main():
|
||||||
|
|||||||
@@ -1,98 +0,0 @@
|
|||||||
import os
|
|
||||||
import time
|
|
||||||
|
|
||||||
from fastvideo import VideoGenerator
|
|
||||||
from fastvideo.api import (
|
|
||||||
EngineConfig,
|
|
||||||
GenerationRequest,
|
|
||||||
GeneratorConfig,
|
|
||||||
OffloadConfig,
|
|
||||||
OutputConfig,
|
|
||||||
PipelineSelection,
|
|
||||||
)
|
|
||||||
|
|
||||||
OUTPUT_PATH = "video_samples_dmd2_typed"
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "VIDEO_SPARSE_ATTN"
|
|
||||||
|
|
||||||
model_name = "FastVideo/FastWan2.1-T2V-1.3B-Diffusers"
|
|
||||||
generator_config = GeneratorConfig(
|
|
||||||
model_path=model_name,
|
|
||||||
engine=EngineConfig(
|
|
||||||
num_gpus=1,
|
|
||||||
use_fsdp_inference=False,
|
|
||||||
offload=OffloadConfig(
|
|
||||||
text_encoder=True,
|
|
||||||
pin_cpu_memory=True,
|
|
||||||
dit=False,
|
|
||||||
vae=False,
|
|
||||||
),
|
|
||||||
),
|
|
||||||
# PR 2 still routes a few advanced inference knobs through the
|
|
||||||
# compatibility bridge until they get first-class typed fields.
|
|
||||||
pipeline=PipelineSelection(
|
|
||||||
experimental={
|
|
||||||
"VSA_sparsity": 0.8,
|
|
||||||
},
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
load_start_time = time.perf_counter()
|
|
||||||
generator = VideoGenerator.from_config(generator_config)
|
|
||||||
load_end_time = time.perf_counter()
|
|
||||||
load_time = load_end_time - load_start_time
|
|
||||||
|
|
||||||
prompt = (
|
|
||||||
"A neon-lit alley in futuristic Tokyo during a heavy rainstorm at night. "
|
|
||||||
"The puddles reflect glowing signs in kanji, advertising ramen, karaoke, "
|
|
||||||
"and VR arcades. A woman in a translucent raincoat walks briskly with an "
|
|
||||||
"LED umbrella. Steam rises from a street food cart, and a cat darts "
|
|
||||||
"across the screen. Raindrops are visible on the camera lens, creating "
|
|
||||||
"a cinematic bokeh effect."
|
|
||||||
)
|
|
||||||
request = GenerationRequest(
|
|
||||||
prompt=prompt,
|
|
||||||
output=OutputConfig(
|
|
||||||
output_path=OUTPUT_PATH,
|
|
||||||
save_video=True,
|
|
||||||
return_frames=False,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
start_time = time.perf_counter()
|
|
||||||
result = generator.generate(request)
|
|
||||||
end_time = time.perf_counter()
|
|
||||||
gen_time = end_time - start_time
|
|
||||||
|
|
||||||
prompt2 = (
|
|
||||||
"A majestic lion strides across the golden savanna, its powerful frame "
|
|
||||||
"glistening under the warm afternoon sun. The tall grass ripples gently "
|
|
||||||
"in the breeze, enhancing the lion's commanding presence. The tone is "
|
|
||||||
"vibrant, embodying the raw energy of the wild. Low angle, steady "
|
|
||||||
"tracking shot, cinematic."
|
|
||||||
)
|
|
||||||
request2 = GenerationRequest(
|
|
||||||
prompt=prompt2,
|
|
||||||
output=OutputConfig(
|
|
||||||
output_path=OUTPUT_PATH,
|
|
||||||
save_video=True,
|
|
||||||
return_frames=False,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
start_time = time.perf_counter()
|
|
||||||
result2 = generator.generate(request2)
|
|
||||||
end_time = time.perf_counter()
|
|
||||||
gen_time2 = end_time - start_time
|
|
||||||
|
|
||||||
print(f"Time taken to load model: {load_time} seconds")
|
|
||||||
print(f"Time taken to generate video: {gen_time} seconds")
|
|
||||||
print(f"First output written to: {result.video_path}")
|
|
||||||
print(f"Time taken to generate video2: {gen_time2} seconds")
|
|
||||||
print(f"Second output written to: {result2.video_path}")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,109 +0,0 @@
|
|||||||
"""
|
|
||||||
GEN3C: 3D-aware camera-controlled video generation.
|
|
||||||
|
|
||||||
This example generates a video from a single input image with camera control.
|
|
||||||
The pipeline uses MoGe depth estimation, 3D point cloud forward warping,
|
|
||||||
and the GEN3C diffusion model.
|
|
||||||
|
|
||||||
Requirements:
|
|
||||||
1. Install MoGe:
|
|
||||||
uv pip install git+https://github.com/microsoft/MoGe.git
|
|
||||||
If you hit `ImportError: libGL.so.1`, install:
|
|
||||||
sudo apt-get update && sudo apt-get install -y libgl1 libglib2.0-0 libsm6 libxext6 libxrender1
|
|
||||||
2. Download and convert weights:
|
|
||||||
huggingface-cli download nvidia/GEN3C-Cosmos-7B --local-dir official_weights/GEN3C-Cosmos-7B
|
|
||||||
python scripts/checkpoint_conversion/convert_gen3c_to_fastvideo.py \
|
|
||||||
--source ./official_weights/GEN3C-Cosmos-7B/model.pt \
|
|
||||||
--output ./converted_weights/GEN3C-Cosmos-7B \
|
|
||||||
--components-source nvidia/Cosmos-Predict2-2B-Video2World
|
|
||||||
3. Provide an input image for 3D-conditioned generation.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
|
|
||||||
from fastvideo import VideoGenerator
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = argparse.ArgumentParser(description="GEN3C video generation")
|
|
||||||
parser.add_argument("--model_path",
|
|
||||||
type=str,
|
|
||||||
default="converted_weights/GEN3C-Cosmos-7B")
|
|
||||||
parser.add_argument("--image_path",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
help="Input image for 3D cache conditioning")
|
|
||||||
parser.add_argument("--prompt",
|
|
||||||
type=str,
|
|
||||||
default="A slow camera pan over a sunlit landscape.")
|
|
||||||
parser.add_argument(
|
|
||||||
"--negative_prompt",
|
|
||||||
type=str,
|
|
||||||
default=(
|
|
||||||
"The video captures a series of frames showing ugly scenes, static with no motion, motion blur, "
|
|
||||||
"over-saturation, shaky footage, low resolution, grainy texture, pixelated images, poorly lit areas, "
|
|
||||||
"underexposed and overexposed scenes, poor color balance, washed out colors, choppy sequences, "
|
|
||||||
"jerky movements, low frame rate, artifacting, color banding, unnatural transitions, outdated special "
|
|
||||||
"effects, fake elements, unconvincing visuals, poorly edited content, jump cuts, visual noise, and "
|
|
||||||
"flickering. Overall, the video is of poor quality."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
parser.add_argument("--trajectory",
|
|
||||||
type=str,
|
|
||||||
default="left",
|
|
||||||
choices=[
|
|
||||||
"left", "right", "up", "down", "zoom_in",
|
|
||||||
"zoom_out", "clockwise", "counterclockwise", "none"
|
|
||||||
])
|
|
||||||
parser.add_argument("--movement_distance", type=float, default=0.3)
|
|
||||||
parser.add_argument("--camera_rotation",
|
|
||||||
type=str,
|
|
||||||
default="center_facing",
|
|
||||||
choices=[
|
|
||||||
"center_facing", "no_rotation",
|
|
||||||
"trajectory_aligned"
|
|
||||||
])
|
|
||||||
parser.add_argument("--height", type=int, default=704)
|
|
||||||
parser.add_argument("--width", type=int, default=1280)
|
|
||||||
parser.add_argument("--num_frames", type=int, default=121)
|
|
||||||
parser.add_argument("--num_inference_steps", type=int, default=35)
|
|
||||||
parser.add_argument("--guidance_scale", type=float, default=1.0)
|
|
||||||
parser.add_argument("--output_path",
|
|
||||||
type=str,
|
|
||||||
default="outputs_video/gen3c.mp4")
|
|
||||||
parser.add_argument("--seed", type=int, default=42)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
generator = VideoGenerator.from_pretrained(
|
|
||||||
args.model_path,
|
|
||||||
num_gpus=1,
|
|
||||||
use_fsdp_inference=False,
|
|
||||||
dit_cpu_offload=False,
|
|
||||||
vae_cpu_offload=True,
|
|
||||||
text_encoder_cpu_offload=True,
|
|
||||||
pin_cpu_memory=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
video = generator.generate_video(
|
|
||||||
args.prompt,
|
|
||||||
negative_prompt=args.negative_prompt,
|
|
||||||
image_path=args.image_path,
|
|
||||||
trajectory_type=args.trajectory,
|
|
||||||
movement_distance=args.movement_distance,
|
|
||||||
camera_rotation=args.camera_rotation,
|
|
||||||
height=args.height,
|
|
||||||
width=args.width,
|
|
||||||
num_frames=args.num_frames,
|
|
||||||
num_inference_steps=args.num_inference_steps,
|
|
||||||
guidance_scale=args.guidance_scale,
|
|
||||||
fps=24,
|
|
||||||
seed=args.seed,
|
|
||||||
output_path=args.output_path,
|
|
||||||
save_video=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
generator.shutdown()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
from fastvideo import VideoGenerator
|
from fastvideo import VideoGenerator
|
||||||
import json
|
import json
|
||||||
# from fastvideo.api.sampling_param import SamplingParam
|
# from fastvideo.configs.sample import SamplingParam
|
||||||
|
|
||||||
OUTPUT_PATH = "video_samples_hy15"
|
OUTPUT_PATH = "video_samples_hy15"
|
||||||
def main():
|
def main():
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
from fastvideo import VideoGenerator
|
from fastvideo import VideoGenerator
|
||||||
import json
|
import json
|
||||||
# from fastvideo.api.sampling_param import SamplingParam
|
# from fastvideo.configs.sample import SamplingParam
|
||||||
|
|
||||||
OUTPUT_PATH = "video_samples_hy15_1080p"
|
OUTPUT_PATH = "video_samples_hy15_1080p"
|
||||||
def main():
|
def main():
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
from fastvideo import VideoGenerator
|
from fastvideo import VideoGenerator
|
||||||
from fastvideo.models.dits.lingbotworld.cam_utils import prepare_camera_embedding
|
from fastvideo.models.dits.lingbotworld.cam_utils import prepare_camera_embedding
|
||||||
|
|
||||||
# from fastvideo.api.sampling_param import SamplingParam
|
# from fastvideo.configs.sample import SamplingParam
|
||||||
OUTPUT_PATH = "video_samples_lingbotworld"
|
OUTPUT_PATH = "video_samples_lingbotworld"
|
||||||
def main():
|
def main():
|
||||||
# FastVideo will automatically use the optimal default arguments for the
|
# FastVideo will automatically use the optimal default arguments for the
|
||||||
|
|||||||
@@ -1,51 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
"""Minimal user-runnable example for the daVinci-MagiHuman base AV pipeline.
|
|
||||||
|
|
||||||
Produces an mp4 with both video (Wan 2.2 TI2V-5B VAE) and audio (Stable
|
|
||||||
Audio Open 1.0 VAE, first-class FastVideo port in
|
|
||||||
`fastvideo/models/vaes/oobleck.py`) muxed together via PyAV.
|
|
||||||
|
|
||||||
Prerequisites (one-off):
|
|
||||||
|
|
||||||
# Accept terms of use on the gated HF repos with your HF_TOKEN:
|
|
||||||
# - https://huggingface.co/google/t5gemma-9b-9b-ul2
|
|
||||||
# - https://huggingface.co/stabilityai/stable-audio-open-1.0
|
|
||||||
# All four cross-variant shared components (Wan 2.2 VAE, T5-Gemma
|
|
||||||
# encoder + tokenizer, Stable Audio VAE) are lazy-loaded from their
|
|
||||||
# canonical upstream HF repos on first build, so a single ~25 GB
|
|
||||||
# cache is shared across every MagiHuman variant.
|
|
||||||
|
|
||||||
The umbrella HF repo `FastVideo/MagiHuman-Diffusers` holds all four
|
|
||||||
variants (base / distill / sr_540p / sr_1080p) under sibling subfolders
|
|
||||||
and FastVideo will download just the requested subfolder. Local
|
|
||||||
conversion via `scripts/checkpoint_conversion/convert_magi_human_to_diffusers.py`
|
|
||||||
is also supported.
|
|
||||||
"""
|
|
||||||
from fastvideo import VideoGenerator
|
|
||||||
|
|
||||||
|
|
||||||
PROMPT = (
|
|
||||||
"A warm afternoon scene: a person sits on a park bench reading a book, "
|
|
||||||
"surrounded by softly swaying trees."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
generator = VideoGenerator.from_pretrained(
|
|
||||||
"FastVideo/MagiHuman-Diffusers/base",
|
|
||||||
num_gpus=1,
|
|
||||||
)
|
|
||||||
output_path = "outputs_video/magi_human_basic/output_magi_human.mp4"
|
|
||||||
generator.generate_video(
|
|
||||||
prompt=PROMPT,
|
|
||||||
output_path=output_path,
|
|
||||||
save_video=True,
|
|
||||||
# Defaults pulled from the registered preset (magi_human_base):
|
|
||||||
# height=256, width=448, fps=25, num_inference_steps=32, seed=42.
|
|
||||||
# Override here only if you have a specific QA scenario.
|
|
||||||
)
|
|
||||||
generator.shutdown()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,53 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
"""Minimal user-runnable example for the daVinci-MagiHuman DMD-2 distilled
|
|
||||||
text-to-AV pipeline.
|
|
||||||
|
|
||||||
Same arch as the base model (`basic_magi_human.py`) but with DMD-2 distilled
|
|
||||||
weights: 8 denoising steps, no classifier-free guidance. ~4x faster than
|
|
||||||
base at the same 256x480 resolution. Mirrors upstream
|
|
||||||
`daVinci-MagiHuman/example/distill/run_T2V.sh`.
|
|
||||||
|
|
||||||
Prerequisites (one-off):
|
|
||||||
|
|
||||||
# 1) Accept terms on the gated HF repos with your HF_TOKEN:
|
|
||||||
# - https://huggingface.co/google/t5gemma-9b-9b-ul2
|
|
||||||
# - https://huggingface.co/stabilityai/stable-audio-open-1.0
|
|
||||||
# Cross-variant shared components (Wan 2.2 VAE + T5-Gemma + Stable
|
|
||||||
# Audio VAE) are lazy-loaded from their canonical upstream HF repos
|
|
||||||
# and shared with the base variant cache.
|
|
||||||
# 2) Convert the distill subfolder of GAIR/daVinci-MagiHuman:
|
|
||||||
python scripts/checkpoint_conversion/convert_magi_human_to_diffusers.py \\
|
|
||||||
--source GAIR/daVinci-MagiHuman \\
|
|
||||||
--subfolder distill \\
|
|
||||||
--output converted_weights/magi_human_distill \\
|
|
||||||
--cast-bf16
|
|
||||||
# `--cast-bf16` is recommended (61 GB fp32 -> 30 GB bf16); the FV pipeline
|
|
||||||
# loads bf16 anyway, and the conversion keeps norms / RoPE bands fp32.
|
|
||||||
"""
|
|
||||||
from fastvideo import VideoGenerator
|
|
||||||
|
|
||||||
|
|
||||||
PROMPT = (
|
|
||||||
"A warm afternoon scene: a person sits on a park bench reading a book, "
|
|
||||||
"surrounded by softly swaying trees."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
generator = VideoGenerator.from_pretrained(
|
|
||||||
"FastVideo/MagiHuman-Diffusers/distill",
|
|
||||||
num_gpus=1,
|
|
||||||
)
|
|
||||||
output_path = "outputs_video/magi_human_basic/output_magi_human_distill.mp4"
|
|
||||||
generator.generate_video(
|
|
||||||
prompt=PROMPT,
|
|
||||||
output_path=output_path,
|
|
||||||
save_video=True,
|
|
||||||
# Defaults pulled from the registered preset (magi_human_distill):
|
|
||||||
# height=256, width=480, fps=25, num_inference_steps=8, cfg=1, seed=42.
|
|
||||||
)
|
|
||||||
generator.shutdown()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
"""Minimal daVinci-MagiHuman DMD-2 distilled text+image-to-AV example."""
|
|
||||||
from fastvideo import VideoGenerator
|
|
||||||
from fastvideo.pipelines.basic.magi_human.pipeline_configs import (
|
|
||||||
MagiHumanDistillI2VConfig,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
PROMPT = (
|
|
||||||
"A cheerful saxophonist performs a short line with expressive facial "
|
|
||||||
"motion, natural head movement, and synchronized audio in a small jazz club."
|
|
||||||
)
|
|
||||||
IMAGE_PATH = "assets/images/saxophonist.jpg"
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
generator = VideoGenerator.from_pretrained(
|
|
||||||
"FastVideo/MagiHuman-Diffusers/distill",
|
|
||||||
num_gpus=1,
|
|
||||||
workload_type="i2v",
|
|
||||||
override_pipeline_cls_name="MagiHumanI2VPipeline",
|
|
||||||
pipeline_config=MagiHumanDistillI2VConfig(),
|
|
||||||
)
|
|
||||||
generator.generate_video(
|
|
||||||
prompt=PROMPT,
|
|
||||||
image_path=IMAGE_PATH,
|
|
||||||
output_path="outputs_video/magi_human_distill_ti2v/output_magi_human_distill_ti2v.mp4",
|
|
||||||
save_video=True,
|
|
||||||
)
|
|
||||||
generator.shutdown()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,45 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
"""Run daVinci-MagiHuman SR-1080p text-to-AV in FastVideo.
|
|
||||||
|
|
||||||
Build the converted repo on large local storage, then symlink it into the
|
|
||||||
workspace:
|
|
||||||
|
|
||||||
python scripts/checkpoint_conversion/convert_magi_human_to_diffusers.py \
|
|
||||||
--source GAIR/daVinci-MagiHuman \
|
|
||||||
--subfolder base \
|
|
||||||
--sr-source GAIR/daVinci-MagiHuman \
|
|
||||||
--sr-subfolder 1080p_sr \
|
|
||||||
--output /raid/william5lin_converted_weights/magi_human_sr_1080p \
|
|
||||||
--cast-bf16
|
|
||||||
ln -s /raid/william5lin_converted_weights/magi_human_sr_1080p \
|
|
||||||
converted_weights/magi_human_sr_1080p
|
|
||||||
"""
|
|
||||||
from fastvideo import VideoGenerator
|
|
||||||
from fastvideo.pipelines.basic.magi_human.pipeline_configs import (
|
|
||||||
MagiHumanSR1080pConfig,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
PROMPT = (
|
|
||||||
"A warm afternoon scene: a person sits on a park bench reading a book, "
|
|
||||||
"surrounded by softly swaying trees."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
generator = VideoGenerator.from_pretrained(
|
|
||||||
"FastVideo/MagiHuman-Diffusers/sr_1080p",
|
|
||||||
num_gpus=1,
|
|
||||||
override_pipeline_cls_name="MagiHumanSR1080pPipeline",
|
|
||||||
pipeline_config=MagiHumanSR1080pConfig(),
|
|
||||||
)
|
|
||||||
generator.generate_video(
|
|
||||||
prompt=PROMPT,
|
|
||||||
output_path="outputs_video/magi_human_sr1080p/output_magi_human_sr1080p.mp4",
|
|
||||||
save_video=True,
|
|
||||||
)
|
|
||||||
generator.shutdown()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
"""Run daVinci-MagiHuman SR-1080p text+image-to-AV in FastVideo."""
|
|
||||||
from fastvideo import VideoGenerator
|
|
||||||
from fastvideo.pipelines.basic.magi_human.pipeline_configs import (
|
|
||||||
MagiHumanSR1080pI2VConfig,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
PROMPT = (
|
|
||||||
"A cheerful saxophonist performs a short line with expressive facial "
|
|
||||||
"motion, natural head movement, and synchronized audio in a small jazz club."
|
|
||||||
)
|
|
||||||
IMAGE_PATH = "assets/images/saxophonist.jpg"
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
generator = VideoGenerator.from_pretrained(
|
|
||||||
"FastVideo/MagiHuman-Diffusers/sr_1080p",
|
|
||||||
num_gpus=1,
|
|
||||||
workload_type="i2v",
|
|
||||||
override_pipeline_cls_name="MagiHumanSR1080pI2VPipeline",
|
|
||||||
pipeline_config=MagiHumanSR1080pI2VConfig(),
|
|
||||||
)
|
|
||||||
generator.generate_video(
|
|
||||||
prompt=PROMPT,
|
|
||||||
image_path=IMAGE_PATH,
|
|
||||||
output_path="outputs_video/magi_human_sr1080p_ti2v/output_magi_human_sr1080p_ti2v.mp4",
|
|
||||||
save_video=True,
|
|
||||||
)
|
|
||||||
generator.shutdown()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,37 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
"""Run daVinci-MagiHuman SR-540p text-to-AV in FastVideo.
|
|
||||||
|
|
||||||
The converted repo must contain both ``transformer/`` (base DiT) and
|
|
||||||
``sr_transformer/`` (540p SR DiT). Build it with:
|
|
||||||
|
|
||||||
python scripts/checkpoint_conversion/convert_magi_human_to_diffusers.py \
|
|
||||||
--source GAIR/daVinci-MagiHuman \
|
|
||||||
--subfolder base \
|
|
||||||
--sr-source GAIR/daVinci-MagiHuman \
|
|
||||||
--sr-subfolder 540p_sr \
|
|
||||||
--output converted_weights/magi_human_sr_540p
|
|
||||||
"""
|
|
||||||
from fastvideo import VideoGenerator
|
|
||||||
|
|
||||||
|
|
||||||
PROMPT = (
|
|
||||||
"A warm afternoon scene: a person sits on a park bench reading a book, "
|
|
||||||
"surrounded by softly swaying trees."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
generator = VideoGenerator.from_pretrained(
|
|
||||||
"FastVideo/MagiHuman-Diffusers/sr_540p",
|
|
||||||
num_gpus=1,
|
|
||||||
)
|
|
||||||
generator.generate_video(
|
|
||||||
prompt=PROMPT,
|
|
||||||
output_path="outputs_video/magi_human_sr540p/output_magi_human_sr540p.mp4",
|
|
||||||
save_video=True,
|
|
||||||
)
|
|
||||||
generator.shutdown()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
"""Run daVinci-MagiHuman SR-540p text+image-to-AV in FastVideo."""
|
|
||||||
from fastvideo import VideoGenerator
|
|
||||||
from fastvideo.pipelines.basic.magi_human.pipeline_configs import (
|
|
||||||
MagiHumanSR540pI2VConfig,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
PROMPT = (
|
|
||||||
"A cheerful saxophonist performs a short line with expressive facial "
|
|
||||||
"motion, natural head movement, and synchronized audio in a small jazz club."
|
|
||||||
)
|
|
||||||
IMAGE_PATH = "assets/images/saxophonist.jpg"
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
generator = VideoGenerator.from_pretrained(
|
|
||||||
"FastVideo/MagiHuman-Diffusers/sr_540p",
|
|
||||||
num_gpus=1,
|
|
||||||
workload_type="i2v",
|
|
||||||
override_pipeline_cls_name="MagiHumanSRI2VPipeline",
|
|
||||||
pipeline_config=MagiHumanSR540pI2VConfig(),
|
|
||||||
)
|
|
||||||
generator.generate_video(
|
|
||||||
prompt=PROMPT,
|
|
||||||
image_path=IMAGE_PATH,
|
|
||||||
output_path="outputs_video/magi_human_sr540p_ti2v/output_magi_human_sr540p_ti2v.mp4",
|
|
||||||
save_video=True,
|
|
||||||
)
|
|
||||||
generator.shutdown()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
"""Minimal daVinci-MagiHuman base text+image-to-AV example."""
|
|
||||||
from fastvideo import VideoGenerator
|
|
||||||
from fastvideo.pipelines.basic.magi_human.pipeline_configs import (
|
|
||||||
MagiHumanBaseI2VConfig,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
PROMPT = (
|
|
||||||
"A cheerful saxophonist performs a short line with expressive facial "
|
|
||||||
"motion, natural head movement, and synchronized audio in a small jazz club."
|
|
||||||
)
|
|
||||||
IMAGE_PATH = "assets/images/saxophonist.jpg"
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
generator = VideoGenerator.from_pretrained(
|
|
||||||
"FastVideo/MagiHuman-Diffusers/base",
|
|
||||||
num_gpus=1,
|
|
||||||
workload_type="i2v",
|
|
||||||
override_pipeline_cls_name="MagiHumanI2VPipeline",
|
|
||||||
pipeline_config=MagiHumanBaseI2VConfig(),
|
|
||||||
)
|
|
||||||
generator.generate_video(
|
|
||||||
prompt=PROMPT,
|
|
||||||
image_path=IMAGE_PATH,
|
|
||||||
output_path="outputs_video/magi_human_ti2v/output_magi_human_ti2v.mp4",
|
|
||||||
save_video=True,
|
|
||||||
)
|
|
||||||
generator.shutdown()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
from fastvideo import VideoGenerator, PipelineConfig
|
from fastvideo import VideoGenerator, PipelineConfig
|
||||||
from fastvideo.api.sampling_param import SamplingParam
|
from fastvideo.configs.sample import SamplingParam
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
config = PipelineConfig.from_pretrained("Wan-AI/Wan2.1-T2V-1.3B-Diffusers")
|
config = PipelineConfig.from_pretrained("Wan-AI/Wan2.1-T2V-1.3B-Diffusers")
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
from fastvideo import VideoGenerator
|
from fastvideo import VideoGenerator
|
||||||
|
|
||||||
# from fastvideo.api.sampling_param import SamplingParam
|
# from fastvideo.configs.sample import SamplingParam
|
||||||
|
|
||||||
OUTPUT_PATH = "video_samples"
|
OUTPUT_PATH = "video_samples"
|
||||||
def main():
|
def main():
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
from fastvideo import VideoGenerator, SamplingParam
|
from fastvideo import VideoGenerator, SamplingParam
|
||||||
import json
|
import json
|
||||||
# from fastvideo.api.sampling_param import SamplingParam
|
# from fastvideo.configs.sample import SamplingParam
|
||||||
|
|
||||||
OUTPUT_PATH = "video_samples_self_forcing_causal_wan2_2_14B_i2v"
|
OUTPUT_PATH = "video_samples_self_forcing_causal_wan2_2_14B_i2v"
|
||||||
def main():
|
def main():
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
from fastvideo import VideoGenerator
|
from fastvideo import VideoGenerator
|
||||||
|
|
||||||
# from fastvideo.api.sampling_param import SamplingParam
|
# from fastvideo.configs.sample import SamplingParam
|
||||||
|
|
||||||
OUTPUT_PATH = "video_samples_self_forcing_causal_wan2_2_14B_t2v"
|
OUTPUT_PATH = "video_samples_self_forcing_causal_wan2_2_14B_t2v"
|
||||||
def main():
|
def main():
|
||||||
|
|||||||
@@ -1,77 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
"""Stable Audio Open 1.0 — text-to-audio (baseline) example.
|
|
||||||
|
|
||||||
User story (game-audio designer, prototyping):
|
|
||||||
"I'm prototyping a level and I need 6 seconds of background
|
|
||||||
ambience — gentle wind, distant thunder, a hint of birdsong. I
|
|
||||||
don't want to dig through a sound library; I want to type what I
|
|
||||||
hear in my head and get a wav back. If it's wrong I'll iterate
|
|
||||||
on the prompt. This is the first stop."
|
|
||||||
|
|
||||||
User story (musician sketching ideas):
|
|
||||||
"I want to bounce a 30s lo-fi drum loop to use as a placeholder
|
|
||||||
bed while I build the rest of the track. Type prompt, get audio,
|
|
||||||
drop into the DAW. The actual production beat I'll record
|
|
||||||
myself, but I need *something* to write the chords against."
|
|
||||||
|
|
||||||
User story (researcher exploring the model):
|
|
||||||
"First time touching Stable Audio Open — what does it sound
|
|
||||||
like at default settings? This is the smallest amount of code
|
|
||||||
that goes from prompt to mp4."
|
|
||||||
|
|
||||||
How it works:
|
|
||||||
Pure text-to-audio (T2A). The pipeline runs:
|
|
||||||
T5 + NumberConditioner -> StableAudioDiT -> Oobleck VAE
|
|
||||||
via the `dpmpp-3m-sde` k-diffusion sampler. All components are
|
|
||||||
FastVideo-native — no diffusers / transformers model imports at
|
|
||||||
runtime (see REVIEW item 30). Mirrors upstream
|
|
||||||
`stable_audio_tools.inference.generation.generate_diffusion_cond`
|
|
||||||
bit-for-bit (~0.2% abs_mean drift on 25 steps).
|
|
||||||
|
|
||||||
Tunable knobs (the "creative dials"):
|
|
||||||
audio_end_in_s
|
|
||||||
1–6 — quick ideation (sub-10s wall clock at 100 steps)
|
|
||||||
10–30 — full musical phrase / loop length (the README example
|
|
||||||
uses 30s)
|
|
||||||
47.5 — model maximum (full sample_size = 2097152 / 44100 Hz)
|
|
||||||
num_inference_steps
|
|
||||||
25 — fast preview, occasional artifacts
|
|
||||||
100 — preset default (matches the HF model card)
|
|
||||||
250 — diminishing returns past here
|
|
||||||
guidance_scale
|
|
||||||
3 — looser, more variation per seed
|
|
||||||
7 — preset default; matches README
|
|
||||||
12+ — sharper but can sound "fried"
|
|
||||||
|
|
||||||
Prerequisites:
|
|
||||||
1. Accept the terms on https://huggingface.co/stabilityai/stable-audio-open-1.0
|
|
||||||
and export your HF token in the shell:
|
|
||||||
export HF_TOKEN=hf_...
|
|
||||||
2. Install optional inference deps (one-time):
|
|
||||||
uv pip install k_diffusion einops_exts alias_free_torch torchsde
|
|
||||||
"""
|
|
||||||
from fastvideo import VideoGenerator
|
|
||||||
|
|
||||||
PROMPT = "Lo-fi hip hop instrumental with vinyl crackle and gentle piano."
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
generator = VideoGenerator.from_pretrained(
|
|
||||||
"FastVideo/stable-audio-open-1.0-Diffusers",
|
|
||||||
num_gpus=1,
|
|
||||||
)
|
|
||||||
output_path = "outputs_audio/stable_audio_basic/output_stable_audio.wav"
|
|
||||||
generator.generate_video(
|
|
||||||
prompt=PROMPT,
|
|
||||||
output_path=output_path,
|
|
||||||
save_video=True,
|
|
||||||
# 6-second clip; the model max is ~47.5s.
|
|
||||||
audio_end_in_s=6.0,
|
|
||||||
# The registered preset gives 100 steps + CFG=7.0 by default;
|
|
||||||
# override num_inference_steps / guidance_scale here for QA.
|
|
||||||
)
|
|
||||||
generator.shutdown()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,77 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
"""Stable Audio Open 1.0 — audio-to-audio variation example.
|
|
||||||
|
|
||||||
User story (musician, late at night):
|
|
||||||
"I generated this 12-second lo-fi loop earlier and I love the chord
|
|
||||||
progression and overall vibe, but the snare hit at 0:08 sounds wrong
|
|
||||||
and the rhythm feels stiff. I don't want to start over from scratch
|
|
||||||
and lose what's working — I want the model to keep the harmony and
|
|
||||||
mood but reroll the percussion + groove."
|
|
||||||
|
|
||||||
User story (sound designer, on a deadline):
|
|
||||||
"I have one good 'sword clang' SFX. The art director wants 8 sibling
|
|
||||||
variations that all feel like the same sword from different angles —
|
|
||||||
same metal, same weight, slightly different impact. I'd rather
|
|
||||||
refine my one good take than text-prompt my way through 50 misses."
|
|
||||||
|
|
||||||
Pass `init_audio=path/to/clip` (any wav/mp3/mp4/m4a/flac the standard
|
|
||||||
deps decode) and the model will use it as a starting point for the
|
|
||||||
text prompt instead of pure noise.
|
|
||||||
|
|
||||||
Picking `init_audio_strength` (0.0 to 1.0):
|
|
||||||
|
|
||||||
Higher = closer to the source clip. Lower = more transformation.
|
|
||||||
(Same convention as the "Input Audio Strength" slider in
|
|
||||||
Stability's commercial Stable Audio web UI, so values transfer
|
|
||||||
directly.)
|
|
||||||
|
|
||||||
| strength | what you get |
|
|
||||||
|----------|----------------------------------------------------|
|
|
||||||
| 1.00 | Output ≈ reference. No transformation. |
|
|
||||||
| 0.85 | Texture micro-variation only. |
|
|
||||||
| 0.70 | Light reroll, same instruments. |
|
|
||||||
| 0.60 | Default. Instrument identity is replaceable |
|
|
||||||
| | (cello can take over from piano on the same notes).|
|
|
||||||
| 0.50 | Heavy — only melody / chord progression survives. |
|
|
||||||
| 0.30 | Reference acts as a loose mood prompt. |
|
|
||||||
| 0.00 | Plain T2A — reference ignored. |
|
|
||||||
|
|
||||||
Rule of thumb by intent:
|
|
||||||
* "Fix one part of this clip" -> 0.75 .. 0.85
|
|
||||||
* "Same notes, different instrument" -> 0.55 .. 0.65
|
|
||||||
* "Same chord progression, new content" -> 0.40 .. 0.55
|
|
||||||
* "Use this as a loose mood prompt" -> 0.20 .. 0.35
|
|
||||||
|
|
||||||
If the reference timbre is bleeding through more than you want,
|
|
||||||
lower it; if the structure is gone, raise it.
|
|
||||||
|
|
||||||
Prerequisites: same as `basic_stable_audio.py`.
|
|
||||||
"""
|
|
||||||
from fastvideo import VideoGenerator
|
|
||||||
|
|
||||||
PROMPT = "Change the piano to a cello playing the same notes"
|
|
||||||
# Path to any audio-bearing file (wav, mp3, mp4, m4a, flac, ...).
|
|
||||||
# Set to `None` to skip A2A and run plain T2A.
|
|
||||||
INIT_AUDIO_PATH: str | None = None
|
|
||||||
# Reference fidelity in [0, 1] -- higher = closer to source.
|
|
||||||
INIT_AUDIO_STRENGTH = 0.6
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
generator = VideoGenerator.from_pretrained(
|
|
||||||
"FastVideo/stable-audio-open-1.0-Diffusers",
|
|
||||||
num_gpus=1,
|
|
||||||
)
|
|
||||||
generator.generate_video(
|
|
||||||
prompt=PROMPT,
|
|
||||||
output_path="outputs_audio/stable_audio_a2a/output_a2a.wav",
|
|
||||||
save_video=True,
|
|
||||||
audio_end_in_s=6.0,
|
|
||||||
init_audio=INIT_AUDIO_PATH,
|
|
||||||
init_audio_strength=INIT_AUDIO_STRENGTH,
|
|
||||||
)
|
|
||||||
generator.shutdown()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,84 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
"""Stable Audio Open 1.0 — inpainting / outpainting (loop extension) example.
|
|
||||||
|
|
||||||
User story (loop extension — the killer app):
|
|
||||||
"I have a 6-second drum loop my client likes. They want it as
|
|
||||||
background bed for a 30-second ad. I need it to loop seamlessly,
|
|
||||||
but a hard cut every 6s sounds bad. Let me extend it to 30s,
|
|
||||||
keeping the first 6s exactly as-is and letting the model continue
|
|
||||||
the groove for the remaining 24s."
|
|
||||||
|
|
||||||
User story (audio repair):
|
|
||||||
"There's a microphone bump at 0:14 in this 30-second field
|
|
||||||
recording — really obvious in headphones. Mask out 0:13 to 0:15
|
|
||||||
and let the model regenerate plausible ambience that blends in.
|
|
||||||
Everything else stays exactly as I recorded it."
|
|
||||||
|
|
||||||
User story (transition smoothing):
|
|
||||||
"I have two 10-second clips I want to crossfade. Mask out a 1s
|
|
||||||
overlap region in the middle and let the model invent a coherent
|
|
||||||
transition between the two."
|
|
||||||
|
|
||||||
How it works (RePaint-style blending):
|
|
||||||
Stable Audio Open 1.0 wasn't trained as an inpainting model
|
|
||||||
(`model_type=diffusion_cond`, not `diffusion_cond_inpaint`), so we
|
|
||||||
can't use the upstream's mask-conditioned approach directly. We
|
|
||||||
use the RePaint trick instead, which works on any v-prediction
|
|
||||||
diffusion model:
|
|
||||||
|
|
||||||
1. Encode the reference clip into latent space.
|
|
||||||
2. At every denoising step `i`, replace the kept region of the
|
|
||||||
in-flight latent (where mask == 1) with the reference
|
|
||||||
re-noised to the next timestep's sigma. Only the unkept
|
|
||||||
region (mask == 0) is freely denoised.
|
|
||||||
3. After the loop, the kept region is exactly the reference;
|
|
||||||
the unkept region is freshly generated content.
|
|
||||||
|
|
||||||
This is approximate compared to a properly trained inpainting
|
|
||||||
checkpoint — the seam between kept/unkept can have slight EQ
|
|
||||||
discontinuity — but it works on the existing public model.
|
|
||||||
|
|
||||||
Tunable: the mask is a 1-D tensor in {0, 1} at the model's sample
|
|
||||||
rate. Conventions:
|
|
||||||
1.0 = keep this sample from the reference
|
|
||||||
0.0 = regenerate this sample
|
|
||||||
|
|
||||||
Prerequisites: same as `basic_stable_audio.py`.
|
|
||||||
"""
|
|
||||||
import os
|
|
||||||
|
|
||||||
from fastvideo import VideoGenerator
|
|
||||||
|
|
||||||
PROMPT = "Steady lo-fi hip hop drum loop with vinyl crackle."
|
|
||||||
# Required: path to the reference audio file (wav, mp3, mp4, m4a, flac,
|
|
||||||
# ...) you want to extend or repair. The pipeline raises if a mask is
|
|
||||||
# passed without a reference, so this must be a real path.
|
|
||||||
REFERENCE_AUDIO_PATH = "path/to/your/loop.wav"
|
|
||||||
KEEP_SECONDS = 6.0 # first KEEP_SECONDS preserved exactly
|
|
||||||
TOTAL_SECONDS = 12.0 # extend the loop to this duration
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
if not os.path.isfile(REFERENCE_AUDIO_PATH):
|
|
||||||
raise FileNotFoundError(
|
|
||||||
f"REFERENCE_AUDIO_PATH={REFERENCE_AUDIO_PATH!r} does not exist. "
|
|
||||||
"Edit this script to point at a real audio file (wav/mp3/mp4/"
|
|
||||||
"m4a/flac) before running.")
|
|
||||||
generator = VideoGenerator.from_pretrained(
|
|
||||||
"FastVideo/stable-audio-open-1.0-Diffusers",
|
|
||||||
num_gpus=1,
|
|
||||||
)
|
|
||||||
generator.generate_video(
|
|
||||||
prompt=PROMPT,
|
|
||||||
output_path="outputs_audio/stable_audio_inpaint/output_inpaint.wav",
|
|
||||||
save_video=True,
|
|
||||||
audio_end_in_s=TOTAL_SECONDS,
|
|
||||||
inpaint_audio=REFERENCE_AUDIO_PATH,
|
|
||||||
# Tuple form: keep first KEEP_SECONDS, regenerate the rest.
|
|
||||||
inpaint_mask=(KEEP_SECONDS, TOTAL_SECONDS),
|
|
||||||
)
|
|
||||||
generator.shutdown()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,53 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
"""Stable Audio Open Small — fast / lightweight T2A example.
|
|
||||||
|
|
||||||
User story (interactive UI builder):
|
|
||||||
"I'm building a sound-design UI where the user types a prompt and
|
|
||||||
we want sub-2-second feedback so the experience feels like
|
|
||||||
autocomplete, not a render queue. The full Stable Audio Open 1.0
|
|
||||||
takes ~8s on a single GPU; the small variant takes a fraction of
|
|
||||||
that — quality is lower but completely usable for real-time
|
|
||||||
iteration."
|
|
||||||
|
|
||||||
User story (overnight batch jobs):
|
|
||||||
"I'm generating 10,000 short SFX variants for a procedural game.
|
|
||||||
Wall-clock matters more than per-clip polish — give me the small
|
|
||||||
model so I can fit the run in one night instead of a week."
|
|
||||||
|
|
||||||
How it works:
|
|
||||||
The small variant is a separate Stability AI checkpoint
|
|
||||||
(`stabilityai/stable-audio-open-small`) that ships the same Oobleck
|
|
||||||
VAE as the 1.0 base model but a smaller / faster DiT (`embed_dim=1024`,
|
|
||||||
`depth=16`, `qk_norm="ln"`) and only one duration conditioner
|
|
||||||
(`seconds_total`, no `seconds_start`). FastVideo loads from the
|
|
||||||
converted Diffusers-format repo `FastVideo/stable-audio-open-small-Diffusers`
|
|
||||||
via the standard component loader; per-variant arch fields come
|
|
||||||
from `transformer/config.json` and `conditioner/config.json`.
|
|
||||||
|
|
||||||
Prerequisites: same as `basic_stable_audio.py`. The converted repo is
|
|
||||||
public so no gated-access flow is required.
|
|
||||||
"""
|
|
||||||
from fastvideo import VideoGenerator
|
|
||||||
|
|
||||||
PROMPT = "Lo-fi hip hop instrumental with vinyl crackle and gentle piano."
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
generator = VideoGenerator.from_pretrained(
|
|
||||||
"FastVideo/stable-audio-open-small-Diffusers",
|
|
||||||
num_gpus=1,
|
|
||||||
)
|
|
||||||
output_path = "outputs_audio/stable_audio_small/output_stable_audio_small.wav"
|
|
||||||
generator.generate_video(
|
|
||||||
prompt=PROMPT,
|
|
||||||
output_path=output_path,
|
|
||||||
save_video=True,
|
|
||||||
# Small variant trains on a ~11.9s window — keep `audio_end_in_s`
|
|
||||||
# at or below that.
|
|
||||||
audio_end_in_s=6.0,
|
|
||||||
)
|
|
||||||
generator.shutdown()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
from fastvideo import VideoGenerator
|
from fastvideo import VideoGenerator
|
||||||
|
|
||||||
# from fastvideo.api.sampling_param import SamplingParam
|
# from fastvideo.configs.sample import SamplingParam
|
||||||
|
|
||||||
OUTPUT_PATH = "video_samples_wan2_2_14B_t2v"
|
OUTPUT_PATH = "video_samples_wan2_2_14B_t2v"
|
||||||
def main():
|
def main():
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
from fastvideo import VideoGenerator
|
from fastvideo import VideoGenerator
|
||||||
|
|
||||||
# from fastvideo.api.sampling_param import SamplingParam
|
# from fastvideo.configs.sample import SamplingParam
|
||||||
|
|
||||||
OUTPUT_PATH = "video_samples_wan2_1_Fun"
|
OUTPUT_PATH = "video_samples_wan2_1_Fun"
|
||||||
OUTPUT_NAME = "wan2.1_test"
|
OUTPUT_NAME = "wan2.1_test"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
from fastvideo import VideoGenerator
|
from fastvideo import VideoGenerator
|
||||||
|
|
||||||
# from fastvideo.api.sampling_param import SamplingParam
|
# from fastvideo.configs.sample import SamplingParam
|
||||||
|
|
||||||
OUTPUT_PATH = "video_samples_wan2_2_14B_i2v"
|
OUTPUT_PATH = "video_samples_wan2_2_14B_i2v"
|
||||||
def main():
|
def main():
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import time
|
|||||||
|
|
||||||
import gradio as gr
|
import gradio as gr
|
||||||
from fastvideo.entrypoints.video_generator import VideoGenerator
|
from fastvideo.entrypoints.video_generator import VideoGenerator
|
||||||
from fastvideo.api.sampling_param import SamplingParam
|
from fastvideo.configs.sample.base import SamplingParam
|
||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import tempfile
|
|||||||
|
|
||||||
import gradio as gr
|
import gradio as gr
|
||||||
|
|
||||||
from fastvideo.api.sampling_param import SamplingParam
|
from fastvideo.configs.sample.base import SamplingParam
|
||||||
|
|
||||||
|
|
||||||
MODEL_PATH_MAPPING = {
|
MODEL_PATH_MAPPING = {
|
||||||
|
|||||||
@@ -185,7 +185,7 @@ class BaseModelDeployment:
|
|||||||
|
|
||||||
def _initialize_generator(self, config: Dict[str, Any]) -> None:
|
def _initialize_generator(self, config: Dict[str, Any]) -> None:
|
||||||
from fastvideo.entrypoints.video_generator import VideoGenerator
|
from fastvideo.entrypoints.video_generator import VideoGenerator
|
||||||
from fastvideo.api.sampling_param import SamplingParam
|
from fastvideo.configs.sample.base import SamplingParam
|
||||||
|
|
||||||
print(f"Initializing model: {self.model_path}")
|
print(f"Initializing model: {self.model_path}")
|
||||||
self.generator = VideoGenerator.from_pretrained(
|
self.generator = VideoGenerator.from_pretrained(
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
from fastvideo import VideoGenerator
|
from fastvideo import VideoGenerator
|
||||||
from fastvideo.api.sampling_param import SamplingParam
|
from fastvideo.configs.sample import SamplingParam
|
||||||
|
|
||||||
OUTPUT_PATH = "./lora_out"
|
OUTPUT_PATH = "./lora_out"
|
||||||
def main():
|
def main():
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
Inference using a LoRA checkpoint from FastVideo trainer.
|
Inference using a LoRA checkpoint from FastVideo trainer.
|
||||||
"""
|
"""
|
||||||
from fastvideo import VideoGenerator
|
from fastvideo import VideoGenerator
|
||||||
from fastvideo.api.sampling_param import SamplingParam
|
from fastvideo.configs.sample import SamplingParam
|
||||||
|
|
||||||
OUTPUT_PATH = "./lora_out"
|
OUTPUT_PATH = "./lora_out"
|
||||||
def main():
|
def main():
|
||||||
|
|||||||
@@ -1,73 +0,0 @@
|
|||||||
# Cosmos Predict2 2B T2V finetune config.
|
|
||||||
#
|
|
||||||
# Data must be preprocessed with Cosmos VAE + T5 text encoder
|
|
||||||
# into parquet format before training.
|
|
||||||
|
|
||||||
models:
|
|
||||||
student:
|
|
||||||
_target_: fastvideo.train.models.cosmos.CosmosModel
|
|
||||||
init_from: nvidia/Cosmos-Predict2-2B-Video2World
|
|
||||||
trainable: true
|
|
||||||
|
|
||||||
method:
|
|
||||||
_target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod
|
|
||||||
|
|
||||||
training:
|
|
||||||
distributed:
|
|
||||||
num_gpus: 8
|
|
||||||
sp_size: 1
|
|
||||||
tp_size: 1
|
|
||||||
hsdp_replicate_dim: 8
|
|
||||||
hsdp_shard_dim: 1
|
|
||||||
|
|
||||||
data:
|
|
||||||
data_path: data/cosmos_preprocessed
|
|
||||||
dataloader_num_workers: 4
|
|
||||||
train_batch_size: 1
|
|
||||||
training_cfg_rate: 0.0
|
|
||||||
seed: 1000
|
|
||||||
# Cosmos VAE: 4x temporal, 8x spatial compression.
|
|
||||||
# 93 frames -> 24 latent frames, 480x832 -> 60x104
|
|
||||||
num_latent_t: 24
|
|
||||||
num_height: 480
|
|
||||||
num_width: 832
|
|
||||||
num_frames: 93
|
|
||||||
|
|
||||||
optimizer:
|
|
||||||
learning_rate: 1.0e-5
|
|
||||||
betas: [0.9, 0.999]
|
|
||||||
weight_decay: 0.01
|
|
||||||
lr_scheduler: constant
|
|
||||||
lr_warmup_steps: 0
|
|
||||||
|
|
||||||
loop:
|
|
||||||
max_train_steps: 5000
|
|
||||||
gradient_accumulation_steps: 1
|
|
||||||
|
|
||||||
checkpoint:
|
|
||||||
output_dir: outputs/cosmos_finetune
|
|
||||||
training_state_checkpointing_steps: 500
|
|
||||||
checkpoints_total_limit: 3
|
|
||||||
resume_from_checkpoint: latest
|
|
||||||
|
|
||||||
tracker:
|
|
||||||
project_name: fastvideo_cosmos
|
|
||||||
run_name: cosmos_finetune
|
|
||||||
|
|
||||||
model:
|
|
||||||
enable_gradient_checkpointing_type: full
|
|
||||||
|
|
||||||
callbacks:
|
|
||||||
grad_clip:
|
|
||||||
_target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback
|
|
||||||
max_grad_norm: 1.0
|
|
||||||
validation:
|
|
||||||
_target_: fastvideo.train.callbacks.validation.ValidationCallback
|
|
||||||
pipeline_target: fastvideo.pipelines.basic.cosmos.cosmos_pipeline.Cosmos2VideoToWorldPipeline
|
|
||||||
dataset_file: data/cosmos_preprocessed/validation_prompts.json
|
|
||||||
every_steps: 100
|
|
||||||
sampling_steps: [50]
|
|
||||||
guidance_scale: 6.0
|
|
||||||
|
|
||||||
pipeline:
|
|
||||||
flow_shift: 1.0
|
|
||||||
@@ -1,79 +0,0 @@
|
|||||||
# Cosmos-Predict2.5-2B Text-to-World overfitting test config.
|
|
||||||
#
|
|
||||||
# Overfits on a few short videos (480x832, 93 frames) to verify the
|
|
||||||
# Cosmos 2.5 training plugin works end-to-end.
|
|
||||||
#
|
|
||||||
# Preprocess data first:
|
|
||||||
# CUDA_VISIBLE_DEVICES=0 python fastvideo/pipelines/preprocess/preprocess_cosmos25_overfit.py
|
|
||||||
#
|
|
||||||
# Run:
|
|
||||||
# bash examples/train/run.sh examples/train/configs/overfit_cosmos25_t2w.yaml
|
|
||||||
|
|
||||||
models:
|
|
||||||
student:
|
|
||||||
_target_: fastvideo.train.models.cosmos.CosmosModel
|
|
||||||
init_from: KyleShao/Cosmos-Predict2.5-2B-Diffusers
|
|
||||||
trainable: true
|
|
||||||
enable_gradient_checkpointing_type: full
|
|
||||||
flow_shift: 1.0
|
|
||||||
|
|
||||||
method:
|
|
||||||
_target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod
|
|
||||||
|
|
||||||
training:
|
|
||||||
distributed:
|
|
||||||
num_gpus: 1
|
|
||||||
sp_size: 1
|
|
||||||
tp_size: 1
|
|
||||||
hsdp_replicate_dim: 1
|
|
||||||
hsdp_shard_dim: 1
|
|
||||||
|
|
||||||
data:
|
|
||||||
data_path: data/cosmos25_overfit_preprocessed
|
|
||||||
dataloader_num_workers: 0
|
|
||||||
train_batch_size: 1
|
|
||||||
training_cfg_rate: 0.0
|
|
||||||
seed: 42
|
|
||||||
num_latent_t: 24
|
|
||||||
num_height: 480
|
|
||||||
num_width: 832
|
|
||||||
num_frames: 93
|
|
||||||
|
|
||||||
optimizer:
|
|
||||||
learning_rate: 5.0e-5
|
|
||||||
betas: [0.9, 0.999]
|
|
||||||
weight_decay: 0.0
|
|
||||||
lr_scheduler: constant
|
|
||||||
lr_warmup_steps: 0
|
|
||||||
|
|
||||||
loop:
|
|
||||||
max_train_steps: 300
|
|
||||||
gradient_accumulation_steps: 1
|
|
||||||
|
|
||||||
checkpoint:
|
|
||||||
output_dir: outputs/cosmos25_overfit
|
|
||||||
training_state_checkpointing_steps: 50
|
|
||||||
checkpoints_total_limit: 2
|
|
||||||
|
|
||||||
tracker:
|
|
||||||
project_name: fastvideo_cosmos25
|
|
||||||
run_name: cosmos25_overfit
|
|
||||||
|
|
||||||
model:
|
|
||||||
precondition_outputs: false
|
|
||||||
enable_gradient_checkpointing_type: full
|
|
||||||
|
|
||||||
callbacks:
|
|
||||||
grad_clip:
|
|
||||||
_target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback
|
|
||||||
max_grad_norm: 1.0
|
|
||||||
validation:
|
|
||||||
_target_: fastvideo.train.callbacks.validation.ValidationCallback
|
|
||||||
pipeline_target: fastvideo.pipelines.basic.cosmos.cosmos2_5_pipeline.Cosmos2_5Pipeline
|
|
||||||
dataset_file: data/cosmos25_overfit_preprocessed/validation_prompts.json
|
|
||||||
every_steps: 150
|
|
||||||
sampling_steps: [35]
|
|
||||||
guidance_scale: 7.0
|
|
||||||
|
|
||||||
pipeline:
|
|
||||||
flow_shift: 1.0
|
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
cmake_minimum_required(VERSION 3.26 FATAL_ERROR)
|
cmake_minimum_required(VERSION 3.26 FATAL_ERROR)
|
||||||
project(fastvideo-kernel LANGUAGES CXX)
|
project(fastvideo-kernel LANGUAGES CXX)
|
||||||
|
|
||||||
# Prefer environment variable (used by CI or uv pip install git+repo_addr) if CMake var is not explicitly set.
|
# Prefer environment variable (used by CI or pip install git+repo_addr) if CMake var is not explicitly set.
|
||||||
if(NOT DEFINED GPU_BACKEND AND DEFINED ENV{GPU_BACKEND})
|
if(NOT DEFINED GPU_BACKEND AND DEFINED ENV{GPU_BACKEND})
|
||||||
set(GPU_BACKEND "$ENV{GPU_BACKEND}")
|
set(GPU_BACKEND "$ENV{GPU_BACKEND}")
|
||||||
endif()
|
endif()
|
||||||
|
|||||||
@@ -10,35 +10,6 @@ set -ex
|
|||||||
|
|
||||||
echo "Building fastvideo-kernel..."
|
echo "Building fastvideo-kernel..."
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Neutralise conda-injected compiler toolchains.
|
|
||||||
#
|
|
||||||
# Conda compiler packages (gcc_linux-aarch64, gxx_linux-64, etc.) set
|
|
||||||
# CMAKE_ARGS, CFLAGS, CXXFLAGS, and LDFLAGS on activation. When multiple
|
|
||||||
# toolchains are installed the variables can reference a *cross*-compiler
|
|
||||||
# that doesn't match the host (e.g. aarch64-conda-linux-gnu-c++ on x86_64).
|
|
||||||
# Even when the correct toolchain is active, the flags it injects
|
|
||||||
# (-march=nocona, -mtune=haswell, …) can conflict with nvcc's host-compiler
|
|
||||||
# expectations. Clear them so CMake discovers the system compiler instead.
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
if [[ -n "${CONDA_PREFIX:-}" ]]; then
|
|
||||||
_need_clean=0
|
|
||||||
# Detect conda cross-compiler that doesn't match the host.
|
|
||||||
_host_arch="$(uname -m)"
|
|
||||||
if [[ "${CXX:-}" == *"conda"* ]] || [[ "${CC:-}" == *"conda"* ]]; then
|
|
||||||
_need_clean=1
|
|
||||||
fi
|
|
||||||
if [[ "${CMAKE_ARGS:-}" == *"conda"* ]]; then
|
|
||||||
_need_clean=1
|
|
||||||
fi
|
|
||||||
if (( _need_clean )); then
|
|
||||||
echo "NOTE: Clearing conda-injected compiler settings (CC/CXX/CMAKE_ARGS/CFLAGS/...)"
|
|
||||||
echo " to use the system compiler for CUDA extension builds."
|
|
||||||
unset CC CXX CMAKE_ARGS CFLAGS CXXFLAGS LDFLAGS
|
|
||||||
fi
|
|
||||||
unset _need_clean _host_arch
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Ensure submodules are initialized if needed (tk)
|
# Ensure submodules are initialized if needed (tk)
|
||||||
git submodule update --init --recursive
|
git submodule update --init --recursive
|
||||||
|
|
||||||
@@ -61,16 +32,7 @@ has_cmake_arg() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
detect_with_torch() {
|
detect_with_torch() {
|
||||||
# Prefer the active venv's python directly over `uv run --active --no-project`,
|
uv run --active --no-project python -c "import torch
|
||||||
# which on some uv versions provisions its own interpreter and misses packages
|
|
||||||
# installed into VIRTUAL_ENV.
|
|
||||||
local py
|
|
||||||
if [[ -n "${VIRTUAL_ENV:-}" && -x "${VIRTUAL_ENV}/bin/python" ]]; then
|
|
||||||
py="${VIRTUAL_ENV}/bin/python"
|
|
||||||
else
|
|
||||||
py="$(command -v python3 || command -v python)"
|
|
||||||
fi
|
|
||||||
"${py}" -c "import torch
|
|
||||||
if not torch.cuda.is_available():
|
if not torch.cuda.is_available():
|
||||||
raise RuntimeError('torch.cuda.is_available() is false')
|
raise RuntimeError('torch.cuda.is_available() is false')
|
||||||
mj, mn = torch.cuda.get_device_capability(0)
|
mj, mn = torch.cuda.get_device_capability(0)
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ classifiers = [
|
|||||||
]
|
]
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"torch>=2.5.0",
|
"torch>=2.5.0",
|
||||||
"triton>=2.0.0; sys_platform == 'linux'",
|
"triton>=2.0.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.urls]
|
[project.urls]
|
||||||
|
|||||||
@@ -5,11 +5,6 @@ from fastvideo_kernel.ops import (
|
|||||||
video_sparse_attn,
|
video_sparse_attn,
|
||||||
)
|
)
|
||||||
|
|
||||||
from fastvideo_kernel.block_sparse_attn import (
|
|
||||||
block_sparse_attn,
|
|
||||||
block_sparse_attn_from_indices,
|
|
||||||
)
|
|
||||||
|
|
||||||
from fastvideo_kernel.vmoba import (
|
from fastvideo_kernel.vmoba import (
|
||||||
moba_attn_varlen,
|
moba_attn_varlen,
|
||||||
process_moba_input,
|
process_moba_input,
|
||||||
@@ -27,8 +22,6 @@ from fastvideo_kernel.turbodiffusion_ops import (
|
|||||||
__all__ = [
|
__all__ = [
|
||||||
"sliding_tile_attention",
|
"sliding_tile_attention",
|
||||||
"video_sparse_attn",
|
"video_sparse_attn",
|
||||||
"block_sparse_attn",
|
|
||||||
"block_sparse_attn_from_indices",
|
|
||||||
"moba_attn_varlen",
|
"moba_attn_varlen",
|
||||||
"process_moba_input",
|
"process_moba_input",
|
||||||
"process_moba_output",
|
"process_moba_output",
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user