nodes/hao-ai-lab-FastVideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[ci]: gate Cosmos3 DiT with exact fingerprints

Browse Source
This commit is contained in:
SolitaryThinker
2026-07-21 16:31:59 -07:00
parent 0d5094eb1f
commit 96ad3f2f92
6 changed files with 105 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.buildkite/pipeline.yml
+12 -1
View File
@@ -286,15 +286,21 @@ steps:
agents:
queue: "default"
- path:
- "fastvideo/configs/models/dits/**"
- "fastvideo/models/dits/**"
- "fastvideo/models/loader/**"
- "fastvideo/tests/transformers/**"
- "fastvideo/layers/**"
- "fastvideo/attention/**"
- "fastvideo/distributed/**"
- "fastvideo/platforms/**"
- "fastvideo/forward_context.py"
- "fastvideo/envs.py"
- "scripts/checkpoint_conversion/**"
- "pyproject.toml"
- "docker/Dockerfile"
config:
command: "timeout 15m .buildkite/scripts/pr_test.sh"
command: "timeout 25m .buildkite/scripts/pr_test.sh"
label: ":microscope: Transformer Tests"
env:
- TEST_TYPE=transformer
@@ -358,6 +364,11 @@ steps:
- "fastvideo/**/*.py"
- "pyproject.toml"
- "docker/Dockerfile"
# The exact one-layer Cosmos3 fingerprint gates these two files.
# Its unskipped test file makes an expected-hash update run SSIM.
skip_path:
- "fastvideo/models/dits/cosmos3.py"
- "fastvideo/configs/models/dits/cosmos3.py"
config:
command: "timeout 90m .buildkite/scripts/pr_test.sh"
label: ":bar_chart: SSIM Tests"
.buildkite/scripts/pr_test.sh
+45 -27
View File
@@ -76,7 +76,7 @@ EFFECTIVE_PR=${BUILDKITE_PULL_REQUEST:-false}
if [ "$EFFECTIVE_PR" = "false" ] && [ -n "${PR_NUMBER:-}" ]; then
EFFECTIVE_PR=$PR_NUMBER
fi
MODAL_ENV="BUILDKITE_REPO=$BUILDKITE_REPO BUILDKITE_COMMIT=$BUILDKITE_COMMIT BUILDKITE_PULL_REQUEST=$EFFECTIVE_PR BUILDKITE_BRANCH=${BUILDKITE_BRANCH:-} BUILDKITE_SOURCE=${BUILDKITE_SOURCE:-} TEST_SCOPE=${TEST_SCOPE:-} BUILDKITE_BUILD_URL=${BUILDKITE_BUILD_URL:-} BUILDKITE_BUILD_ID=${BUILDKITE_BUILD_ID:-} BUILDKITE_JOB_ID=${BUILDKITE_JOB_ID:-} IMAGE_VERSION=$IMAGE_VERSION"
export BUILDKITE_PULL_REQUEST="$EFFECTIVE_PR"
POST_RUN_HOOK=""
@@ -87,14 +87,15 @@ is_truthy() {
esac
}
ssim_bootstrap_args() {
ssim_bootstrap_enabled() {
local title="${PR_TITLE:-}"
local message="${BUILDKITE_MESSAGE:-}"
if is_truthy "${FASTVIDEO_SSIM_BOOTSTRAP_MODE:-}" \
|| [[ "$title" == *"[new-model]"* ]] \
|| [[ "$message" == *"[new-model]"* ]]; then
printf ' --bootstrap-mode'
return 0
fi
return 1
}
upload_performance_artifacts() {
@@ -177,86 +178,103 @@ upload_performance_artifacts() {
case "$TEST_TYPE" in
"encoder")
log "Running encoder tests..."
MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_encoder_tests"
MODAL_COMMAND=(env "HF_API_KEY=$HF_API_KEY" python3 -m modal run "${MODAL_TEST_FILE}::run_encoder_tests")
;;
"vae")
log "Running VAE tests..."
MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_vae_tests"
MODAL_COMMAND=(env "HF_API_KEY=$HF_API_KEY" python3 -m modal run "${MODAL_TEST_FILE}::run_vae_tests")
;;
"transformer")
log "Running transformer tests..."
MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_transformer_tests"
MODAL_COMMAND=(env "HF_API_KEY=$HF_API_KEY" python3 -m modal run "${MODAL_TEST_FILE}::run_transformer_tests")
;;
"ssim")
log "Running SSIM tests..."
SSIM_BOOTSTRAP_ARGS=$(ssim_bootstrap_args)
if [ -n "$SSIM_BOOTSTRAP_ARGS" ]; then
MODAL_COMMAND=(env "HF_API_KEY=$HF_API_KEY" python3 -m modal run "${MODAL_SSIM_TEST_FILE}::run_ssim_tests")
if ssim_bootstrap_enabled; then
log "SSIM bootstrap mode enabled for new-model reference draft generation"
MODAL_COMMAND+=(--bootstrap-mode)
fi
MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run "
MODAL_COMMAND+="$MODAL_SSIM_TEST_FILE::run_ssim_tests$SSIM_BOOTSTRAP_ARGS"
;;
"training")
log "Running training tests..."
MODAL_COMMAND="$MODAL_ENV WANDB_API_KEY=$WANDB_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_training_tests"
MODAL_COMMAND=(env "WANDB_API_KEY=$WANDB_API_KEY" python3 -m modal run "${MODAL_TEST_FILE}::run_training_tests")
;;
"training_lora")
log "Running LoRA training tests..."
MODAL_COMMAND="$MODAL_ENV WANDB_API_KEY=$WANDB_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_training_lora_tests"
MODAL_COMMAND=(
env "WANDB_API_KEY=$WANDB_API_KEY" python3 -m modal run
"${MODAL_TEST_FILE}::run_training_lora_tests"
)
;;
"training_vsa")
log "Running training VSA tests..."
MODAL_COMMAND="$MODAL_ENV WANDB_API_KEY=$WANDB_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_training_tests_VSA"
MODAL_COMMAND=(
env "WANDB_API_KEY=$WANDB_API_KEY" python3 -m modal run
"${MODAL_TEST_FILE}::run_training_tests_VSA"
)
;;
"kernel_tests")
log "Running kernel tests..."
MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_kernel_tests"
MODAL_COMMAND=(python3 -m modal run "${MODAL_TEST_FILE}::run_kernel_tests")
;;
"inference_lora")
log "Running LoRA tests..."
MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_inference_lora_tests"
MODAL_COMMAND=(python3 -m modal run "${MODAL_TEST_FILE}::run_inference_lora_tests")
;;
"distillation_dmd")
log "Running distillation DMD tests..."
MODAL_COMMAND="$MODAL_ENV WANDB_API_KEY=$WANDB_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_distill_dmd_tests"
MODAL_COMMAND=(
env "WANDB_API_KEY=$WANDB_API_KEY" python3 -m modal run
"${MODAL_TEST_FILE}::run_distill_dmd_tests"
)
;;
# run_inference_tests_vmoba
"self_forcing")
log "Running self-forcing tests..."
MODAL_COMMAND="$MODAL_ENV WANDB_API_KEY=$WANDB_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_self_forcing_tests"
MODAL_COMMAND=(
env "WANDB_API_KEY=$WANDB_API_KEY" python3 -m modal run
"${MODAL_TEST_FILE}::run_self_forcing_tests"
)
;;
"inference_vmoba")
log "Running V-MoBA inference tests..."
MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_inference_tests_vmoba"
MODAL_COMMAND=(python3 -m modal run "${MODAL_TEST_FILE}::run_inference_tests_vmoba")
;;
"unit_test")
log "Running unit tests..."
MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_unit_test"
MODAL_COMMAND=(python3 -m modal run "${MODAL_TEST_FILE}::run_unit_test")
;;
"dreamverse_app")
log "Running DreamVerse app tests..."
MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_dreamverse_app_tests"
MODAL_COMMAND=(python3 -m modal run "${MODAL_TEST_FILE}::run_dreamverse_app_tests")
;;
"train_framework")
log "Running fastvideo.train framework tests..."
MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_train_framework_tests"
MODAL_COMMAND=(
env "HF_API_KEY=$HF_API_KEY" python3 -m modal run
"${MODAL_TEST_FILE}::run_train_framework_tests"
)
;;
"eval")
log "Running eval metric tests..."
MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_eval_tests"
MODAL_COMMAND=(env "HF_API_KEY=$HF_API_KEY" python3 -m modal run "${MODAL_TEST_FILE}::run_eval_tests")
;;
"lora_extraction")
log "Running LoRA extraction tests..."
MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_lora_extraction_tests"
MODAL_COMMAND=(
env "HF_API_KEY=$HF_API_KEY" python3 -m modal run
"${MODAL_TEST_FILE}::run_lora_extraction_tests"
)
;;
"performance")
log "Running performance tests on Modal..."
MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_performance_tests"
MODAL_COMMAND=(env "HF_API_KEY=$HF_API_KEY" python3 -m modal run "${MODAL_TEST_FILE}::run_performance_tests")
POST_RUN_HOOK="upload_performance_artifacts"
;;
"api_server")
log "Running API server integration tests..."
MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_api_server_tests"
MODAL_COMMAND=(env "HF_API_KEY=$HF_API_KEY" python3 -m modal run "${MODAL_TEST_FILE}::run_api_server_tests")
;;
*)
log "Error: Unknown test type: $TEST_TYPE"
@@ -264,8 +282,8 @@ case "$TEST_TYPE" in
;;
esac
log "Executing: $MODAL_COMMAND"
eval "$MODAL_COMMAND"
log "Executing Modal test: $TEST_TYPE"
"${MODAL_COMMAND[@]}"
TEST_EXIT_CODE=$?
if [ $TEST_EXIT_CODE -eq 0 ]; then
docs/contributing/ci_architecture.md
+17 -2
View File
@@ -82,11 +82,15 @@ status.
|---|---|---|
| Encoder Tests | `encoder` | `fastvideo/models/encoders/**`, `fastvideo/models/loader/**`, `fastvideo/tests/encoders/**`, `pyproject.toml`, `docker/Dockerfile` |
| VAE Tests | `vae` | `fastvideo/models/vaes/**`, `fastvideo/models/loader/**`, `fastvideo/tests/vaes/**`, `pyproject.toml`, `docker/Dockerfile` |
| Transformer Tests | `transformer` | `fastvideo/models/dits/**`, `fastvideo/models/loader/**`, `fastvideo/tests/transformers/**`, `fastvideo/layers/**`, `fastvideo/attention/**`, `pyproject.toml`, `docker/Dockerfile` |
| Transformer Tests | `transformer` | DiT configs/models, loader, transformer tests, layers, attention, distributed/platform runtime, forward context/env, checkpoint converters, dependencies, and container |
| Kernel Tests | `kernel_tests` | `fastvideo-kernel/**`, `pyproject.toml`, `docker/Dockerfile` |
| Unit Tests | `unit_test` | `fastvideo/**`, `.buildkite/**`, `.github/**`, `pyproject.toml`, `docker/Dockerfile` |
| DreamVerse App Tests | `dreamverse_app` | `apps/dreamverse/**`, `pyproject.toml` |
The transformer launcher has a 25-minute Buildkite timeout around a 20-minute
Modal function. The five-minute margin covers checkout, dependency setup, and
Modal teardown without racing the remote timeout.
### Tier 3: Full Suite
| Attribute | Value |
@@ -108,7 +112,7 @@ it), while a GitHub outage or a >25 min wait lets it run anyway (fail open).
| Buildkite label | `TEST_TYPE` | Main watched paths |
|---|---|---|
| SSIM Tests | `ssim` | `fastvideo/**/*.py`, `pyproject.toml`, `docker/Dockerfile` |
| SSIM Tests | `ssim` | `fastvideo/**/*.py`, except the fingerprint-gated Cosmos3 DiT/config files; also `pyproject.toml` and `docker/Dockerfile` |
| LoRA Inference Tests | `inference_lora` | LoRA tests, loader, transformer tests, pipelines, LoRA layers |
| LoRA Extraction Tests | `lora_extraction` | LoRA extraction scripts/tests, loader, training utilities, LoRA layers |
| Training Tests | `training` | `fastvideo/**`, `pyproject.toml`, `docker/Dockerfile` |
@@ -122,6 +126,13 @@ it), while a GitHub outage or a >25 min wait lets it run anyway (fail open).
| Train Framework Tests | `train_framework` | `fastvideo/train/**`, train model/method tests, model loader, DiTs |
| Eval Metrics Tests | `eval` | `fastvideo/eval/**`, `fastvideo/tests/eval/**`, `pyproject.toml`, `docker/Dockerfile` |
Cosmos3's DiT implementation and DiT config are guarded in Fastcheck by an
exact one-layer transformer fingerprint. A PR that changes only those two
files does not schedule SSIM. The fingerprint test itself is not excluded, so
changing its expected hashes schedules SSIM automatically. Pipeline, packing,
scheduler, encoder, VAE, shared-layer, dependency, and container changes still
schedule end-to-end SSIM.
See [Performance Benchmarks](performance_benchmarks.md) for the performance
lane's thresholds, rolling baseline, artifacts, and reseeding process.
@@ -241,6 +252,10 @@ All Buildkite test jobs go through `.buildkite/scripts/pr_test.sh`, which:
`fastvideo/tests/modal/ssim_test.py`.
5. Uploads performance artifacts for `TEST_TYPE=performance`.
The Modal runner fetches the immutable `BUILDKITE_COMMIT` recorded for the
build. It does not follow a live pull-request ref that may have advanced while
the job was queued.
If you add a new CI test category:
1. Add the Modal function in `fastvideo/tests/modal/pr_test.py` or a focused
docs/contributing/testing.md
+14
View File
@@ -33,6 +33,20 @@ pytest fastvideo/tests/vaes -vs
GPU-heavy suites need the right hardware, credentials, local caches, and
sometimes custom kernels. Document those assumptions in new tests.
## Transformer Fingerprints
An exact transformer fingerprint runs a fixed, production-shaped one-layer
forward on a pinned GPU/software/backend profile and compares SHA-256 hashes of
the complete output tensors. It loads only the checkpoint tensors used by that
forward. This is the normal Fastcheck gate for a covered DiT; it is bit-exact
within that surface and cheaper than generating media.
The fingerprint must exercise every implementation file that it exempts from
SSIM. Keep pipeline, sequence packing, scheduler, encoder, VAE, and shared
layer changes on the SSIM path. Updating an expected hash requires official
parity evidence and an end-to-end SSIM run; the fingerprint test remains an
SSIM-watched path so such updates escalate automatically.
## SSIM Tests
SSIM tests generate videos using specific models and parameters, then compare
fastvideo/tests/modal/pr_test.py
+9 -11
View File
@@ -134,6 +134,10 @@ def _checkout_repository(git_repo: str,
if not git_repo or git_repo.startswith("-"):
raise RuntimeError("BUILDKITE_REPO must be a non-empty repository URL.")
if re.fullmatch(r"(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})",
git_commit) is None:
raise RuntimeError(f"Invalid BUILDKITE_COMMIT value: {git_commit}")
if pr_number and pr_number != "false":
try:
pr_id = int(pr_number)
@@ -143,15 +147,9 @@ def _checkout_repository(git_repo: str,
if pr_id <= 0:
raise RuntimeError(
f"Invalid BUILDKITE_PULL_REQUEST value: {pr_number}")
target = f"refs/pull/{pr_id}/head"
print(f"Using PR ref for checkout: {target}")
else:
if not git_commit or re.fullmatch(r"[0-9a-fA-F]{7,64}",
git_commit) is None:
raise RuntimeError(
f"Invalid BUILDKITE_COMMIT value: {git_commit}")
target = git_commit
print(f"Using direct commit checkout: {target}")
target = git_commit
print(f"Using exact Buildkite commit checkout: {target}")
clone_command = [
"git",
@@ -270,13 +268,13 @@ def run_vae_tests():
@app.function(gpu="L40S:1",
image=image,
timeout=900,
timeout=1200,
secrets=[hf_secret, ci_env_secret],
volumes={"/root/data": model_vol})
def run_transformer_tests():
run_test(
"export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && "
"FASTVIDEO_FA4=0 pytest ./fastvideo/tests/transformers -vs"
"CUBLAS_WORKSPACE_CONFIG=:4096:8 FASTVIDEO_FA4=0 pytest ./fastvideo/tests/transformers -vs"
)
fastvideo/tests/modal/test_pr_test.py
+8 -4
View File
@@ -72,7 +72,8 @@ def _load_pr_test_module(monkeypatch):
return module
def test_checkout_repository_retries_clone_and_fetches_pr_ref(monkeypatch):
def test_checkout_repository_retries_clone_and_fetches_exact_pr_commit(
monkeypatch):
module = _load_pr_test_module(monkeypatch)
commands = []
cleanup_paths = []
@@ -131,7 +132,7 @@ def test_checkout_repository_retries_clone_and_fetches_pr_ref(monkeypatch):
fetch_command = commands[2][0]
assert fetch_command[-2:] == [
"origin",
"refs/pull/1584/head",
"0123456789abcdef0123456789abcdef01234567",
]
assert "--depth=1" in fetch_command
assert "--filter=blob:none" in fetch_command
@@ -211,8 +212,11 @@ def test_git_retry_exhaustion_is_bounded_and_cleans_each_attempt(monkeypatch):
("-bad-option", "0123456789abcdef", "false"),
("https://example.com/repo.git", "", "false"),
("https://example.com/repo.git", "not-a-commit", "false"),
("https://example.com/repo.git", "0123456789abcdef", "not-a-pr"),
("https://example.com/repo.git", "0123456789abcdef", "0"),
("https://example.com/repo.git", "0123456789abcdef", "1584"),
("https://example.com/repo.git",
"0123456789abcdef0123456789abcdef01234567", "not-a-pr"),
("https://example.com/repo.git",
"0123456789abcdef0123456789abcdef01234567", "0"),
],
)
def test_checkout_repository_rejects_invalid_buildkite_values(