diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 11739792c..ff6961c87 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -286,15 +286,21 @@ steps: agents: queue: "default" - path: + - "fastvideo/configs/models/dits/**" - "fastvideo/models/dits/**" - "fastvideo/models/loader/**" - "fastvideo/tests/transformers/**" - "fastvideo/layers/**" - "fastvideo/attention/**" + - "fastvideo/distributed/**" + - "fastvideo/platforms/**" + - "fastvideo/forward_context.py" + - "fastvideo/envs.py" + - "scripts/checkpoint_conversion/**" - "pyproject.toml" - "docker/Dockerfile" config: - command: "timeout 15m .buildkite/scripts/pr_test.sh" + command: "timeout 25m .buildkite/scripts/pr_test.sh" label: ":microscope: Transformer Tests" env: - TEST_TYPE=transformer @@ -358,6 +364,11 @@ steps: - "fastvideo/**/*.py" - "pyproject.toml" - "docker/Dockerfile" + # The exact one-layer Cosmos3 fingerprint gates these two files. + # Its unskipped test file makes an expected-hash update run SSIM. + skip_path: + - "fastvideo/models/dits/cosmos3.py" + - "fastvideo/configs/models/dits/cosmos3.py" config: command: "timeout 90m .buildkite/scripts/pr_test.sh" label: ":bar_chart: SSIM Tests" diff --git a/.buildkite/scripts/pr_test.sh b/.buildkite/scripts/pr_test.sh index fd46e1d4d..8d12d5a25 100755 --- a/.buildkite/scripts/pr_test.sh +++ b/.buildkite/scripts/pr_test.sh @@ -76,7 +76,7 @@ EFFECTIVE_PR=${BUILDKITE_PULL_REQUEST:-false} if [ "$EFFECTIVE_PR" = "false" ] && [ -n "${PR_NUMBER:-}" ]; then EFFECTIVE_PR=$PR_NUMBER fi -MODAL_ENV="BUILDKITE_REPO=$BUILDKITE_REPO BUILDKITE_COMMIT=$BUILDKITE_COMMIT BUILDKITE_PULL_REQUEST=$EFFECTIVE_PR BUILDKITE_BRANCH=${BUILDKITE_BRANCH:-} BUILDKITE_SOURCE=${BUILDKITE_SOURCE:-} TEST_SCOPE=${TEST_SCOPE:-} BUILDKITE_BUILD_URL=${BUILDKITE_BUILD_URL:-} BUILDKITE_BUILD_ID=${BUILDKITE_BUILD_ID:-} BUILDKITE_JOB_ID=${BUILDKITE_JOB_ID:-} IMAGE_VERSION=$IMAGE_VERSION" +export BUILDKITE_PULL_REQUEST="$EFFECTIVE_PR" POST_RUN_HOOK="" @@ -87,14 +87,15 @@ is_truthy() { esac } -ssim_bootstrap_args() { +ssim_bootstrap_enabled() { local title="${PR_TITLE:-}" local message="${BUILDKITE_MESSAGE:-}" if is_truthy "${FASTVIDEO_SSIM_BOOTSTRAP_MODE:-}" \ || [[ "$title" == *"[new-model]"* ]] \ || [[ "$message" == *"[new-model]"* ]]; then - printf ' --bootstrap-mode' + return 0 fi + return 1 } upload_performance_artifacts() { @@ -177,86 +178,103 @@ upload_performance_artifacts() { case "$TEST_TYPE" in "encoder") log "Running encoder tests..." - MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_encoder_tests" + MODAL_COMMAND=(env "HF_API_KEY=$HF_API_KEY" python3 -m modal run "${MODAL_TEST_FILE}::run_encoder_tests") ;; "vae") log "Running VAE tests..." - MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_vae_tests" + MODAL_COMMAND=(env "HF_API_KEY=$HF_API_KEY" python3 -m modal run "${MODAL_TEST_FILE}::run_vae_tests") ;; "transformer") log "Running transformer tests..." - MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_transformer_tests" + MODAL_COMMAND=(env "HF_API_KEY=$HF_API_KEY" python3 -m modal run "${MODAL_TEST_FILE}::run_transformer_tests") ;; "ssim") log "Running SSIM tests..." - SSIM_BOOTSTRAP_ARGS=$(ssim_bootstrap_args) - if [ -n "$SSIM_BOOTSTRAP_ARGS" ]; then + MODAL_COMMAND=(env "HF_API_KEY=$HF_API_KEY" python3 -m modal run "${MODAL_SSIM_TEST_FILE}::run_ssim_tests") + if ssim_bootstrap_enabled; then log "SSIM bootstrap mode enabled for new-model reference draft generation" + MODAL_COMMAND+=(--bootstrap-mode) fi - MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run " - MODAL_COMMAND+="$MODAL_SSIM_TEST_FILE::run_ssim_tests$SSIM_BOOTSTRAP_ARGS" ;; "training") log "Running training tests..." - MODAL_COMMAND="$MODAL_ENV WANDB_API_KEY=$WANDB_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_training_tests" + MODAL_COMMAND=(env "WANDB_API_KEY=$WANDB_API_KEY" python3 -m modal run "${MODAL_TEST_FILE}::run_training_tests") ;; "training_lora") log "Running LoRA training tests..." - MODAL_COMMAND="$MODAL_ENV WANDB_API_KEY=$WANDB_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_training_lora_tests" + MODAL_COMMAND=( + env "WANDB_API_KEY=$WANDB_API_KEY" python3 -m modal run + "${MODAL_TEST_FILE}::run_training_lora_tests" + ) ;; "training_vsa") log "Running training VSA tests..." - MODAL_COMMAND="$MODAL_ENV WANDB_API_KEY=$WANDB_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_training_tests_VSA" + MODAL_COMMAND=( + env "WANDB_API_KEY=$WANDB_API_KEY" python3 -m modal run + "${MODAL_TEST_FILE}::run_training_tests_VSA" + ) ;; "kernel_tests") log "Running kernel tests..." - MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_kernel_tests" + MODAL_COMMAND=(python3 -m modal run "${MODAL_TEST_FILE}::run_kernel_tests") ;; "inference_lora") log "Running LoRA tests..." - MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_inference_lora_tests" + MODAL_COMMAND=(python3 -m modal run "${MODAL_TEST_FILE}::run_inference_lora_tests") ;; "distillation_dmd") log "Running distillation DMD tests..." - MODAL_COMMAND="$MODAL_ENV WANDB_API_KEY=$WANDB_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_distill_dmd_tests" + MODAL_COMMAND=( + env "WANDB_API_KEY=$WANDB_API_KEY" python3 -m modal run + "${MODAL_TEST_FILE}::run_distill_dmd_tests" + ) ;; # run_inference_tests_vmoba "self_forcing") log "Running self-forcing tests..." - MODAL_COMMAND="$MODAL_ENV WANDB_API_KEY=$WANDB_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_self_forcing_tests" + MODAL_COMMAND=( + env "WANDB_API_KEY=$WANDB_API_KEY" python3 -m modal run + "${MODAL_TEST_FILE}::run_self_forcing_tests" + ) ;; "inference_vmoba") log "Running V-MoBA inference tests..." - MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_inference_tests_vmoba" + MODAL_COMMAND=(python3 -m modal run "${MODAL_TEST_FILE}::run_inference_tests_vmoba") ;; "unit_test") log "Running unit tests..." - MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_unit_test" + MODAL_COMMAND=(python3 -m modal run "${MODAL_TEST_FILE}::run_unit_test") ;; "dreamverse_app") log "Running DreamVerse app tests..." - MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_dreamverse_app_tests" + MODAL_COMMAND=(python3 -m modal run "${MODAL_TEST_FILE}::run_dreamverse_app_tests") ;; "train_framework") log "Running fastvideo.train framework tests..." - MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_train_framework_tests" + MODAL_COMMAND=( + env "HF_API_KEY=$HF_API_KEY" python3 -m modal run + "${MODAL_TEST_FILE}::run_train_framework_tests" + ) ;; "eval") log "Running eval metric tests..." - MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_eval_tests" + MODAL_COMMAND=(env "HF_API_KEY=$HF_API_KEY" python3 -m modal run "${MODAL_TEST_FILE}::run_eval_tests") ;; "lora_extraction") log "Running LoRA extraction tests..." - MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_lora_extraction_tests" + MODAL_COMMAND=( + env "HF_API_KEY=$HF_API_KEY" python3 -m modal run + "${MODAL_TEST_FILE}::run_lora_extraction_tests" + ) ;; "performance") log "Running performance tests on Modal..." - MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_performance_tests" + MODAL_COMMAND=(env "HF_API_KEY=$HF_API_KEY" python3 -m modal run "${MODAL_TEST_FILE}::run_performance_tests") POST_RUN_HOOK="upload_performance_artifacts" ;; "api_server") log "Running API server integration tests..." - MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_api_server_tests" + MODAL_COMMAND=(env "HF_API_KEY=$HF_API_KEY" python3 -m modal run "${MODAL_TEST_FILE}::run_api_server_tests") ;; *) log "Error: Unknown test type: $TEST_TYPE" @@ -264,8 +282,8 @@ case "$TEST_TYPE" in ;; esac -log "Executing: $MODAL_COMMAND" -eval "$MODAL_COMMAND" +log "Executing Modal test: $TEST_TYPE" +"${MODAL_COMMAND[@]}" TEST_EXIT_CODE=$? if [ $TEST_EXIT_CODE -eq 0 ]; then diff --git a/docs/contributing/ci_architecture.md b/docs/contributing/ci_architecture.md index 6bbd77fa8..b54aad8ce 100644 --- a/docs/contributing/ci_architecture.md +++ b/docs/contributing/ci_architecture.md @@ -82,11 +82,15 @@ status. |---|---|---| | Encoder Tests | `encoder` | `fastvideo/models/encoders/**`, `fastvideo/models/loader/**`, `fastvideo/tests/encoders/**`, `pyproject.toml`, `docker/Dockerfile` | | VAE Tests | `vae` | `fastvideo/models/vaes/**`, `fastvideo/models/loader/**`, `fastvideo/tests/vaes/**`, `pyproject.toml`, `docker/Dockerfile` | -| Transformer Tests | `transformer` | `fastvideo/models/dits/**`, `fastvideo/models/loader/**`, `fastvideo/tests/transformers/**`, `fastvideo/layers/**`, `fastvideo/attention/**`, `pyproject.toml`, `docker/Dockerfile` | +| Transformer Tests | `transformer` | DiT configs/models, loader, transformer tests, layers, attention, distributed/platform runtime, forward context/env, checkpoint converters, dependencies, and container | | Kernel Tests | `kernel_tests` | `fastvideo-kernel/**`, `pyproject.toml`, `docker/Dockerfile` | | Unit Tests | `unit_test` | `fastvideo/**`, `.buildkite/**`, `.github/**`, `pyproject.toml`, `docker/Dockerfile` | | DreamVerse App Tests | `dreamverse_app` | `apps/dreamverse/**`, `pyproject.toml` | +The transformer launcher has a 25-minute Buildkite timeout around a 20-minute +Modal function. The five-minute margin covers checkout, dependency setup, and +Modal teardown without racing the remote timeout. + ### Tier 3: Full Suite | Attribute | Value | @@ -108,7 +112,7 @@ it), while a GitHub outage or a >25 min wait lets it run anyway (fail open). | Buildkite label | `TEST_TYPE` | Main watched paths | |---|---|---| -| SSIM Tests | `ssim` | `fastvideo/**/*.py`, `pyproject.toml`, `docker/Dockerfile` | +| SSIM Tests | `ssim` | `fastvideo/**/*.py`, except the fingerprint-gated Cosmos3 DiT/config files; also `pyproject.toml` and `docker/Dockerfile` | | LoRA Inference Tests | `inference_lora` | LoRA tests, loader, transformer tests, pipelines, LoRA layers | | LoRA Extraction Tests | `lora_extraction` | LoRA extraction scripts/tests, loader, training utilities, LoRA layers | | Training Tests | `training` | `fastvideo/**`, `pyproject.toml`, `docker/Dockerfile` | @@ -122,6 +126,13 @@ it), while a GitHub outage or a >25 min wait lets it run anyway (fail open). | Train Framework Tests | `train_framework` | `fastvideo/train/**`, train model/method tests, model loader, DiTs | | Eval Metrics Tests | `eval` | `fastvideo/eval/**`, `fastvideo/tests/eval/**`, `pyproject.toml`, `docker/Dockerfile` | +Cosmos3's DiT implementation and DiT config are guarded in Fastcheck by an +exact one-layer transformer fingerprint. A PR that changes only those two +files does not schedule SSIM. The fingerprint test itself is not excluded, so +changing its expected hashes schedules SSIM automatically. Pipeline, packing, +scheduler, encoder, VAE, shared-layer, dependency, and container changes still +schedule end-to-end SSIM. + See [Performance Benchmarks](performance_benchmarks.md) for the performance lane's thresholds, rolling baseline, artifacts, and reseeding process. @@ -241,6 +252,10 @@ All Buildkite test jobs go through `.buildkite/scripts/pr_test.sh`, which: `fastvideo/tests/modal/ssim_test.py`. 5. Uploads performance artifacts for `TEST_TYPE=performance`. +The Modal runner fetches the immutable `BUILDKITE_COMMIT` recorded for the +build. It does not follow a live pull-request ref that may have advanced while +the job was queued. + If you add a new CI test category: 1. Add the Modal function in `fastvideo/tests/modal/pr_test.py` or a focused diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index 7cd5c1060..3d434c5c0 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -33,6 +33,20 @@ pytest fastvideo/tests/vaes -vs GPU-heavy suites need the right hardware, credentials, local caches, and sometimes custom kernels. Document those assumptions in new tests. +## Transformer Fingerprints + +An exact transformer fingerprint runs a fixed, production-shaped one-layer +forward on a pinned GPU/software/backend profile and compares SHA-256 hashes of +the complete output tensors. It loads only the checkpoint tensors used by that +forward. This is the normal Fastcheck gate for a covered DiT; it is bit-exact +within that surface and cheaper than generating media. + +The fingerprint must exercise every implementation file that it exempts from +SSIM. Keep pipeline, sequence packing, scheduler, encoder, VAE, and shared +layer changes on the SSIM path. Updating an expected hash requires official +parity evidence and an end-to-end SSIM run; the fingerprint test remains an +SSIM-watched path so such updates escalate automatically. + ## SSIM Tests SSIM tests generate videos using specific models and parameters, then compare diff --git a/fastvideo/tests/modal/pr_test.py b/fastvideo/tests/modal/pr_test.py index b972458cf..dc2716360 100644 --- a/fastvideo/tests/modal/pr_test.py +++ b/fastvideo/tests/modal/pr_test.py @@ -134,6 +134,10 @@ def _checkout_repository(git_repo: str, if not git_repo or git_repo.startswith("-"): raise RuntimeError("BUILDKITE_REPO must be a non-empty repository URL.") + if re.fullmatch(r"(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})", + git_commit) is None: + raise RuntimeError(f"Invalid BUILDKITE_COMMIT value: {git_commit}") + if pr_number and pr_number != "false": try: pr_id = int(pr_number) @@ -143,15 +147,9 @@ def _checkout_repository(git_repo: str, if pr_id <= 0: raise RuntimeError( f"Invalid BUILDKITE_PULL_REQUEST value: {pr_number}") - target = f"refs/pull/{pr_id}/head" - print(f"Using PR ref for checkout: {target}") - else: - if not git_commit or re.fullmatch(r"[0-9a-fA-F]{7,64}", - git_commit) is None: - raise RuntimeError( - f"Invalid BUILDKITE_COMMIT value: {git_commit}") - target = git_commit - print(f"Using direct commit checkout: {target}") + + target = git_commit + print(f"Using exact Buildkite commit checkout: {target}") clone_command = [ "git", @@ -270,13 +268,13 @@ def run_vae_tests(): @app.function(gpu="L40S:1", image=image, - timeout=900, + timeout=1200, secrets=[hf_secret, ci_env_secret], volumes={"/root/data": model_vol}) def run_transformer_tests(): run_test( "export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && " - "FASTVIDEO_FA4=0 pytest ./fastvideo/tests/transformers -vs" + "CUBLAS_WORKSPACE_CONFIG=:4096:8 FASTVIDEO_FA4=0 pytest ./fastvideo/tests/transformers -vs" ) diff --git a/fastvideo/tests/modal/test_pr_test.py b/fastvideo/tests/modal/test_pr_test.py index 0315043a3..8f5da697f 100644 --- a/fastvideo/tests/modal/test_pr_test.py +++ b/fastvideo/tests/modal/test_pr_test.py @@ -72,7 +72,8 @@ def _load_pr_test_module(monkeypatch): return module -def test_checkout_repository_retries_clone_and_fetches_pr_ref(monkeypatch): +def test_checkout_repository_retries_clone_and_fetches_exact_pr_commit( + monkeypatch): module = _load_pr_test_module(monkeypatch) commands = [] cleanup_paths = [] @@ -131,7 +132,7 @@ def test_checkout_repository_retries_clone_and_fetches_pr_ref(monkeypatch): fetch_command = commands[2][0] assert fetch_command[-2:] == [ "origin", - "refs/pull/1584/head", + "0123456789abcdef0123456789abcdef01234567", ] assert "--depth=1" in fetch_command assert "--filter=blob:none" in fetch_command @@ -211,8 +212,11 @@ def test_git_retry_exhaustion_is_bounded_and_cleans_each_attempt(monkeypatch): ("-bad-option", "0123456789abcdef", "false"), ("https://example.com/repo.git", "", "false"), ("https://example.com/repo.git", "not-a-commit", "false"), - ("https://example.com/repo.git", "0123456789abcdef", "not-a-pr"), - ("https://example.com/repo.git", "0123456789abcdef", "0"), + ("https://example.com/repo.git", "0123456789abcdef", "1584"), + ("https://example.com/repo.git", + "0123456789abcdef0123456789abcdef01234567", "not-a-pr"), + ("https://example.com/repo.git", + "0123456789abcdef0123456789abcdef01234567", "0"), ], ) def test_checkout_repository_rejects_invalid_buildkite_values(