Compare commits

...
Author SHA1 Message Date
SolitaryThinker 3ab4a7f980 [ci] add CI runner 2026-08-19 12:27:50 -07:00
14 changed files with 187 additions and 42 deletions
+7
View File
@@ -90,6 +90,13 @@ steps:
limit: 2
agents:
queue: "default"
- label: ":microscope: Unit Tests"
key: "unit-ci"
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "unit_test_ci"
command: "/opt/fastvideo-ci-runner/run-unit"
timeout_in_minutes: 90
agents:
queue: "ci-runner"
- label: ":microscope: DreamVerse App Tests"
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "dreamverse_app"
command: "timeout 90m .buildkite/scripts/pr_test.sh"
+22
View File
@@ -0,0 +1,22 @@
#!/usr/bin/env bash
set -euo pipefail
exec pytest \
./fastvideo/tests/api/ \
./fastvideo/tests/contract/ \
./fastvideo/tests/dataset/ \
./fastvideo/tests/workflow/ \
./fastvideo/tests/entrypoints/ \
./fastvideo/tests/train/ \
./fastvideo/tests/stages/ \
./fastvideo/tests/ops/ \
./fastvideo/tests/worker/ \
./fastvideo/tests/training/test_trackers.py \
./fastvideo/tests/attention/test_sdpa_metadata_mask_contract.py \
./fastvideo/tests/modal/test_kernel_build_cache.py \
./fastvideo/tests/modal/test_pr_test.py \
./fastvideo/tests/modal/test_ssim_test.py \
--ignore=./fastvideo/tests/entrypoints/test_openai_api_integration.py \
--ignore=./fastvideo/tests/train/models \
--ignore=./fastvideo/tests/train/methods \
-vs
@@ -190,6 +190,7 @@ jobs:
if: ${{ !inputs.push_by_digest }}
run: |
echo "✅ Python ${{ inputs.python_version }} image successfully built and pushed to ${{ steps.image.outputs.name }}:${{ inputs.tag_suffix }}-sha-${GITHUB_SHA::7}"
echo "Digest: ${{ steps.build-push.outputs.digest }}"
echo "To run tests with this image, manually trigger the 'Run Tests' workflow."
- name: Digest success message
+3 -2
View File
@@ -129,7 +129,7 @@ jobs:
set -euo pipefail
TEST_NAME=$(echo "$COMMENT" | grep -oP '(?<=/test\s)\S+' | head -1 || true)
VALID="encoder vae transformer kernel unit dreamverse ssim golden-gate training lora-inference lora-training lora-extraction distillation self-forcing vsa vmoba performance api train-framework eval full fastcheck pre-commit"
VALID="encoder vae transformer kernel unit unit-ci dreamverse ssim golden-gate training lora-inference lora-training lora-extraction distillation self-forcing vsa vmoba performance api train-framework eval full fastcheck pre-commit"
if [ -z "$TEST_NAME" ] || ! echo "$VALID" | grep -qw "$TEST_NAME"; then
echo "Unknown test: '$TEST_NAME'. Valid: $VALID"
exit 1
@@ -137,7 +137,8 @@ jobs:
declare -A MAP=(
[encoder]=encoder [vae]=vae [transformer]=transformer
[kernel]=kernel_tests [unit]=unit_test [dreamverse]=dreamverse_app
[kernel]=kernel_tests [unit]=unit_test [unit-ci]=unit_test_ci
[dreamverse]=dreamverse_app
[ssim]=ssim [golden-gate]=golden_gate [training]=training
[lora-inference]=inference_lora [lora-training]=training_lora
[lora-extraction]=lora_extraction
+27
View File
@@ -13,6 +13,11 @@ on:
required: false
default: false
type: boolean
build_ci_runner_image:
description: 'Build the ARM64 CUDA 13 CI runner image (sm_100)'
required: false
default: false
type: boolean
# Auto-rebuild the CUDA images when a repository-controlled image input
# changes on main. This includes the trusted SM89 kernel artifact's source,
# metadata/key helper, ABI dependency metadata, and build orchestration.
@@ -198,6 +203,28 @@ jobs:
docker buildx imagetools create "${TAG_ARGS[@]}" "${IMAGE_REFS[@]}"
docker buildx imagetools inspect "${TAGS[0]}"
# The CI runner is ARM64 like DGX Spark, but its GB200 GPUs are sm_100 rather
# than sm_121. Publish a single-architecture variant so Slurm CI can reuse
# the exact prebuilt kernel instead of compiling it in every job.
build-ci-runner-image:
if: ${{ (github.event_name == 'push' && github.repository == 'hao-ai-lab/FastVideo') || github.event.inputs.build_ci_runner_image == 'true' }}
uses: ./.github/workflows/_template-build-image.yml
with:
python_version: '3.12'
dockerfile_path: docker/Dockerfile
tag_suffix: py3.12-cuda13.0.0-sm100
runner: ubuntu-24.04-arm
architecture: arm64
build_args: |
PYTHON_VERSION=3.12
CUDA_VERSION=13.0.0
UV_TORCH_BACKEND=cu130
TORCH_CUDA_ARCH_LIST=10.0
CMAKE_BUILD_PARALLEL_LEVEL=1
FLASH_ATTN_WHEEL_TAG=cu130torch2.12
FLASH_ATTN_WHEEL_RELEASE_ARM64=https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.22
secrets: inherit
# Dreamverse matrix: {backend, UI} x {12.6.3, 13.0.0}, Python 3.12. Torch backend
# matches the base CUDA (cu126 / cu130). Keep these images amd64-only until the
# required FA4 dependency stack is available and validated on arm64.
+4
View File
@@ -89,6 +89,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
zsh \
vim \
curl \
ffmpeg \
libgl1 \
libglib2.0-0 \
gcc-11 \
g++-11 \
clang-11 \
@@ -138,6 +141,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \
source /opt/venv/bin/activate && \
uv pip install --upgrade pip && \
uv pip install --excludes docker/uv-excludes ".[dev]" && \
python -c "import cv2; print('OpenCV', cv2.__version__)" && \
PYTAG=cp$(echo "${PYTHON_VERSION}" | tr -d .) && \
case "${TARGETARCH:-amd64}" in \
amd64) \
@@ -119,12 +119,14 @@ surfaces:
vae_precision: "Precision override pending dedicated typed component precision design."
vae_decode_precision: "Decode-only precision override pending dedicated typed component precision design."
image_encoder_precision: "Precision override pending dedicated typed component precision design."
image_encoder_precisions: "Precision overrides pending dedicated typed component precision design."
text_encoder_precisions: "Precision override pending dedicated typed component precision design."
internal_only:
dit_config: "Legacy internal component config object."
upsampler_config: "Legacy internal component config object."
vae_config: "Legacy internal component config object."
image_encoder_config: "Legacy internal component config object."
image_encoder_configs: "Legacy internal component config objects."
text_encoder_configs: "Legacy internal component config object."
preprocess_text_funcs: "Internal text preprocessing hooks."
postprocess_text_funcs: "Internal text postprocessing hooks."
@@ -361,6 +363,30 @@ surfaces:
sources: [fastvideo.configs.pipelines.matrixgame2.MatrixGame2I2V480PConfig]
num_frames_per_block:
sources: [fastvideo.configs.pipelines.matrixgame2.MatrixGame2I2V480PConfig]
duration_s:
sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig]
spectrogram_frame_rate:
sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig]
latent_downsample_rate:
sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig]
clip_frame_rate:
sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig]
sync_frame_rate:
sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig]
sync_segment_size:
sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig]
sync_segment_stride:
sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig]
sync_downsample_rate:
sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig]
clip_image_size:
sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig]
sync_image_size:
sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig]
clip_batch_size_multiplier:
sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig]
sync_batch_size_multiplier:
sources: [fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig]
audio_channels:
sources:
- fastvideo.configs.pipelines.stable_audio.StableAudioT2AConfig
@@ -375,6 +401,7 @@ surfaces:
- fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig
max_audio_duration_s:
sources:
- fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig
- fastvideo.configs.pipelines.stable_audio.StableAudioT2AConfig
- fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig
sample_size:
@@ -383,6 +410,7 @@ surfaces:
- fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig
sampling_rate:
sources:
- fastvideo.configs.pipelines.mmaudio.MMAudioV2AConfig
- fastvideo.configs.pipelines.stable_audio.StableAudioT2AConfig
- fastvideo.configs.pipelines.stable_audio.StableAudioOpenSmallConfig
audio_txt_guidance_scale:
@@ -4,11 +4,11 @@ the real platform resolver.
``is_attn_qat_infer_available()`` used to test only whether the kernel
extension imports. CUDA 13 wheel builds can carry the sm_120/sm_121
extension on any host (e.g. H100 sm_90, GB200 sm_100): the import
succeeds, ``CudaPlatformBase.get_attn_backend_cls`` selects the
consumer-Blackwell backend, and the first kernel call fails with an
unsupported-capability error -- instead of the FlashAttention fallback
the QAD README documents for non-sm_120 GPUs.
extension on any host (e.g. H100 sm_90, GB200 sm_100). Without the
capability gate, that successful import selects the consumer-Blackwell
backend and defers failure until the first unsupported kernel call.
Explicit ATTN_QAT_INFER requests must instead fail closed during
resolution.
These tests drive the REAL resolver (``fastvideo.platforms.cuda``) and
the REAL availability function; only the two physical facts are faked --
@@ -17,7 +17,7 @@ active" (``torch.cuda``). The stage-guard test
(fastvideo/tests/stages/test_kandinsky5_attention_backend_guard.py)
injects an already-resolved backend and by design cannot see this bug.
CPU-only: the ATTN_QAT_INFER branch and its fallback never require a
CPU-only: the ATTN_QAT_INFER branch and its error path never require a
physical GPU to *resolve* (only to run).
"""
from __future__ import annotations
@@ -35,13 +35,6 @@ from fastvideo.platforms.cuda import NonNvmlCudaPlatform
from fastvideo.platforms.interface import AttentionBackendEnum
ATTN_QAT_INFER_CLS = "fastvideo.attention.backends.attn_qat_infer.AttnQatInferBackend"
# What the resolver's fallthrough legitimately returns when ATTN_QAT_INFER
# is unavailable: FlashAttention, or SDPA when flash_attn isn't installed
# in the running environment (e.g. CPU-only CI).
FALLBACK_CLASSES = {
"fastvideo.attention.backends.flash_attn.FlashAttentionBackend",
"fastvideo.attention.backends.sdpa.SDPABackend",
}
def _fake_gpu(monkeypatch, *, capability: tuple[int, int], extension_imports: bool, fa4_imports: bool = False) -> None:
@@ -66,22 +59,24 @@ def _resolve() -> str:
)
def test_sm90_host_with_bundled_extension_falls_back(monkeypatch):
def test_sm90_host_with_bundled_extension_fails_closed(monkeypatch):
"""The reviewed failure: H100 + CUDA 13 wheel that bundles the sm_120
extension. Import succeeds; selection must still fall back."""
extension. Import succeeds; explicit selection must still fail closed."""
_fake_gpu(monkeypatch, capability=(9, 0), extension_imports=True)
assert not is_attn_qat_infer_available()
assert _resolve() in FALLBACK_CLASSES
with pytest.raises(ImportError, match="ATTN_QAT_INFER selected but"):
_resolve()
def test_sm100_host_with_bundled_extension_falls_back(monkeypatch):
def test_sm100_host_with_bundled_extension_fails_closed(monkeypatch):
"""sm_100 with only the (unrunnable) bundled sm_12x extension and no
FP4 FA4 kernel still falls back -- the original reviewed failure class."""
FP4 FA4 kernel still fails closed -- the original reviewed failure class."""
_fake_gpu(monkeypatch, capability=(10, 0), extension_imports=True, fa4_imports=False)
assert not is_attn_qat_infer_available()
assert _resolve() in FALLBACK_CLASSES
with pytest.raises(ImportError, match="ATTN_QAT_INFER selected but"):
_resolve()
@pytest.mark.parametrize("capability", [(10, 0), (10, 3)])
@@ -102,11 +97,12 @@ def test_consumer_blackwell_with_extension_selects_backend(monkeypatch, capabili
assert _resolve() == ATTN_QAT_INFER_CLS
def test_consumer_blackwell_without_extension_falls_back(monkeypatch):
def test_consumer_blackwell_without_extension_fails_closed(monkeypatch):
_fake_gpu(monkeypatch, capability=(12, 0), extension_imports=False)
assert not is_attn_qat_infer_available()
assert _resolve() in FALLBACK_CLASSES
with pytest.raises(ImportError, match="ATTN_QAT_INFER selected but"):
_resolve()
def test_no_cuda_reports_unavailable(monkeypatch):
@@ -23,6 +23,7 @@ CI_SOURCES = [
TESTS_ROOT / "modal" / "ssim_test.py",
*sorted((REPO_ROOT / ".buildkite").rglob("*.yml")),
*sorted((REPO_ROOT / ".buildkite").rglob("*.sh")),
*sorted((REPO_ROOT / ".github/workflows").glob("ci-*.yml")),
]
# Directories that intentionally have no CI lane today. Every entry needs a
@@ -68,6 +69,22 @@ def test_local_tests_stays_out_of_ci():
"reference or move the tests into a fastvideo/tests/ lane.")
def test_unit_ci_routes_to_trusted_static_driver():
slash_commands = (REPO_ROOT / ".github/workflows/ci-slash-commands.yml").read_text()
pipeline = (REPO_ROOT / ".buildkite/pipeline.yml").read_text()
valid_line = next(line for line in slash_commands.splitlines() if line.strip().startswith("VALID="))
assert "unit-ci" in valid_line
assert "[unit-ci]=unit_test_ci" in slash_commands
assert ''' - label: ":microscope: Unit Tests"
key: "unit-ci"
if: build.env("TEST_SCOPE") == "direct" && build.env("TEST_TYPE") == "unit_test_ci"
command: "/opt/fastvideo-ci-runner/run-unit"
timeout_in_minutes: 90
agents:
queue: "ci-runner"''' in pipeline
def test_allowlist_entries_are_still_real_directories():
# A stale allowlist hides regressions; entries must track reality.
missing = [name for name in ALLOWLIST if name != "modal" and not (TESTS_ROOT / name).is_dir()]
@@ -27,13 +27,13 @@ def _function_strings(path: Path, function_name: str) -> str:
def test_generic_l40s_launcher_defaults_fa4_off():
source = LAUNCH_L40S_JOB.read_text(encoding="utf-8")
assert '"FASTVIDEO_FA4": os.environ.get("FASTVIDEO_FA4", "0")' in source
source = ast.unparse(ast.parse(LAUNCH_L40S_JOB.read_text(encoding="utf-8")))
assert "'FASTVIDEO_FA4': os.environ.get('FASTVIDEO_FA4', '0')" in source
def test_ssim_launcher_keeps_fa4_enabled_by_default():
source = SSIM_TEST.read_text(encoding="utf-8")
assert '"FASTVIDEO_FA4": os.environ.get("FASTVIDEO_FA4", "1")' in source
source = ast.unparse(ast.parse(SSIM_TEST.read_text(encoding="utf-8")))
assert "'FASTVIDEO_FA4': os.environ.get('FASTVIDEO_FA4', '1')" in source
def test_performance_identity_env_reaches_modal_runtime():
+10 -2
View File
@@ -22,7 +22,7 @@ import zipfile
from email.parser import Parser
from pathlib import Path
CACHE_SCHEMA_VERSION = 3
CACHE_SCHEMA_VERSION = 4
DEFAULT_PREBUILT_INFO_PATH = "/opt/fastvideo-kernel-prebuilt"
KERNEL_RELATIVE_DIR = "fastvideo-kernel"
DEFAULT_BUILD_INFO_OUTPUT = "/opt/fastvideo-kernel-prebuilt/default/metadata.json"
@@ -166,6 +166,7 @@ def _torch_metadata() -> dict[str, object]:
return {
"torch_version": str(torch.__version__),
"torch_cuda_version": str(torch.version.cuda),
"torch_git_version": str(getattr(torch.version, "git_version", "")),
"torch_file": str(getattr(torch, "__file__", "")),
"torch_config": str(torch.__config__.show()),
"cxx11_abi": cxx11_abi,
@@ -174,6 +175,7 @@ def _torch_metadata() -> dict[str, object]:
return {
"torch_version": f"<unavailable: {error}>",
"torch_cuda_version": "<unavailable>",
"torch_git_version": "<unavailable>",
"torch_file": "<unavailable>",
"torch_config": "<unavailable>",
"cxx11_abi": "<unavailable>",
@@ -223,6 +225,11 @@ def _compiler_libc_metadata() -> dict[str, object]:
def _build_metadata(repo_root: Path) -> dict[str, object]:
explicit_arch = os.environ.get("TORCH_CUDA_ARCH_LIST", "").strip()
resolved_arch = explicit_arch or _detect_arch_from_torch()
torch_metadata = _torch_metadata()
torch_cache_metadata = {
name: torch_metadata[name]
for name in ("torch_version", "torch_cuda_version", "torch_git_version", "cxx11_abi")
}
cache_key_build = {
"gpu_backend": os.environ.get("GPU_BACKEND", "CUDA"),
"resolved_torch_cuda_arch_list": resolved_arch,
@@ -242,7 +249,7 @@ def _build_metadata(repo_root: Path) -> dict[str, object]:
"platform": sysconfig.get_platform(),
"machine": platform.machine(),
},
"torch": _torch_metadata(),
"torch": torch_cache_metadata,
"cuda": {
"cuda_home": os.environ.get("CUDA_HOME", ""),
"nvcc": _selected_command_metadata("CUDACXX", "nvcc"),
@@ -252,6 +259,7 @@ def _build_metadata(repo_root: Path) -> dict[str, object]:
}
metadata = {
**cache_key_metadata,
"torch": torch_metadata,
"build": {
**cache_key_build,
"torch_cuda_arch_list": explicit_arch,
+1 -9
View File
@@ -349,15 +349,7 @@ def run_self_forcing_tests():
@app.function(gpu="L40S:1", image=image, timeout=900, secrets=[ci_env_secret])
def run_unit_test():
run_test("pytest ./fastvideo/tests/api/ ./fastvideo/tests/contract/ ./fastvideo/tests/dataset/ "
"./fastvideo/tests/workflow/ ./fastvideo/tests/entrypoints/ ./fastvideo/tests/train/ "
"./fastvideo/tests/stages/ ./fastvideo/tests/ops/ ./fastvideo/tests/worker/ "
"./fastvideo/tests/training/test_trackers.py "
"./fastvideo/tests/attention/test_sdpa_metadata_mask_contract.py "
"./fastvideo/tests/modal/test_kernel_build_cache.py ./fastvideo/tests/modal/test_pr_test.py "
"./fastvideo/tests/modal/test_ssim_test.py "
"--ignore=./fastvideo/tests/entrypoints/test_openai_api_integration.py "
"--ignore=./fastvideo/tests/train/models --ignore=./fastvideo/tests/train/methods -vs")
run_test("bash .buildkite/scripts/unit_test.sh")
# TODO: David: GPU only used to resolve import time requirement (not needed for this test). Maybe make those imports lazy?
@@ -106,6 +106,7 @@ def _patch_stable_metadata(monkeypatch) -> None:
lambda: {
"torch_version": "2.9.0",
"torch_cuda_version": "12.8",
"torch_git_version": "stable-build-commit",
"torch_file": "/opt/venv/lib/python3.12/site-packages/torch/__init__.py",
"torch_config": "USE_CUDA=ON",
"cxx11_abi": True,
@@ -155,6 +156,27 @@ def test_kernel_only_main_change_republishes_trusted_l40s_artifact() -> None:
assert '--output "${l40s_wheel_dir}/metadata.json"' in dockerfile
def test_ci_runner_image_targets_arm64_sm100_with_opencv_runtime() -> None:
workflow = yaml.load(
(REPO_ROOT / ".github/workflows/infra-build-image.yml").read_text(encoding="utf-8"),
Loader=yaml.BaseLoader,
)
job = workflow["jobs"]["build-ci-runner-image"]
assert job["with"]["architecture"] == "arm64"
assert job["with"]["runner"] == "ubuntu-24.04-arm"
assert job["with"]["tag_suffix"] == "py3.12-cuda13.0.0-sm100"
assert "CUDA_VERSION=13.0.0" in job["with"]["build_args"]
assert "UV_TORCH_BACKEND=cu130" in job["with"]["build_args"]
assert "TORCH_CUDA_ARCH_LIST=10.0" in job["with"]["build_args"]
dockerfile = (REPO_ROOT / "docker/Dockerfile").read_text(encoding="utf-8")
assert " ffmpeg \\\n" in dockerfile
assert " libgl1 \\\n" in dockerfile
assert " libglib2.0-0 \\\n" in dockerfile
assert 'python -c "import cv2; print(\'OpenCV\', cv2.__version__)"' in dockerfile
def test_cache_key_uses_resolved_arch_not_raw_env(monkeypatch, tmp_path) -> None:
_patch_stable_metadata(monkeypatch)
monkeypatch.setattr(kernel_build_cache, "_detect_arch_from_torch", lambda: "9.0a")
@@ -174,6 +196,25 @@ def test_cache_key_uses_resolved_arch_not_raw_env(monkeypatch, tmp_path) -> None
assert detected_l40s["cache_key"] != detected_hopper["cache_key"]
def test_cache_key_ignores_runtime_only_torch_config(monkeypatch, tmp_path) -> None:
_patch_stable_metadata(monkeypatch)
build_host = kernel_build_cache._build_metadata(tmp_path)
torch_metadata = kernel_build_cache._torch_metadata()
monkeypatch.setattr(
kernel_build_cache,
"_torch_metadata",
lambda: {
**torch_metadata,
"torch_config": torch_metadata["torch_config"] + "\nCUDA Runtime 13.0\nCuDNN 92.0",
},
)
gpu_host = kernel_build_cache._build_metadata(tmp_path)
assert gpu_host["torch"]["torch_config"] != build_host["torch"]["torch_config"]
assert gpu_host["cache_key"] == build_host["cache_key"]
@pytest.mark.parametrize("environment_name", ["CFLAGS", "CXXFLAGS", "LDFLAGS"])
def test_cache_key_changes_with_build_flags(monkeypatch, tmp_path, environment_name) -> None:
_patch_stable_metadata(monkeypatch)
@@ -213,7 +254,7 @@ def test_cache_key_changes_with_compiler_or_torch_abi(monkeypatch, tmp_path) ->
monkeypatch.setattr(kernel_build_cache, "_torch_metadata", lambda: {**torch_metadata, "cxx11_abi": False})
torch_abi_changed = kernel_build_cache._build_metadata(tmp_path)
assert baseline["schema_version"] == 3
assert baseline["schema_version"] == 4
assert baseline["cache_key"] != compiler_changed["cache_key"]
assert baseline["cache_key"] != torch_abi_changed["cache_key"]
+4 -3
View File
@@ -316,17 +316,18 @@ def test_run_test_command_uses_nonshared_kernel_install_before_tests(monkeypatch
assert not hasattr(module, "kernel_cache_vol")
def test_run_unit_test_collects_modal_cache_runner_tests(monkeypatch):
def test_run_unit_test_uses_shared_command(monkeypatch):
module = _load_pr_test_module(monkeypatch)
commands = []
monkeypatch.setattr(module, "run_test", commands.append)
module.run_unit_test()
assert len(commands) == 1
assert commands == ["bash .buildkite/scripts/unit_test.sh"]
unit_command = (Path(__file__).resolve().parents[3] / ".buildkite/scripts/unit_test.sh").read_text()
for test_path in (
"./fastvideo/tests/modal/test_kernel_build_cache.py",
"./fastvideo/tests/modal/test_pr_test.py",
"./fastvideo/tests/modal/test_ssim_test.py",
):
assert test_path in commands[0]
assert test_path in unit_command