diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 000000000..ee2dd6368 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,66 @@ +env: + IMAGE_VERSION: "py3.12-latest" + +steps: + - block: "Start Build" + blocked_state: "running" + prompt: "Approve build?" + + - label: "Trigger Tests" + command: | + echo "Current working directory: $(pwd)" + echo "Current branch:" + git branch --show-current + echo "Full diff:" + git diff --name-only $BUILDKITE_PULL_REQUEST_BASE_BRANCH...HEAD + plugins: + - monorepo-diff#v1.4.0: + diff: "git diff --name-only $BUILDKITE_PULL_REQUEST_BASE_BRANCH...HEAD" + watch: + - path: + - "fastvideo/v1/models/encoders/**" + - "fastvideo/v1/models/loaders/**" + - "fastvideo/v1/tests/encoders/**" + config: + command: "timeout 30m .buildkite/scripts/pr_test.sh" + label: "Encoder Tests" + env: + - BUILDKITE_CLEAN_CHECKOUT=true + - TEST_TYPE=encoder + agents: + queue: "default" + - path: + - "fastvideo/v1/models/vaes/**" + - "fastvideo/v1/models/loaders/**" + - "fastvideo/v1/tests/vaes/**" + config: + command: "timeout 30m .buildkite/scripts/pr_test.sh" + label: "VAE Tests" + env: + - BUILDKITE_CLEAN_CHECKOUT=true + - TEST_TYPE=vae + agents: + queue: "default" + - path: + - "fastvideo/v1/models/dits/**" + - "fastvideo/v1/models/loaders/**" + - "fastvideo/v1/tests/transformers/**" + - "fastvideo/v1/layers/**" + - "fastvideo/v1/attention/**" + config: + command: "timeout 30m .buildkite/scripts/pr_test.sh" + label: "Transformer Tests" + env: + - BUILDKITE_CLEAN_CHECKOUT=true + - TEST_TYPE=transformer + agents: + queue: "default" + - path: "fastvideo/v1/**/*.py" + config: + command: "timeout 60m .buildkite/scripts/pr_test.sh" + label: "SSIM Tests" + env: + - BUILDKITE_CLEAN_CHECKOUT=true + - TEST_TYPE=ssim + agents: + queue: "default" diff --git a/.buildkite/scripts/pr_test.sh b/.buildkite/scripts/pr_test.sh new file mode 100755 index 000000000..c980426f3 --- /dev/null +++ b/.buildkite/scripts/pr_test.sh @@ -0,0 +1,91 @@ +#!/bin/bash +set -uo pipefail + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" +} + +log "=== Starting Modal test execution ===" + +# Change to the project directory +cd "$(dirname "$0")/../.." +PROJECT_ROOT=$(pwd) +log "Project root: $PROJECT_ROOT" + +# Install Modal if not available +if ! python3 -m modal --version &> /dev/null; then + log "Modal not found, installing..." + python3 -m pip install modal + + # Verify installation + if ! python3 -m modal --version &> /dev/null; then + log "Error: Failed to install modal. Please install it manually." + exit 1 + fi +fi + +log "modal version: $(python3 -m modal --version)" + +# Set up Modal authentication using Buildkite secrets +log "Setting up Modal authentication from Buildkite secrets..." +MODAL_TOKEN_ID=$(buildkite-agent secret get modal_token_id) +MODAL_TOKEN_SECRET=$(buildkite-agent secret get modal_token_secret) + +if [ -n "$MODAL_TOKEN_ID" ] && [ -n "$MODAL_TOKEN_SECRET" ]; then + log "Retrieved Modal credentials from Buildkite secrets" + python3 -m modal token set --token-id "$MODAL_TOKEN_ID" --token-secret "$MODAL_TOKEN_SECRET" --profile buildkite-ci --activate --verify + if [ $? -eq 0 ]; then + log "Modal authentication successful" + else + log "Error: Failed to set Modal credentials" + exit 1 + fi +else + log "Error: Could not retrieve Modal credentials from Buildkite secrets." + log "Please ensure 'modal_token_id' and 'modal_token_secret' secrets are set in Buildkite." + exit 1 +fi + +MODAL_TEST_FILE="fastvideo/v1/tests/modal/pr_test.py" + +if [ -z "${TEST_TYPE:-}" ]; then + log "Error: TEST_TYPE environment variable is not set" + exit 1 +fi +log "Test type: $TEST_TYPE" + +case "$TEST_TYPE" in + "encoder") + log "Running encoder tests..." + MODAL_COMMAND="python3 -m modal run $MODAL_TEST_FILE::run_encoder_tests" + ;; + "vae") + log "Running VAE tests..." + MODAL_COMMAND="python3 -m modal run $MODAL_TEST_FILE::run_vae_tests" + ;; + "transformer") + log "Running transformer tests..." + MODAL_COMMAND="python3 -m modal run $MODAL_TEST_FILE::run_transformer_tests" + ;; + "ssim") + log "Running SSIM tests..." + MODAL_COMMAND="python3 -m modal run $MODAL_TEST_FILE::run_ssim_tests" + ;; + *) + log "Error: Unknown test type: $TEST_TYPE" + exit 1 + ;; +esac + +log "Executing: $MODAL_COMMAND" +eval "$MODAL_COMMAND" +TEST_EXIT_CODE=$? + +if [ $TEST_EXIT_CODE -eq 0 ]; then + log "Modal test completed successfully" +else + log "Error: Modal test failed with exit code: $TEST_EXIT_CODE" +fi + +log "=== Test execution completed with exit code: $TEST_EXIT_CODE ===" +exit $TEST_EXIT_CODE diff --git a/.github/scripts/runpod_api.py b/.github/scripts/runpod_api.py index ac06415da..280332a0c 100644 --- a/.github/scripts/runpod_api.py +++ b/.github/scripts/runpod_api.py @@ -160,8 +160,7 @@ def execute_command(pod_id): setup_steps = [ "tar -xzf /tmp/repo.tar.gz --no-same-owner -C /workspace/", f"cd /workspace/{repo_name}", - "source /opt/conda/etc/profile.d/conda.sh", - "conda activate fastvideo-dev", + "source $HOME/.local/bin/env && source /opt/venv/bin/activate", args.test_command ] diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 6b624ad2c..3205c7443 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -105,7 +105,7 @@ jobs: gpu_count: 1 volume_size: 100 image: "ghcr.io/${{ github.repository }}/${{ github.event.inputs.custom_image || 'fastvideo-dev:latest' }}" - test_command: "pip install -e .[test] && pytest ./fastvideo/v1/tests/encoders -s" + test_command: "uv pip install -e .[test] && pytest ./fastvideo/v1/tests/encoders -s" timeout_minutes: 30 secrets: RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }} @@ -123,7 +123,7 @@ jobs: gpu_count: 1 volume_size: 100 image: "ghcr.io/${{ github.repository }}/${{ github.event.inputs.custom_image || 'fastvideo-dev:latest' }}" - test_command: "pip install -e .[test] && pytest ./fastvideo/v1/tests/vaes -s" + test_command: "uv pip install -e .[test] && pytest ./fastvideo/v1/tests/vaes -s" timeout_minutes: 30 secrets: RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }} @@ -141,7 +141,7 @@ jobs: gpu_count: 1 volume_size: 100 image: "ghcr.io/${{ github.repository }}/${{ github.event.inputs.custom_image || 'fastvideo-dev:latest' }}" - test_command: "pip install -e .[test] && pytest ./fastvideo/v1/tests/transformers -s" + test_command: "uv pip install -e .[test] && pytest ./fastvideo/v1/tests/transformers -s" timeout_minutes: 30 secrets: RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }} @@ -168,7 +168,7 @@ jobs: volume_size: 200 disk_size: 200 image: "ghcr.io/${{ github.repository }}/fastvideo-dev:${{ matrix.python-version.tag }}" - test_command: "pip install -e .[test] && pytest ./fastvideo/v1/tests/ssim -vs" + test_command: "uv pip install -e .[test] && pytest ./fastvideo/v1/tests/ssim -vs" timeout_minutes: 60 secrets: RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }} diff --git a/fastvideo/v1/tests/modal/pr_test.py b/fastvideo/v1/tests/modal/pr_test.py new file mode 100644 index 000000000..2bc20f828 --- /dev/null +++ b/fastvideo/v1/tests/modal/pr_test.py @@ -0,0 +1,98 @@ +import modal + +app = modal.App() + +import os + +image_version = os.getenv("IMAGE_VERSION", "latest") +image_tag = f"ghcr.io/hao-ai-lab/fastvideo/fastvideo-dev:{image_version}" +print(f"Using image: {image_tag}") + +image = ( + modal.Image.from_registry(image_tag, add_python="3.12") + .apt_install("cmake", "pkg-config", "build-essential", "curl", "libssl-dev") + .run_commands("curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable") + .run_commands("echo 'source ~/.cargo/env' >> ~/.bashrc") + .env({"PATH": "/root/.cargo/bin:$PATH"}) + .run_commands("/bin/bash -c 'source $HOME/.local/bin/env && source /opt/venv/bin/activate && cd /FastVideo && uv pip install -e .[test]'") +) + +@app.function(gpu="L40S:1", image=image, timeout=1800) +def run_encoder_tests(): + """Run encoder tests on L40S GPU""" + import subprocess + import sys + import os + + os.chdir("/FastVideo") + + command = """ + source /opt/venv/bin/activate && + pytest ./fastvideo/v1/tests/encoders -s + """ + + result = subprocess.run([ + "/bin/bash", "-c", command + ], stdout=sys.stdout, stderr=sys.stderr, check=False) + + sys.exit(result.returncode) + +@app.function(gpu="L40S:1", image=image, timeout=1800) +def run_vae_tests(): + """Run VAE tests on L40S GPU""" + import subprocess + import sys + import os + + os.chdir("/FastVideo") + + command = """ + source /opt/venv/bin/activate && + pytest ./fastvideo/v1/tests/vaes -s + """ + + result = subprocess.run([ + "/bin/bash", "-c", command + ], stdout=sys.stdout, stderr=sys.stderr, check=False) + + sys.exit(result.returncode) + +@app.function(gpu="L40S:1", image=image, timeout=1800) +def run_transformer_tests(): + """Run transformer tests on L40S GPU""" + import subprocess + import sys + import os + + os.chdir("/FastVideo") + + command = """ + source /opt/venv/bin/activate && + pytest ./fastvideo/v1/tests/transformers -s + """ + + result = subprocess.run([ + "/bin/bash", "-c", command + ], stdout=sys.stdout, stderr=sys.stderr, check=False) + + sys.exit(result.returncode) + +@app.function(gpu="L40S:2", image=image, timeout=3600) +def run_ssim_tests(): + """Run SSIM tests on 2x L40S GPUs""" + import subprocess + import sys + import os + + os.chdir("/FastVideo") + + command = """ + source /opt/venv/bin/activate && + pytest ./fastvideo/v1/tests/ssim -vs + """ + + result = subprocess.run([ + "/bin/bash", "-c", command + ], stdout=sys.stdout, stderr=sys.stderr, check=False) + + sys.exit(result.returncode)