Compare commits

...
Author SHA1 Message Date
Jinzhe Pan 15c79dda56 [ci]: fix fork PR checkout — pass PR_NUMBER through Buildkite to Modal for ref-based checkout 2026-03-31 01:32:20 -04:00
Jinzhe Pan f7ad5ef7e0 [ci]: remove deprecated delete_head_branch rule 2026-03-30 12:55:44 -04:00
Jinzhe Pan 2cb6bdd0b2 [ci]: remove Buildkite CI failure comment rule 2026-03-30 12:52:25 -04:00
Jinzhe Pan 4df8a7e209 [ci]: cancel old Buildkite builds and re-trigger Full Suite on new commits 2026-03-30 12:45:23 -04:00
Jinzhe Pan 6371f19638 [ci]: prevent auto-rebase from interrupting Full Suite — require full-suite-passed 2026-03-30 12:37:43 -04:00
Jinzhe Pan 78173b6741 [ci]: restore missing Trigger Full Suite label in pipeline.yml 2026-03-30 12:29:34 -04:00
Jinzhe Pan a3e1591f93 [docs]: update CI docs — replace Merge Queue with auto-merge flow 2026-03-30 12:20:39 -04:00
Jinzhe Pan 9225604322 [ci]: remove merge-queue branch exception from pre-commit draft skip 2026-03-30 12:17:11 -04:00
Jinzhe Pan 98f628a49c [ci]: simplify /merge command — remove @mergifyio queue, use label-based trigger 2026-03-30 12:16:45 -04:00
Jinzhe Pan cc4889de98 [ci]: update Buildkite notify and Full Suite gate — remove queue branch detection 2026-03-30 12:16:21 -04:00
Jinzhe Pan 05eb6ccc60 [ci]: recreate Full Suite trigger workflow — labeled event with ready label check 2026-03-30 12:15:42 -04:00
Jinzhe Pan 4da7beccbc [ci]: replace Merge Queue with auto-merge flow — add auto-rebase and CI failure label removal 2026-03-30 12:15:22 -04:00
9 changed files with 256 additions and 232 deletions
+4 -4
View File
@@ -7,7 +7,7 @@ notify:
context: "fastcheck-passed"
- github_commit_status:
context: "full-suite-passed"
if: build.branch =~ /^mergify\/merge-queue\//
if: build.env("FULL_SUITE") == "true"
steps:
# ============================================================
@@ -93,14 +93,14 @@ steps:
queue: "default"
# ============================================================
# Full Suite: Runs on Merge Queue branches or when FULL_SUITE=true
# Triggered by Mergify Merge Queue (mergify/merge-queue/* branches)
# Full Suite: Runs when FULL_SUITE=true
# Triggered by adding the 'ready' label (via ci-trigger-full-suite.yml)
# or on-demand via /test full slash command.
# Includes integration tests, SSIM regression, training pipelines,
# and performance benchmarks.
# ============================================================
- label: "Trigger Full Suite"
if: build.env("FULL_SUITE") == "true" || build.branch =~ /^mergify\/merge-queue\//
if: build.env("FULL_SUITE") == "true"
retry:
automatic:
- exit_status: 128
+5 -1
View File
@@ -59,7 +59,11 @@ if [ -z "${TEST_TYPE:-}" ]; then
fi
log "Test type: $TEST_TYPE"
MODAL_ENV="BUILDKITE_REPO=$BUILDKITE_REPO BUILDKITE_COMMIT=$BUILDKITE_COMMIT BUILDKITE_PULL_REQUEST=$BUILDKITE_PULL_REQUEST IMAGE_VERSION=$IMAGE_VERSION"
EFFECTIVE_PR=${BUILDKITE_PULL_REQUEST:-false}
if [ "$EFFECTIVE_PR" = "false" ] && [ -n "${PR_NUMBER:-}" ]; then
EFFECTIVE_PR=$PR_NUMBER
fi
MODAL_ENV="BUILDKITE_REPO=$BUILDKITE_REPO BUILDKITE_COMMIT=$BUILDKITE_COMMIT BUILDKITE_PULL_REQUEST=$EFFECTIVE_PR IMAGE_VERSION=$IMAGE_VERSION"
case "$TEST_TYPE" in
"encoder")
+33 -56
View File
@@ -1,22 +1,3 @@
queue_rules:
- name: default
queue_conditions:
- "title~=(?i)^\\[(feat|feature|bugfix|fix|refactor|perf|ci|doc|docs|misc|chore|kernel|new.?model)\\]"
- check-success~=pre-commit
- check-success=fastcheck-passed
- "#approved-reviews-by>=1"
- -draft
- -conflict
- -closed
- label!=do-not-merge
merge_conditions:
- check-success=full-suite-passed
merge_method: squash
commit_message_template: |
{{ title }} (#{{ number }})
{{ body }}
merge_protections:
- name: PR merge requirements
if:
@@ -239,32 +220,6 @@ pull_request_rules:
For future commits, `pre-commit` will run automatically on changed files before each commit.
# ============================================================
# Buildkite CI failure notification
# ============================================================
- name: comment on Buildkite CI failure
conditions:
- check-failure~=buildkite/ci
- -closed
actions:
comment:
message: |
## ❌ CI tests failed
@{{author}} — to see what failed:
1. Scroll to the **Checks** section below
2. Find the check marked with ❌ (e.g. `buildkite/ci/microscope-transformer-tests`)
3. Click **Details** to view the full build log
Or [view all builds for this branch on Buildkite →](https://buildkite.com/fastvideo/ci/builds?branch={{head}})
**Common causes:**
- **Assertion error / test failure** — check the failing test's traceback
- **Import error** — new dependency missing from `pyproject.toml`
- **OOM** — some tests need specific GPUs (L40S, H100 NVL)
If the failure looks unrelated to your changes, comment why and a maintainer will review.
# ============================================================
# Merge conflict detection
@@ -298,20 +253,48 @@ pull_request_rules:
label:
remove: [needs-rebase]
- name: enter merge queue when ready and approved
# ============================================================
# Auto-merge and auto-rebase
# ============================================================
- name: auto-merge when ready and all checks pass
conditions:
- label=ready
- "title~=(?i)^\\[(feat|feature|bugfix|fix|refactor|perf|ci|doc|docs|misc|chore|kernel|new.?model)\\]"
- "#approved-reviews-by>=1"
- check-success~=pre-commit
- check-success=fastcheck-passed
- check-success=full-suite-passed
- -conflict
- -closed
- -draft
actions:
queue:
name: default
merge:
method: squash
- name: remove ready label on dequeue
- name: auto-rebase when ready and Full Suite passed
conditions:
- queue-dequeue-reason!=none
- label=ready
- "#approved-reviews-by>=1"
- check-success=full-suite-passed
- -conflict
- -closed
- -draft
actions:
rebase: {}
- name: remove ready label on Full Suite failure
conditions:
- label=ready
- check-failure=full-suite-passed
actions:
label:
remove: [ready]
# ============================================================
# PR title format help
# ============================================================
- name: comment on invalid PR title format
conditions:
- -closed
@@ -336,9 +319,3 @@ pull_request_rules:
Please update your PR title and the merge protection check will pass automatically.
- name: delete merged branch
conditions:
- merged
- -head~=^(main|master|release[/-])$
actions:
delete_head_branch: {}
+1 -3
View File
@@ -14,9 +14,7 @@ permissions:
jobs:
pre-commit:
if: >-
github.event.pull_request.draft != true
|| startsWith(github.event.pull_request.head.ref, 'mergify/merge-queue/')
if: github.event.pull_request.draft != true
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
+4 -3
View File
@@ -31,7 +31,7 @@ jobs:
}
core.setOutput('has_write', String(hasWrite));
- name: Queue PR via Mergify and react
- name: Add ready label and react
if: steps.perm.outputs.has_write == 'true'
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
@@ -39,8 +39,9 @@ jobs:
const owner = context.repo.owner;
const repo = context.repo.repo;
const prNumber = context.payload.issue.number;
// Remove ready first to allow re-trigger (labeled event fires on add, not if already present)
try { await github.rest.issues.removeLabel({ owner, repo, issue_number: prNumber, name: 'ready' }); } catch {}
await github.rest.issues.addLabels({ owner, repo, issue_number: prNumber, labels: ['ready'] });
await github.rest.issues.createComment({ owner, repo, issue_number: prNumber, body: '@mergifyio queue' });
await github.rest.reactions.createForIssueComment({
owner, repo,
comment_id: context.payload.comment.id,
@@ -151,7 +152,7 @@ jobs:
branch: "${{ needs.parse-command.outputs.pr_branch }}"
commit: "${{ needs.parse-command.outputs.pr_sha }}"
message: "/test ${{ needs.parse-command.outputs.test_type }} on PR #${{ github.event.issue.number }}"
build_env_vars: '{"FULL_SUITE": "${{ needs.parse-command.outputs.full_suite }}", "TEST_TYPE": "${{ needs.parse-command.outputs.test_type }}"}'
build_env_vars: '{"FULL_SUITE": "${{ needs.parse-command.outputs.full_suite }}", "TEST_TYPE": "${{ needs.parse-command.outputs.test_type }}", "PR_NUMBER": "${{ github.event.issue.number }}"}'
send_pull_request: "true"
pull_request_id: "${{ github.event.issue.number }}"
pull_request_base_branch: "main"
@@ -0,0 +1,64 @@
name: Trigger Full Suite
on:
pull_request:
types: [labeled, synchronize]
permissions:
contents: read
pull-requests: read
concurrency:
group: full-suite-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
trigger:
if: >-
(github.event.action == 'labeled' && github.event.label.name == 'ready')
|| github.event.action == 'synchronize'
runs-on: ubuntu-latest
steps:
- name: Check ready label
id: check
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
script: |
const { data: pr } = await github.rest.pulls.get({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.payload.pull_request.number,
});
const hasReady = pr.labels.some(l => l.name === 'ready');
core.setOutput('has_ready', String(hasReady));
if (!hasReady) core.info('No ready label — skipping Full Suite trigger.');
- name: Cancel previous Buildkite builds
if: steps.check.outputs.has_ready == 'true'
env:
BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
PR_BRANCH: ${{ github.event.pull_request.head.ref }}
run: |
# Find running builds for this branch with FULL_SUITE=true and cancel them
builds=$(curl -sS -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
"https://api.buildkite.com/v2/organizations/${{ vars.BUILDKITE_ORG_SLUG }}/pipelines/${{ vars.BUILDKITE_PIPELINE_SLUG }}/builds?branch=${PR_BRANCH}&state=running,scheduled" \
| jq -r '.[] | select(.env.FULL_SUITE == "true") | .number')
for build_num in $builds; do
echo "Cancelling Buildkite build #$build_num"
curl -sS -X PUT -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
"https://api.buildkite.com/v2/organizations/${{ vars.BUILDKITE_ORG_SLUG }}/pipelines/${{ vars.BUILDKITE_PIPELINE_SLUG }}/builds/${build_num}/cancel"
done
- name: Trigger Buildkite Full Suite
if: steps.check.outputs.has_ready == 'true'
uses: buildkite/trigger-pipeline-action@v2.4.1
with:
buildkite_api_access_token: ${{ secrets.BUILDKITE_API_TOKEN }}
pipeline: "${{ vars.BUILDKITE_ORG_SLUG }}/${{ vars.BUILDKITE_PIPELINE_SLUG }}"
branch: "${{ github.event.pull_request.head.ref }}"
commit: "${{ github.event.pull_request.head.sha }}"
message: "Full Suite for PR #${{ github.event.pull_request.number }}"
build_env_vars: '{"FULL_SUITE": "true", "PR_NUMBER": "${{ github.event.pull_request.number }}"}'
send_pull_request: "true"
pull_request_id: "${{ github.event.pull_request.number }}"
pull_request_base_branch: "main"
+36 -31
View File
@@ -16,18 +16,15 @@ PR push
Buildkite / Modal GPU instances
Only runs tests for paths you changed
│ (maintainer adds 'ready' label or comments /merge)
▼
Merge Queue
Mergify creates mergify/merge-queue/<branch>
│
│ (developer comments /merge or maintainer adds 'ready' label)
▼
Tier 3: Full Suite (~60-90 min)
Buildkite / Modal GPU instances
All integration, SSIM, training, and performance tests
Runs on the PR branch directly
│
pass ──► squash-merge to main, branch deleted
fail ──► PR ejected from queue; fix and re-queue
pass ──► Mergify auto-squash-merges to main, branch deleted
fail ──► Mergify removes 'ready' label; fix and /merge again
```
---
@@ -82,11 +79,11 @@ failing test's output.
---
### Tier 3: Full Test Suite (Merge Queue only)
### Tier 3: Full Test Suite (triggered by `ready` label)
| Attribute | Value |
|-----------|-------|
| Triggered by | Mergify creating a `mergify/merge-queue/*` branch, OR a `/test full` command |
| Triggered by | Adding the `ready` label to the PR (via `/merge` command), or a `/test full` command |
| Runs on | Buildkite, Modal GPU instances |
| Duration | 60-90 minutes total (tests run in parallel, path-filtered) |
@@ -105,44 +102,48 @@ failing test's output.
| Performance Tests | `performance` | 30 min |
| API Server Tests | `api_server` | 30 min |
A Full Suite failure means the PR is ejected from the Merge Queue. A Mergify comment will
link to the Buildkite build. Fix the regression, push, and re-queue.
A Full Suite failure removes the `ready` label automatically. A Mergify comment links to
the Buildkite build. Fix the regression, push, and comment `/merge` again.
---
## Merge Queue
## Auto-merge Flow
The Merge Queue prevents untested code from landing on `main`. Mergify manages the queue;
Buildkite validates each entry.
Mergify prevents untested code from landing on `main` by gating squash-merge on the Full
Suite passing directly on the PR branch.
**How it works:**
1. A developer comments `/merge` on an approved PR (or a maintainer adds the `ready` label).
2. The `ready` label triggers the Mergify rule `enter merge queue when ready and approved`.
3. Mergify checks the **queue conditions** before accepting:
- `pre-commit` check must be green
2. The `ready` label triggers `ci-trigger-full-suite.yml`, which calls the Buildkite API to
run the Full Suite on the PR branch itself.
3. While the Full Suite runs, Mergify also auto-rebases the PR branch against `main` if it
is behind and has no conflicts.
4. Once the Full Suite posts `full-suite-passed`, Mergify checks all **merge conditions**:
- `pre-commit` check is green
- `fastcheck-passed` check is green
- `full-suite-passed` check is green
- At least 1 approved review (`#approved-reviews-by>=1`)
- PR title starts with a valid `[type]` tag
- PR is not a draft
- No merge conflicts
- Not closed
- No `do-not-merge` label
4. If conditions pass, Mergify creates a temporary branch: `mergify/merge-queue/<branch-name>`
5. Buildkite Full Suite runs on that branch (triggered by the branch name pattern).
6. Once all Full Suite checks pass (`check-success~=buildkite/ci`), Mergify squash-merges
to `main` with the commit message: `<title> (#<number>)` followed by the PR body.
7. If any Full Suite test fails, the PR is ejected from the queue and Mergify posts a comment.
The developer fixes the issue, pushes, and comments `/merge` again.
5. If all conditions pass, Mergify squash-merges to `main` automatically. The branch is
deleted after merge.
6. If the Full Suite fails, Mergify removes the `ready` label and posts a comment linking to
the Buildkite build. The developer fixes the issue, pushes, and comments `/merge` again.
**Queue conditions summary:**
**Merge conditions summary:**
| Condition | Meaning |
|-----------|---------|
| `check-success~=pre-commit` | Tier 1 pre-commit must be green |
| `check-success=fastcheck-passed` | Tier 2 Fastcheck must be green |
| `check-success=full-suite-passed` | Tier 3 Full Suite must be green |
| `#approved-reviews-by>=1` | At least one approved review |
| `-draft` | Not a draft PR |
| `title~=(?i)^\[(feat|bugfix|...)` | PR title has a valid type tag |
| `-draft` | PR is not in draft state |
| `-conflict` | No merge conflicts with base branch |
| `-closed` | PR is still open |
| `label!=do-not-merge` | No `do-not-merge` label present |
---
@@ -186,7 +187,7 @@ Applied by Mergify based on which paths you modified. Multiple scope labels can
| Label | Who sets it | Meaning |
|-------|-------------|---------|
| `ready` | Developer (`/merge` command) or maintainer | Enters the Merge Queue |
| `ready` | Developer (`/merge` command) or maintainer | Triggers Full Suite and enables auto-merge |
| `needs-rebase` | Mergify (automatic) | PR has merge conflicts; rebase needed |
| `do-not-merge` | Maintainer | Blocks queue entry regardless of other conditions |
@@ -245,8 +246,11 @@ The command is recognized within a few seconds. The workflow reacts with a 🚀
/merge
```
Adds the `ready` label to the PR, which triggers Merge Queue entry (subject to queue
conditions being met).
Adds the `ready` label to the PR, which triggers the Full Suite on your PR branch and
enables Mergify to auto-squash-merge once all conditions pass.
The command first removes the `ready` label if it is already present, then re-adds it. This
ensures the `labeled` event fires and a fresh Full Suite build is started even on a re-try.
### `/test <name>`
@@ -290,6 +294,7 @@ Protected branches (`main`, `master`, `release/*`) are never deleted.
| Filename | Trigger | What it does |
|----------|---------|-------------|
| `ci-precommit.yml` | Every push / PR against `main` | Runs pre-commit hooks (yapf, ruff, mypy, codespell, pymarkdown, actionlint, check-filenames) |
| `ci-trigger-full-suite.yml` | `ready` label added to a PR | Calls Buildkite API to run Full Suite on the PR branch |
| `ci-slash-commands.yml` | PR comment starting with `/merge` or `/test` | Handles slash commands; adds `ready` label or triggers Buildkite |
| `community-issue-labeler.yml` | Issue opened or edited | Auto-labels issues by keyword matching against title and body |
| `community-welcome.yml` | First contribution | Posts a welcome comment for first-time contributors |
+18 -18
View File
@@ -62,7 +62,7 @@ need to set them manually.
| Label | Who sets it | Meaning |
|-------|-------------|---------|
| `ready` | You (`/merge` comment) or a maintainer | Requests entry into the Merge Queue |
| `ready` | You (`/merge` comment) or a maintainer | Triggers Full Suite and enables auto-merge |
| `needs-rebase` | Mergify (automatic) | Your PR has conflicts; rebase against `main` |
| `do-not-merge` | Maintainer | Blocks merge regardless of CI status |
@@ -82,11 +82,11 @@ hooks: yapf, ruff, mypy, codespell, pymarkdown, actionlint, and check-filenames.
Buildkite runs GPU tests only for the components you changed. If you only modified
`fastvideo/models/vaes/`, only VAE Tests run. Tests run in parallel.
**Tier 3: Full Suite (~60-90 min) — runs only in the Merge Queue**
**Tier 3: Full Suite (~60-90 min) — triggered by the `ready` label**
When your PR enters the Merge Queue, Buildkite runs the complete test suite: SSIM
regression, LoRA inference and training, distillation, self-forcing, VSA, VMoBA,
performance benchmarks, and API server tests.
When you comment `/merge` (or a maintainer adds the `ready` label), Buildkite runs the
complete test suite on your PR branch: SSIM regression, LoRA inference and training,
distillation, self-forcing, VSA, VMoBA, performance benchmarks, and API server tests.
---
@@ -99,14 +99,13 @@ performance benchmarks, and API server tests.
3. Fix any pre-commit failures locally (`pre-commit run --all-files`) and push again.
4. Wait for at least one approving review.
5. Once approved and pre-commit is green, comment `/merge` on the PR.
6. Mergify adds the `ready` label and checks queue conditions (approval, no draft, no
conflicts, no `do-not-merge`).
7. If conditions pass, Mergify creates a `mergify/merge-queue/<branch>` branch and runs
the Full Suite.
8. If all Full Suite tests pass, your PR is squash-merged to `main` automatically. Your
branch is deleted.
9. If a Full Suite test fails, the PR is ejected from the queue and Mergify posts a comment
with a link to the Buildkite build. Fix the issue, push, and comment `/merge` again.
6. The `ready` label is added, which triggers the Full Suite on your PR branch.
7. Mergify also auto-rebases your branch against `main` if it is behind and conflict-free.
8. If all Full Suite tests pass and all merge conditions are met (approval, valid title,
pre-commit green, fastcheck green, no draft, no conflicts), Mergify squash-merges to
`main` automatically. Your branch is deleted.
9. If a Full Suite test fails, Mergify removes the `ready` label and posts a comment with a
link to the Buildkite build. Fix the issue, push, and comment `/merge` again.
!!! note
Only contributors with write permission to the repository can trigger slash commands.
@@ -117,7 +116,7 @@ performance benchmarks, and API server tests.
## Running Tests On Demand
Comment on your PR to trigger specific tests without waiting for the Merge Queue.
Comment on your PR to trigger specific tests independently of the auto-merge flow.
**Trigger the entire Full Suite:**
@@ -198,10 +197,11 @@ git push --force-with-lease
Mergify removes the `needs-rebase` label automatically once conflicts are resolved.
### Merge Queue ejected my PR
### Full Suite failed after `/merge`
The Full Suite found a regression. The Mergify comment links to the Buildkite build. Check
the failing step's output for assertion errors or tracebacks.
The Full Suite found a regression. Mergify removes the `ready` label and posts a comment
linking to the Buildkite build. Check the failing step's output for assertion errors or
tracebacks.
Common causes:
@@ -210,7 +210,7 @@ Common causes:
- GPU memory issue (some tests require specific hardware like L40S or H100)
- Kernel build failure (if you changed `fastvideo-kernel/`)
After fixing, push and comment `/merge` to re-queue.
After fixing, push and comment `/merge` again.
### I'm an external contributor without write permission
+91 -116
View File
@@ -9,24 +9,22 @@ image_version = os.getenv("IMAGE_VERSION")
image_tag = f"ghcr.io/hao-ai-lab/fastvideo/fastvideo-dev:{image_version}"
print(f"Using image: {image_tag}")
image = (modal.Image.from_registry(
image_tag, add_python="3.12"
).run_commands("rm -rf /FastVideo").apt_install(
"cmake", "pkg-config", "build-essential", "curl", "libssl-dev", "ffmpeg"
).run_commands(
"curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable"
).run_commands("echo 'source ~/.cargo/env' >> ~/.bashrc").env({
"PATH":
"/root/.cargo/bin:$PATH",
"BUILDKITE_REPO":
os.environ.get("BUILDKITE_REPO", ""),
"BUILDKITE_COMMIT":
os.environ.get("BUILDKITE_COMMIT", ""),
"BUILDKITE_PULL_REQUEST":
os.environ.get("BUILDKITE_PULL_REQUEST", ""),
"IMAGE_VERSION":
os.environ.get("IMAGE_VERSION", ""),
}))
image = (
modal.Image.from_registry(image_tag, add_python="3.12")
.run_commands("rm -rf /FastVideo")
.apt_install("cmake", "pkg-config", "build-essential", "curl", "libssl-dev", "ffmpeg")
.run_commands("curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable")
.run_commands("echo 'source ~/.cargo/env' >> ~/.bashrc")
.env(
{
"PATH": "/root/.cargo/bin:$PATH",
"BUILDKITE_REPO": os.environ.get("BUILDKITE_REPO", ""),
"BUILDKITE_COMMIT": os.environ.get("BUILDKITE_COMMIT", ""),
"BUILDKITE_PULL_REQUEST": os.environ.get("BUILDKITE_PULL_REQUEST", ""),
"IMAGE_VERSION": os.environ.get("IMAGE_VERSION", ""),
}
)
)
def run_test(pytest_command: str):
@@ -41,8 +39,7 @@ def run_test(pytest_command: str):
print(f"Cloning repository: {git_repo}")
print(f"Target commit: {git_commit}")
if pr_number:
print(f"PR number: {pr_number}")
print(f"PR number: {pr_number}")
# For PRs (including forks), use GitHub's PR refs to get the correct commit
if pr_number and pr_number != "false":
@@ -66,95 +63,84 @@ def run_test(pytest_command: str):
{pytest_command}
"""
result = subprocess.run(["/bin/bash", "-c", command],
stdout=sys.stdout,
stderr=sys.stderr,
check=False)
result = subprocess.run(["/bin/bash", "-c", command], stdout=sys.stdout, stderr=sys.stderr, check=False)
sys.exit(result.returncode)
@app.function(gpu="H100:1",
image=image,
timeout=1200,
secrets=[
modal.Secret.from_dict(
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
],
volumes={"/root/data": model_vol})
@app.function(
gpu="H100:1",
image=image,
timeout=1200,
secrets=[modal.Secret.from_dict({"HF_API_KEY": os.environ.get("HF_API_KEY", "")})],
volumes={"/root/data": model_vol},
)
def run_encoder_tests():
run_test(
"export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/encoders -vs"
)
@app.function(gpu="L40S:1",
image=image,
timeout=1200,
secrets=[
modal.Secret.from_dict(
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
],
volumes={"/root/data": model_vol})
@app.function(
gpu="L40S:1",
image=image,
timeout=1200,
secrets=[modal.Secret.from_dict({"HF_API_KEY": os.environ.get("HF_API_KEY", "")})],
volumes={"/root/data": model_vol},
)
def run_vae_tests():
run_test(
"export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/vaes -vs"
)
@app.function(gpu="L40S:1",
image=image,
timeout=900,
secrets=[
modal.Secret.from_dict(
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
],
volumes={"/root/data": model_vol})
@app.function(
gpu="L40S:1",
image=image,
timeout=900,
secrets=[modal.Secret.from_dict({"HF_API_KEY": os.environ.get("HF_API_KEY", "")})],
volumes={"/root/data": model_vol},
)
def run_transformer_tests():
run_test(
"export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/transformers -vs"
)
@app.function(gpu="L40S:4",
image=image,
timeout=900,
secrets=[
modal.Secret.from_dict(
{"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})
],
volumes={"/root/data": model_vol})
@app.function(
gpu="L40S:4",
image=image,
timeout=900,
secrets=[modal.Secret.from_dict({"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})],
volumes={"/root/data": model_vol},
)
def run_training_tests():
run_test(
"export HF_HOME='/root/data/.cache' && wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/Vanilla -srP"
)
@app.function(gpu="L40S:2",
image=image,
timeout=900,
secrets=[
modal.Secret.from_dict(
{"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})
],
volumes={"/root/data": model_vol})
@app.function(
gpu="L40S:2",
image=image,
timeout=900,
secrets=[modal.Secret.from_dict({"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})],
volumes={"/root/data": model_vol},
)
def run_training_lora_tests():
run_test(
"export HF_HOME='/root/data/.cache' && wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/lora/test_lora_training.py -srP"
)
@app.function(gpu="H100:2",
image=image,
timeout=900,
secrets=[
modal.Secret.from_dict(
{"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})
])
@app.function(
gpu="H100:2",
image=image,
timeout=900,
secrets=[modal.Secret.from_dict({"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})],
)
def run_training_tests_VSA():
run_test(
"wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/VSA -srP"
)
run_test("wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/VSA -srP")
@app.function(gpu="H100:1", image=image, timeout=900)
@@ -174,33 +160,27 @@ def run_kernel_tests():
@app.function(gpu="L40S:1", image=image, timeout=900)
def run_inference_tests_vmoba():
run_test('python fastvideo/tests/inference/vmoba/test_vmoba_inference.py')
run_test("python fastvideo/tests/inference/vmoba/test_vmoba_inference.py")
@app.function(gpu="L40S:1", image=image, timeout=1200)
def run_inference_lora_tests():
run_test(
"pytest ./fastvideo/tests/inference/lora/test_lora_inference_similarity.py -vs"
)
run_test("pytest ./fastvideo/tests/inference/lora/test_lora_inference_similarity.py -vs")
@app.function(gpu="L40S:2", image=image, timeout=900)
def run_distill_dmd_tests():
run_test(
"pytest ./fastvideo/tests/training/distill/test_distill_dmd.py -vs")
run_test("pytest ./fastvideo/tests/training/distill/test_distill_dmd.py -vs")
@app.function(gpu="L40S:2",
image=image,
timeout=900,
secrets=[
modal.Secret.from_dict(
{"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})
])
@app.function(
gpu="L40S:2",
image=image,
timeout=900,
secrets=[modal.Secret.from_dict({"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})],
)
def run_self_forcing_tests():
run_test(
"wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/self-forcing/test_self_forcing.py -vs"
)
run_test("wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/self-forcing/test_self_forcing.py -vs")
@app.function(gpu="L40S:1", image=image, timeout=900)
@@ -210,41 +190,36 @@ def run_unit_test():
)
@app.function(gpu="L40S:1",
image=image,
timeout=3600,
secrets=[
modal.Secret.from_dict(
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
])
@app.function(
gpu="L40S:1",
image=image,
timeout=3600,
secrets=[modal.Secret.from_dict({"HF_API_KEY": os.environ.get("HF_API_KEY", "")})],
)
def run_lora_extraction_tests():
run_test(
"hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/lora_extraction/test_lora_extraction.py"
)
run_test("hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/lora_extraction/test_lora_extraction.py")
@app.function(gpu="L40S:2",
image=image,
timeout=1800,
secrets=[
modal.Secret.from_dict(
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
],
volumes={"/root/data": model_vol})
@app.function(
gpu="L40S:2",
image=image,
timeout=1800,
secrets=[modal.Secret.from_dict({"HF_API_KEY": os.environ.get("HF_API_KEY", "")})],
volumes={"/root/data": model_vol},
)
def run_performance_tests():
run_test(
"export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/performance -vs"
)
@app.function(gpu="L40S:1",
image=image,
timeout=1800,
secrets=[
modal.Secret.from_dict(
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
],
volumes={"/root/data": model_vol})
@app.function(
gpu="L40S:1",
image=image,
timeout=1800,
secrets=[modal.Secret.from_dict({"HF_API_KEY": os.environ.get("HF_API_KEY", "")})],
volumes={"/root/data": model_vol},
)
def run_api_server_tests():
run_test(
"export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/entrypoints/test_openai_api_integration.py -vs"