Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
15c79dda56 | ||
|
|
f7ad5ef7e0 | ||
|
|
2cb6bdd0b2 | ||
|
|
4df8a7e209 | ||
|
|
6371f19638 | ||
|
|
78173b6741 | ||
|
|
a3e1591f93 | ||
|
|
9225604322 | ||
|
|
98f628a49c | ||
|
|
cc4889de98 | ||
|
|
05eb6ccc60 | ||
|
|
4da7beccbc |
@@ -7,7 +7,7 @@ notify:
|
||||
context: "fastcheck-passed"
|
||||
- github_commit_status:
|
||||
context: "full-suite-passed"
|
||||
if: build.branch =~ /^mergify\/merge-queue\//
|
||||
if: build.env("FULL_SUITE") == "true"
|
||||
|
||||
steps:
|
||||
# ============================================================
|
||||
@@ -93,14 +93,14 @@ steps:
|
||||
queue: "default"
|
||||
|
||||
# ============================================================
|
||||
# Full Suite: Runs on Merge Queue branches or when FULL_SUITE=true
|
||||
# Triggered by Mergify Merge Queue (mergify/merge-queue/* branches)
|
||||
# Full Suite: Runs when FULL_SUITE=true
|
||||
# Triggered by adding the 'ready' label (via ci-trigger-full-suite.yml)
|
||||
# or on-demand via /test full slash command.
|
||||
# Includes integration tests, SSIM regression, training pipelines,
|
||||
# and performance benchmarks.
|
||||
# ============================================================
|
||||
- label: "Trigger Full Suite"
|
||||
if: build.env("FULL_SUITE") == "true" || build.branch =~ /^mergify\/merge-queue\//
|
||||
if: build.env("FULL_SUITE") == "true"
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: 128
|
||||
|
||||
@@ -59,7 +59,11 @@ if [ -z "${TEST_TYPE:-}" ]; then
|
||||
fi
|
||||
log "Test type: $TEST_TYPE"
|
||||
|
||||
MODAL_ENV="BUILDKITE_REPO=$BUILDKITE_REPO BUILDKITE_COMMIT=$BUILDKITE_COMMIT BUILDKITE_PULL_REQUEST=$BUILDKITE_PULL_REQUEST IMAGE_VERSION=$IMAGE_VERSION"
|
||||
EFFECTIVE_PR=${BUILDKITE_PULL_REQUEST:-false}
|
||||
if [ "$EFFECTIVE_PR" = "false" ] && [ -n "${PR_NUMBER:-}" ]; then
|
||||
EFFECTIVE_PR=$PR_NUMBER
|
||||
fi
|
||||
MODAL_ENV="BUILDKITE_REPO=$BUILDKITE_REPO BUILDKITE_COMMIT=$BUILDKITE_COMMIT BUILDKITE_PULL_REQUEST=$EFFECTIVE_PR IMAGE_VERSION=$IMAGE_VERSION"
|
||||
|
||||
case "$TEST_TYPE" in
|
||||
"encoder")
|
||||
|
||||
+33
-56
@@ -1,22 +1,3 @@
|
||||
queue_rules:
|
||||
- name: default
|
||||
queue_conditions:
|
||||
- "title~=(?i)^\\[(feat|feature|bugfix|fix|refactor|perf|ci|doc|docs|misc|chore|kernel|new.?model)\\]"
|
||||
- check-success~=pre-commit
|
||||
- check-success=fastcheck-passed
|
||||
- "#approved-reviews-by>=1"
|
||||
- -draft
|
||||
- -conflict
|
||||
- -closed
|
||||
- label!=do-not-merge
|
||||
merge_conditions:
|
||||
- check-success=full-suite-passed
|
||||
merge_method: squash
|
||||
commit_message_template: |
|
||||
{{ title }} (#{{ number }})
|
||||
|
||||
{{ body }}
|
||||
|
||||
merge_protections:
|
||||
- name: PR merge requirements
|
||||
if:
|
||||
@@ -239,32 +220,6 @@ pull_request_rules:
|
||||
|
||||
For future commits, `pre-commit` will run automatically on changed files before each commit.
|
||||
|
||||
# ============================================================
|
||||
# Buildkite CI failure notification
|
||||
# ============================================================
|
||||
|
||||
- name: comment on Buildkite CI failure
|
||||
conditions:
|
||||
- check-failure~=buildkite/ci
|
||||
- -closed
|
||||
actions:
|
||||
comment:
|
||||
message: |
|
||||
## ❌ CI tests failed
|
||||
|
||||
@{{author}} — to see what failed:
|
||||
1. Scroll to the **Checks** section below
|
||||
2. Find the check marked with ❌ (e.g. `buildkite/ci/microscope-transformer-tests`)
|
||||
3. Click **Details** to view the full build log
|
||||
|
||||
Or [view all builds for this branch on Buildkite →](https://buildkite.com/fastvideo/ci/builds?branch={{head}})
|
||||
|
||||
**Common causes:**
|
||||
- **Assertion error / test failure** — check the failing test's traceback
|
||||
- **Import error** — new dependency missing from `pyproject.toml`
|
||||
- **OOM** — some tests need specific GPUs (L40S, H100 NVL)
|
||||
|
||||
If the failure looks unrelated to your changes, comment why and a maintainer will review.
|
||||
|
||||
# ============================================================
|
||||
# Merge conflict detection
|
||||
@@ -298,20 +253,48 @@ pull_request_rules:
|
||||
label:
|
||||
remove: [needs-rebase]
|
||||
|
||||
- name: enter merge queue when ready and approved
|
||||
# ============================================================
|
||||
# Auto-merge and auto-rebase
|
||||
# ============================================================
|
||||
|
||||
- name: auto-merge when ready and all checks pass
|
||||
conditions:
|
||||
- label=ready
|
||||
- "title~=(?i)^\\[(feat|feature|bugfix|fix|refactor|perf|ci|doc|docs|misc|chore|kernel|new.?model)\\]"
|
||||
- "#approved-reviews-by>=1"
|
||||
- check-success~=pre-commit
|
||||
- check-success=fastcheck-passed
|
||||
- check-success=full-suite-passed
|
||||
- -conflict
|
||||
- -closed
|
||||
- -draft
|
||||
actions:
|
||||
queue:
|
||||
name: default
|
||||
merge:
|
||||
method: squash
|
||||
|
||||
- name: remove ready label on dequeue
|
||||
- name: auto-rebase when ready and Full Suite passed
|
||||
conditions:
|
||||
- queue-dequeue-reason!=none
|
||||
- label=ready
|
||||
- "#approved-reviews-by>=1"
|
||||
- check-success=full-suite-passed
|
||||
- -conflict
|
||||
- -closed
|
||||
- -draft
|
||||
actions:
|
||||
rebase: {}
|
||||
|
||||
- name: remove ready label on Full Suite failure
|
||||
conditions:
|
||||
- label=ready
|
||||
- check-failure=full-suite-passed
|
||||
actions:
|
||||
label:
|
||||
remove: [ready]
|
||||
|
||||
# ============================================================
|
||||
# PR title format help
|
||||
# ============================================================
|
||||
|
||||
- name: comment on invalid PR title format
|
||||
conditions:
|
||||
- -closed
|
||||
@@ -336,9 +319,3 @@ pull_request_rules:
|
||||
|
||||
Please update your PR title and the merge protection check will pass automatically.
|
||||
|
||||
- name: delete merged branch
|
||||
conditions:
|
||||
- merged
|
||||
- -head~=^(main|master|release[/-])$
|
||||
actions:
|
||||
delete_head_branch: {}
|
||||
|
||||
@@ -14,9 +14,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
if: >-
|
||||
github.event.pull_request.draft != true
|
||||
|| startsWith(github.event.pull_request.head.ref, 'mergify/merge-queue/')
|
||||
if: github.event.pull_request.draft != true
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
}
|
||||
core.setOutput('has_write', String(hasWrite));
|
||||
|
||||
- name: Queue PR via Mergify and react
|
||||
- name: Add ready label and react
|
||||
if: steps.perm.outputs.has_write == 'true'
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
with:
|
||||
@@ -39,8 +39,9 @@ jobs:
|
||||
const owner = context.repo.owner;
|
||||
const repo = context.repo.repo;
|
||||
const prNumber = context.payload.issue.number;
|
||||
// Remove ready first to allow re-trigger (labeled event fires on add, not if already present)
|
||||
try { await github.rest.issues.removeLabel({ owner, repo, issue_number: prNumber, name: 'ready' }); } catch {}
|
||||
await github.rest.issues.addLabels({ owner, repo, issue_number: prNumber, labels: ['ready'] });
|
||||
await github.rest.issues.createComment({ owner, repo, issue_number: prNumber, body: '@mergifyio queue' });
|
||||
await github.rest.reactions.createForIssueComment({
|
||||
owner, repo,
|
||||
comment_id: context.payload.comment.id,
|
||||
@@ -151,7 +152,7 @@ jobs:
|
||||
branch: "${{ needs.parse-command.outputs.pr_branch }}"
|
||||
commit: "${{ needs.parse-command.outputs.pr_sha }}"
|
||||
message: "/test ${{ needs.parse-command.outputs.test_type }} on PR #${{ github.event.issue.number }}"
|
||||
build_env_vars: '{"FULL_SUITE": "${{ needs.parse-command.outputs.full_suite }}", "TEST_TYPE": "${{ needs.parse-command.outputs.test_type }}"}'
|
||||
build_env_vars: '{"FULL_SUITE": "${{ needs.parse-command.outputs.full_suite }}", "TEST_TYPE": "${{ needs.parse-command.outputs.test_type }}", "PR_NUMBER": "${{ github.event.issue.number }}"}'
|
||||
send_pull_request: "true"
|
||||
pull_request_id: "${{ github.event.issue.number }}"
|
||||
pull_request_base_branch: "main"
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
name: Trigger Full Suite
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [labeled, synchronize]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
concurrency:
|
||||
group: full-suite-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
trigger:
|
||||
if: >-
|
||||
(github.event.action == 'labeled' && github.event.label.name == 'ready')
|
||||
|| github.event.action == 'synchronize'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check ready label
|
||||
id: check
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
with:
|
||||
script: |
|
||||
const { data: pr } = await github.rest.pulls.get({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
pull_number: context.payload.pull_request.number,
|
||||
});
|
||||
const hasReady = pr.labels.some(l => l.name === 'ready');
|
||||
core.setOutput('has_ready', String(hasReady));
|
||||
if (!hasReady) core.info('No ready label — skipping Full Suite trigger.');
|
||||
|
||||
- name: Cancel previous Buildkite builds
|
||||
if: steps.check.outputs.has_ready == 'true'
|
||||
env:
|
||||
BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
|
||||
PR_BRANCH: ${{ github.event.pull_request.head.ref }}
|
||||
run: |
|
||||
# Find running builds for this branch with FULL_SUITE=true and cancel them
|
||||
builds=$(curl -sS -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
|
||||
"https://api.buildkite.com/v2/organizations/${{ vars.BUILDKITE_ORG_SLUG }}/pipelines/${{ vars.BUILDKITE_PIPELINE_SLUG }}/builds?branch=${PR_BRANCH}&state=running,scheduled" \
|
||||
| jq -r '.[] | select(.env.FULL_SUITE == "true") | .number')
|
||||
for build_num in $builds; do
|
||||
echo "Cancelling Buildkite build #$build_num"
|
||||
curl -sS -X PUT -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
|
||||
"https://api.buildkite.com/v2/organizations/${{ vars.BUILDKITE_ORG_SLUG }}/pipelines/${{ vars.BUILDKITE_PIPELINE_SLUG }}/builds/${build_num}/cancel"
|
||||
done
|
||||
|
||||
- name: Trigger Buildkite Full Suite
|
||||
if: steps.check.outputs.has_ready == 'true'
|
||||
uses: buildkite/trigger-pipeline-action@v2.4.1
|
||||
with:
|
||||
buildkite_api_access_token: ${{ secrets.BUILDKITE_API_TOKEN }}
|
||||
pipeline: "${{ vars.BUILDKITE_ORG_SLUG }}/${{ vars.BUILDKITE_PIPELINE_SLUG }}"
|
||||
branch: "${{ github.event.pull_request.head.ref }}"
|
||||
commit: "${{ github.event.pull_request.head.sha }}"
|
||||
message: "Full Suite for PR #${{ github.event.pull_request.number }}"
|
||||
build_env_vars: '{"FULL_SUITE": "true", "PR_NUMBER": "${{ github.event.pull_request.number }}"}'
|
||||
send_pull_request: "true"
|
||||
pull_request_id: "${{ github.event.pull_request.number }}"
|
||||
pull_request_base_branch: "main"
|
||||
@@ -16,18 +16,15 @@ PR push
|
||||
Buildkite / Modal GPU instances
|
||||
Only runs tests for paths you changed
|
||||
|
||||
│ (maintainer adds 'ready' label or comments /merge)
|
||||
▼
|
||||
Merge Queue
|
||||
Mergify creates mergify/merge-queue/<branch>
|
||||
│
|
||||
│ (developer comments /merge or maintainer adds 'ready' label)
|
||||
▼
|
||||
Tier 3: Full Suite (~60-90 min)
|
||||
Buildkite / Modal GPU instances
|
||||
All integration, SSIM, training, and performance tests
|
||||
Runs on the PR branch directly
|
||||
│
|
||||
pass ──► squash-merge to main, branch deleted
|
||||
fail ──► PR ejected from queue; fix and re-queue
|
||||
pass ──► Mergify auto-squash-merges to main, branch deleted
|
||||
fail ──► Mergify removes 'ready' label; fix and /merge again
|
||||
```
|
||||
|
||||
---
|
||||
@@ -82,11 +79,11 @@ failing test's output.
|
||||
|
||||
---
|
||||
|
||||
### Tier 3: Full Test Suite (Merge Queue only)
|
||||
### Tier 3: Full Test Suite (triggered by `ready` label)
|
||||
|
||||
| Attribute | Value |
|
||||
|-----------|-------|
|
||||
| Triggered by | Mergify creating a `mergify/merge-queue/*` branch, OR a `/test full` command |
|
||||
| Triggered by | Adding the `ready` label to the PR (via `/merge` command), or a `/test full` command |
|
||||
| Runs on | Buildkite, Modal GPU instances |
|
||||
| Duration | 60-90 minutes total (tests run in parallel, path-filtered) |
|
||||
|
||||
@@ -105,44 +102,48 @@ failing test's output.
|
||||
| Performance Tests | `performance` | 30 min |
|
||||
| API Server Tests | `api_server` | 30 min |
|
||||
|
||||
A Full Suite failure means the PR is ejected from the Merge Queue. A Mergify comment will
|
||||
link to the Buildkite build. Fix the regression, push, and re-queue.
|
||||
A Full Suite failure removes the `ready` label automatically. A Mergify comment links to
|
||||
the Buildkite build. Fix the regression, push, and comment `/merge` again.
|
||||
|
||||
---
|
||||
|
||||
## Merge Queue
|
||||
## Auto-merge Flow
|
||||
|
||||
The Merge Queue prevents untested code from landing on `main`. Mergify manages the queue;
|
||||
Buildkite validates each entry.
|
||||
Mergify prevents untested code from landing on `main` by gating squash-merge on the Full
|
||||
Suite passing directly on the PR branch.
|
||||
|
||||
**How it works:**
|
||||
|
||||
1. A developer comments `/merge` on an approved PR (or a maintainer adds the `ready` label).
|
||||
2. The `ready` label triggers the Mergify rule `enter merge queue when ready and approved`.
|
||||
3. Mergify checks the **queue conditions** before accepting:
|
||||
- `pre-commit` check must be green
|
||||
2. The `ready` label triggers `ci-trigger-full-suite.yml`, which calls the Buildkite API to
|
||||
run the Full Suite on the PR branch itself.
|
||||
3. While the Full Suite runs, Mergify also auto-rebases the PR branch against `main` if it
|
||||
is behind and has no conflicts.
|
||||
4. Once the Full Suite posts `full-suite-passed`, Mergify checks all **merge conditions**:
|
||||
- `pre-commit` check is green
|
||||
- `fastcheck-passed` check is green
|
||||
- `full-suite-passed` check is green
|
||||
- At least 1 approved review (`#approved-reviews-by>=1`)
|
||||
- PR title starts with a valid `[type]` tag
|
||||
- PR is not a draft
|
||||
- No merge conflicts
|
||||
- Not closed
|
||||
- No `do-not-merge` label
|
||||
4. If conditions pass, Mergify creates a temporary branch: `mergify/merge-queue/<branch-name>`
|
||||
5. Buildkite Full Suite runs on that branch (triggered by the branch name pattern).
|
||||
6. Once all Full Suite checks pass (`check-success~=buildkite/ci`), Mergify squash-merges
|
||||
to `main` with the commit message: `<title> (#<number>)` followed by the PR body.
|
||||
7. If any Full Suite test fails, the PR is ejected from the queue and Mergify posts a comment.
|
||||
The developer fixes the issue, pushes, and comments `/merge` again.
|
||||
5. If all conditions pass, Mergify squash-merges to `main` automatically. The branch is
|
||||
deleted after merge.
|
||||
6. If the Full Suite fails, Mergify removes the `ready` label and posts a comment linking to
|
||||
the Buildkite build. The developer fixes the issue, pushes, and comments `/merge` again.
|
||||
|
||||
**Queue conditions summary:**
|
||||
**Merge conditions summary:**
|
||||
|
||||
| Condition | Meaning |
|
||||
|-----------|---------|
|
||||
| `check-success~=pre-commit` | Tier 1 pre-commit must be green |
|
||||
| `check-success=fastcheck-passed` | Tier 2 Fastcheck must be green |
|
||||
| `check-success=full-suite-passed` | Tier 3 Full Suite must be green |
|
||||
| `#approved-reviews-by>=1` | At least one approved review |
|
||||
| `-draft` | Not a draft PR |
|
||||
| `title~=(?i)^\[(feat|bugfix|...)` | PR title has a valid type tag |
|
||||
| `-draft` | PR is not in draft state |
|
||||
| `-conflict` | No merge conflicts with base branch |
|
||||
| `-closed` | PR is still open |
|
||||
| `label!=do-not-merge` | No `do-not-merge` label present |
|
||||
|
||||
---
|
||||
|
||||
@@ -186,7 +187,7 @@ Applied by Mergify based on which paths you modified. Multiple scope labels can
|
||||
|
||||
| Label | Who sets it | Meaning |
|
||||
|-------|-------------|---------|
|
||||
| `ready` | Developer (`/merge` command) or maintainer | Enters the Merge Queue |
|
||||
| `ready` | Developer (`/merge` command) or maintainer | Triggers Full Suite and enables auto-merge |
|
||||
| `needs-rebase` | Mergify (automatic) | PR has merge conflicts; rebase needed |
|
||||
| `do-not-merge` | Maintainer | Blocks queue entry regardless of other conditions |
|
||||
|
||||
@@ -245,8 +246,11 @@ The command is recognized within a few seconds. The workflow reacts with a 🚀
|
||||
/merge
|
||||
```
|
||||
|
||||
Adds the `ready` label to the PR, which triggers Merge Queue entry (subject to queue
|
||||
conditions being met).
|
||||
Adds the `ready` label to the PR, which triggers the Full Suite on your PR branch and
|
||||
enables Mergify to auto-squash-merge once all conditions pass.
|
||||
|
||||
The command first removes the `ready` label if it is already present, then re-adds it. This
|
||||
ensures the `labeled` event fires and a fresh Full Suite build is started even on a re-try.
|
||||
|
||||
### `/test <name>`
|
||||
|
||||
@@ -290,6 +294,7 @@ Protected branches (`main`, `master`, `release/*`) are never deleted.
|
||||
| Filename | Trigger | What it does |
|
||||
|----------|---------|-------------|
|
||||
| `ci-precommit.yml` | Every push / PR against `main` | Runs pre-commit hooks (yapf, ruff, mypy, codespell, pymarkdown, actionlint, check-filenames) |
|
||||
| `ci-trigger-full-suite.yml` | `ready` label added to a PR | Calls Buildkite API to run Full Suite on the PR branch |
|
||||
| `ci-slash-commands.yml` | PR comment starting with `/merge` or `/test` | Handles slash commands; adds `ready` label or triggers Buildkite |
|
||||
| `community-issue-labeler.yml` | Issue opened or edited | Auto-labels issues by keyword matching against title and body |
|
||||
| `community-welcome.yml` | First contribution | Posts a welcome comment for first-time contributors |
|
||||
|
||||
@@ -62,7 +62,7 @@ need to set them manually.
|
||||
|
||||
| Label | Who sets it | Meaning |
|
||||
|-------|-------------|---------|
|
||||
| `ready` | You (`/merge` comment) or a maintainer | Requests entry into the Merge Queue |
|
||||
| `ready` | You (`/merge` comment) or a maintainer | Triggers Full Suite and enables auto-merge |
|
||||
| `needs-rebase` | Mergify (automatic) | Your PR has conflicts; rebase against `main` |
|
||||
| `do-not-merge` | Maintainer | Blocks merge regardless of CI status |
|
||||
|
||||
@@ -82,11 +82,11 @@ hooks: yapf, ruff, mypy, codespell, pymarkdown, actionlint, and check-filenames.
|
||||
Buildkite runs GPU tests only for the components you changed. If you only modified
|
||||
`fastvideo/models/vaes/`, only VAE Tests run. Tests run in parallel.
|
||||
|
||||
**Tier 3: Full Suite (~60-90 min) — runs only in the Merge Queue**
|
||||
**Tier 3: Full Suite (~60-90 min) — triggered by the `ready` label**
|
||||
|
||||
When your PR enters the Merge Queue, Buildkite runs the complete test suite: SSIM
|
||||
regression, LoRA inference and training, distillation, self-forcing, VSA, VMoBA,
|
||||
performance benchmarks, and API server tests.
|
||||
When you comment `/merge` (or a maintainer adds the `ready` label), Buildkite runs the
|
||||
complete test suite on your PR branch: SSIM regression, LoRA inference and training,
|
||||
distillation, self-forcing, VSA, VMoBA, performance benchmarks, and API server tests.
|
||||
|
||||
---
|
||||
|
||||
@@ -99,14 +99,13 @@ performance benchmarks, and API server tests.
|
||||
3. Fix any pre-commit failures locally (`pre-commit run --all-files`) and push again.
|
||||
4. Wait for at least one approving review.
|
||||
5. Once approved and pre-commit is green, comment `/merge` on the PR.
|
||||
6. Mergify adds the `ready` label and checks queue conditions (approval, no draft, no
|
||||
conflicts, no `do-not-merge`).
|
||||
7. If conditions pass, Mergify creates a `mergify/merge-queue/<branch>` branch and runs
|
||||
the Full Suite.
|
||||
8. If all Full Suite tests pass, your PR is squash-merged to `main` automatically. Your
|
||||
branch is deleted.
|
||||
9. If a Full Suite test fails, the PR is ejected from the queue and Mergify posts a comment
|
||||
with a link to the Buildkite build. Fix the issue, push, and comment `/merge` again.
|
||||
6. The `ready` label is added, which triggers the Full Suite on your PR branch.
|
||||
7. Mergify also auto-rebases your branch against `main` if it is behind and conflict-free.
|
||||
8. If all Full Suite tests pass and all merge conditions are met (approval, valid title,
|
||||
pre-commit green, fastcheck green, no draft, no conflicts), Mergify squash-merges to
|
||||
`main` automatically. Your branch is deleted.
|
||||
9. If a Full Suite test fails, Mergify removes the `ready` label and posts a comment with a
|
||||
link to the Buildkite build. Fix the issue, push, and comment `/merge` again.
|
||||
|
||||
!!! note
|
||||
Only contributors with write permission to the repository can trigger slash commands.
|
||||
@@ -117,7 +116,7 @@ performance benchmarks, and API server tests.
|
||||
|
||||
## Running Tests On Demand
|
||||
|
||||
Comment on your PR to trigger specific tests without waiting for the Merge Queue.
|
||||
Comment on your PR to trigger specific tests independently of the auto-merge flow.
|
||||
|
||||
**Trigger the entire Full Suite:**
|
||||
|
||||
@@ -198,10 +197,11 @@ git push --force-with-lease
|
||||
|
||||
Mergify removes the `needs-rebase` label automatically once conflicts are resolved.
|
||||
|
||||
### Merge Queue ejected my PR
|
||||
### Full Suite failed after `/merge`
|
||||
|
||||
The Full Suite found a regression. The Mergify comment links to the Buildkite build. Check
|
||||
the failing step's output for assertion errors or tracebacks.
|
||||
The Full Suite found a regression. Mergify removes the `ready` label and posts a comment
|
||||
linking to the Buildkite build. Check the failing step's output for assertion errors or
|
||||
tracebacks.
|
||||
|
||||
Common causes:
|
||||
|
||||
@@ -210,7 +210,7 @@ Common causes:
|
||||
- GPU memory issue (some tests require specific hardware like L40S or H100)
|
||||
- Kernel build failure (if you changed `fastvideo-kernel/`)
|
||||
|
||||
After fixing, push and comment `/merge` to re-queue.
|
||||
After fixing, push and comment `/merge` again.
|
||||
|
||||
### I'm an external contributor without write permission
|
||||
|
||||
|
||||
@@ -9,24 +9,22 @@ image_version = os.getenv("IMAGE_VERSION")
|
||||
image_tag = f"ghcr.io/hao-ai-lab/fastvideo/fastvideo-dev:{image_version}"
|
||||
print(f"Using image: {image_tag}")
|
||||
|
||||
image = (modal.Image.from_registry(
|
||||
image_tag, add_python="3.12"
|
||||
).run_commands("rm -rf /FastVideo").apt_install(
|
||||
"cmake", "pkg-config", "build-essential", "curl", "libssl-dev", "ffmpeg"
|
||||
).run_commands(
|
||||
"curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable"
|
||||
).run_commands("echo 'source ~/.cargo/env' >> ~/.bashrc").env({
|
||||
"PATH":
|
||||
"/root/.cargo/bin:$PATH",
|
||||
"BUILDKITE_REPO":
|
||||
os.environ.get("BUILDKITE_REPO", ""),
|
||||
"BUILDKITE_COMMIT":
|
||||
os.environ.get("BUILDKITE_COMMIT", ""),
|
||||
"BUILDKITE_PULL_REQUEST":
|
||||
os.environ.get("BUILDKITE_PULL_REQUEST", ""),
|
||||
"IMAGE_VERSION":
|
||||
os.environ.get("IMAGE_VERSION", ""),
|
||||
}))
|
||||
image = (
|
||||
modal.Image.from_registry(image_tag, add_python="3.12")
|
||||
.run_commands("rm -rf /FastVideo")
|
||||
.apt_install("cmake", "pkg-config", "build-essential", "curl", "libssl-dev", "ffmpeg")
|
||||
.run_commands("curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable")
|
||||
.run_commands("echo 'source ~/.cargo/env' >> ~/.bashrc")
|
||||
.env(
|
||||
{
|
||||
"PATH": "/root/.cargo/bin:$PATH",
|
||||
"BUILDKITE_REPO": os.environ.get("BUILDKITE_REPO", ""),
|
||||
"BUILDKITE_COMMIT": os.environ.get("BUILDKITE_COMMIT", ""),
|
||||
"BUILDKITE_PULL_REQUEST": os.environ.get("BUILDKITE_PULL_REQUEST", ""),
|
||||
"IMAGE_VERSION": os.environ.get("IMAGE_VERSION", ""),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def run_test(pytest_command: str):
|
||||
@@ -41,8 +39,7 @@ def run_test(pytest_command: str):
|
||||
|
||||
print(f"Cloning repository: {git_repo}")
|
||||
print(f"Target commit: {git_commit}")
|
||||
if pr_number:
|
||||
print(f"PR number: {pr_number}")
|
||||
print(f"PR number: {pr_number}")
|
||||
|
||||
# For PRs (including forks), use GitHub's PR refs to get the correct commit
|
||||
if pr_number and pr_number != "false":
|
||||
@@ -66,95 +63,84 @@ def run_test(pytest_command: str):
|
||||
{pytest_command}
|
||||
"""
|
||||
|
||||
result = subprocess.run(["/bin/bash", "-c", command],
|
||||
stdout=sys.stdout,
|
||||
stderr=sys.stderr,
|
||||
check=False)
|
||||
result = subprocess.run(["/bin/bash", "-c", command], stdout=sys.stdout, stderr=sys.stderr, check=False)
|
||||
|
||||
sys.exit(result.returncode)
|
||||
|
||||
|
||||
@app.function(gpu="H100:1",
|
||||
image=image,
|
||||
timeout=1200,
|
||||
secrets=[
|
||||
modal.Secret.from_dict(
|
||||
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
|
||||
],
|
||||
volumes={"/root/data": model_vol})
|
||||
@app.function(
|
||||
gpu="H100:1",
|
||||
image=image,
|
||||
timeout=1200,
|
||||
secrets=[modal.Secret.from_dict({"HF_API_KEY": os.environ.get("HF_API_KEY", "")})],
|
||||
volumes={"/root/data": model_vol},
|
||||
)
|
||||
def run_encoder_tests():
|
||||
run_test(
|
||||
"export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/encoders -vs"
|
||||
)
|
||||
|
||||
|
||||
@app.function(gpu="L40S:1",
|
||||
image=image,
|
||||
timeout=1200,
|
||||
secrets=[
|
||||
modal.Secret.from_dict(
|
||||
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
|
||||
],
|
||||
volumes={"/root/data": model_vol})
|
||||
@app.function(
|
||||
gpu="L40S:1",
|
||||
image=image,
|
||||
timeout=1200,
|
||||
secrets=[modal.Secret.from_dict({"HF_API_KEY": os.environ.get("HF_API_KEY", "")})],
|
||||
volumes={"/root/data": model_vol},
|
||||
)
|
||||
def run_vae_tests():
|
||||
run_test(
|
||||
"export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/vaes -vs"
|
||||
)
|
||||
|
||||
|
||||
@app.function(gpu="L40S:1",
|
||||
image=image,
|
||||
timeout=900,
|
||||
secrets=[
|
||||
modal.Secret.from_dict(
|
||||
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
|
||||
],
|
||||
volumes={"/root/data": model_vol})
|
||||
@app.function(
|
||||
gpu="L40S:1",
|
||||
image=image,
|
||||
timeout=900,
|
||||
secrets=[modal.Secret.from_dict({"HF_API_KEY": os.environ.get("HF_API_KEY", "")})],
|
||||
volumes={"/root/data": model_vol},
|
||||
)
|
||||
def run_transformer_tests():
|
||||
run_test(
|
||||
"export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/transformers -vs"
|
||||
)
|
||||
|
||||
|
||||
@app.function(gpu="L40S:4",
|
||||
image=image,
|
||||
timeout=900,
|
||||
secrets=[
|
||||
modal.Secret.from_dict(
|
||||
{"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})
|
||||
],
|
||||
volumes={"/root/data": model_vol})
|
||||
@app.function(
|
||||
gpu="L40S:4",
|
||||
image=image,
|
||||
timeout=900,
|
||||
secrets=[modal.Secret.from_dict({"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})],
|
||||
volumes={"/root/data": model_vol},
|
||||
)
|
||||
def run_training_tests():
|
||||
run_test(
|
||||
"export HF_HOME='/root/data/.cache' && wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/Vanilla -srP"
|
||||
)
|
||||
|
||||
|
||||
@app.function(gpu="L40S:2",
|
||||
image=image,
|
||||
timeout=900,
|
||||
secrets=[
|
||||
modal.Secret.from_dict(
|
||||
{"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})
|
||||
],
|
||||
volumes={"/root/data": model_vol})
|
||||
@app.function(
|
||||
gpu="L40S:2",
|
||||
image=image,
|
||||
timeout=900,
|
||||
secrets=[modal.Secret.from_dict({"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})],
|
||||
volumes={"/root/data": model_vol},
|
||||
)
|
||||
def run_training_lora_tests():
|
||||
run_test(
|
||||
"export HF_HOME='/root/data/.cache' && wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/lora/test_lora_training.py -srP"
|
||||
)
|
||||
|
||||
|
||||
@app.function(gpu="H100:2",
|
||||
image=image,
|
||||
timeout=900,
|
||||
secrets=[
|
||||
modal.Secret.from_dict(
|
||||
{"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})
|
||||
])
|
||||
@app.function(
|
||||
gpu="H100:2",
|
||||
image=image,
|
||||
timeout=900,
|
||||
secrets=[modal.Secret.from_dict({"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})],
|
||||
)
|
||||
def run_training_tests_VSA():
|
||||
run_test(
|
||||
"wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/VSA -srP"
|
||||
)
|
||||
run_test("wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/VSA -srP")
|
||||
|
||||
|
||||
@app.function(gpu="H100:1", image=image, timeout=900)
|
||||
@@ -174,33 +160,27 @@ def run_kernel_tests():
|
||||
|
||||
@app.function(gpu="L40S:1", image=image, timeout=900)
|
||||
def run_inference_tests_vmoba():
|
||||
run_test('python fastvideo/tests/inference/vmoba/test_vmoba_inference.py')
|
||||
run_test("python fastvideo/tests/inference/vmoba/test_vmoba_inference.py")
|
||||
|
||||
|
||||
@app.function(gpu="L40S:1", image=image, timeout=1200)
|
||||
def run_inference_lora_tests():
|
||||
run_test(
|
||||
"pytest ./fastvideo/tests/inference/lora/test_lora_inference_similarity.py -vs"
|
||||
)
|
||||
run_test("pytest ./fastvideo/tests/inference/lora/test_lora_inference_similarity.py -vs")
|
||||
|
||||
|
||||
@app.function(gpu="L40S:2", image=image, timeout=900)
|
||||
def run_distill_dmd_tests():
|
||||
run_test(
|
||||
"pytest ./fastvideo/tests/training/distill/test_distill_dmd.py -vs")
|
||||
run_test("pytest ./fastvideo/tests/training/distill/test_distill_dmd.py -vs")
|
||||
|
||||
|
||||
@app.function(gpu="L40S:2",
|
||||
image=image,
|
||||
timeout=900,
|
||||
secrets=[
|
||||
modal.Secret.from_dict(
|
||||
{"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})
|
||||
])
|
||||
@app.function(
|
||||
gpu="L40S:2",
|
||||
image=image,
|
||||
timeout=900,
|
||||
secrets=[modal.Secret.from_dict({"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})],
|
||||
)
|
||||
def run_self_forcing_tests():
|
||||
run_test(
|
||||
"wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/self-forcing/test_self_forcing.py -vs"
|
||||
)
|
||||
run_test("wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/self-forcing/test_self_forcing.py -vs")
|
||||
|
||||
|
||||
@app.function(gpu="L40S:1", image=image, timeout=900)
|
||||
@@ -210,41 +190,36 @@ def run_unit_test():
|
||||
)
|
||||
|
||||
|
||||
@app.function(gpu="L40S:1",
|
||||
image=image,
|
||||
timeout=3600,
|
||||
secrets=[
|
||||
modal.Secret.from_dict(
|
||||
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
|
||||
])
|
||||
@app.function(
|
||||
gpu="L40S:1",
|
||||
image=image,
|
||||
timeout=3600,
|
||||
secrets=[modal.Secret.from_dict({"HF_API_KEY": os.environ.get("HF_API_KEY", "")})],
|
||||
)
|
||||
def run_lora_extraction_tests():
|
||||
run_test(
|
||||
"hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/lora_extraction/test_lora_extraction.py"
|
||||
)
|
||||
run_test("hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/lora_extraction/test_lora_extraction.py")
|
||||
|
||||
|
||||
@app.function(gpu="L40S:2",
|
||||
image=image,
|
||||
timeout=1800,
|
||||
secrets=[
|
||||
modal.Secret.from_dict(
|
||||
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
|
||||
],
|
||||
volumes={"/root/data": model_vol})
|
||||
@app.function(
|
||||
gpu="L40S:2",
|
||||
image=image,
|
||||
timeout=1800,
|
||||
secrets=[modal.Secret.from_dict({"HF_API_KEY": os.environ.get("HF_API_KEY", "")})],
|
||||
volumes={"/root/data": model_vol},
|
||||
)
|
||||
def run_performance_tests():
|
||||
run_test(
|
||||
"export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/performance -vs"
|
||||
)
|
||||
|
||||
|
||||
@app.function(gpu="L40S:1",
|
||||
image=image,
|
||||
timeout=1800,
|
||||
secrets=[
|
||||
modal.Secret.from_dict(
|
||||
{"HF_API_KEY": os.environ.get("HF_API_KEY", "")})
|
||||
],
|
||||
volumes={"/root/data": model_vol})
|
||||
@app.function(
|
||||
gpu="L40S:1",
|
||||
image=image,
|
||||
timeout=1800,
|
||||
secrets=[modal.Secret.from_dict({"HF_API_KEY": os.environ.get("HF_API_KEY", "")})],
|
||||
volumes={"/root/data": model_vol},
|
||||
)
|
||||
def run_api_server_tests():
|
||||
run_test(
|
||||
"export HF_HOME='/root/data/.cache' && hf auth login --token $HF_API_KEY && pytest ./fastvideo/tests/entrypoints/test_openai_api_integration.py -vs"
|
||||
|
||||
Reference in New Issue
Block a user