nodes/hao-ai-lab-FastVideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[ci] Add TEST_SCOPE routing for clean single-test execution (#1203)

Browse Source
This commit is contained in:
Jinzhe Pan
2026-03-31 19:40:59 +08:00
committed by GitHub
parent ff22439f28
commit 81fd80c8ee
6 changed files with 170 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.buildkite/pipeline.yml
+18 -3
View File
@@ -5,17 +5,32 @@ env:
notify:
- github_commit_status:
context: "fastcheck-passed"
if: build.env("TEST_SCOPE") == "fastcheck" || build.env("TEST_SCOPE") == null
- github_commit_status:
context: "full-suite-passed"
if: build.env("FULL_SUITE") == "true"
if: build.env("TEST_SCOPE") == "full"
steps:
# ============================================================
- label: ":dart: Direct Test (${TEST_TYPE})"
if: build.env("TEST_SCOPE") == "direct"
command: "timeout 90m .buildkite/scripts/pr_test.sh"
retry:
automatic:
- exit_status: 128
limit: 3
- exit_status: -1
limit: 2
agents:
queue: "default"
# ============================================================
# Fastcheck: Runs on every PR (~10-15 min parallel)
# Core component validation: encoders, VAEs, transformers,
# CUDA kernels, and unit tests.
# ============================================================
- label: "Trigger Fastcheck"
if: build.env("TEST_SCOPE") == "fastcheck" || build.env("TEST_SCOPE") == null
retry:
automatic:
- exit_status: 128
@@ -93,14 +108,14 @@ steps:
queue: "default"
# ============================================================
# Full Suite: Runs when FULL_SUITE=true
# Full Suite: Runs when TEST_SCOPE=full
# Triggered by adding the 'ready' label (via ci-trigger-full-suite.yml)
# or on-demand via /test full slash command.
# Includes integration tests, SSIM regression, training pipelines,
# and performance benchmarks.
# ============================================================
- label: "Trigger Full Suite"
if: build.env("FULL_SUITE") == "true"
if: build.env("TEST_SCOPE") == "full"
retry:
automatic:
- exit_status: 128
.github/workflows/ci-slash-commands.yml
+54 -17
View File
@@ -54,6 +54,7 @@ jobs:
runs-on: ubuntu-latest
outputs:
test_type: ${{ steps.parse.outputs.test_type }}
test_scope: ${{ steps.parse.outputs.test_scope }}
full_suite: ${{ steps.parse.outputs.full_suite }}
pr_sha: ${{ steps.pr.outputs.sha }}
pr_branch: ${{ steps.pr.outputs.branch }}
@@ -102,14 +103,23 @@ jobs:
)
if [ "$TEST_NAME" = "full" ]; then
echo "test_type=all" >> "$GITHUB_OUTPUT"
echo "full_suite=true" >> "$GITHUB_OUTPUT"
{
echo "test_type=all"
echo "test_scope=full"
echo "full_suite=true"
} >> "$GITHUB_OUTPUT"
elif [ "$TEST_NAME" = "fastcheck" ]; then
echo "test_type=fastcheck" >> "$GITHUB_OUTPUT"
echo "full_suite=false" >> "$GITHUB_OUTPUT"
{
echo "test_type=fastcheck"
echo "test_scope=fastcheck"
echo "full_suite=false"
} >> "$GITHUB_OUTPUT"
else
echo "test_type=${MAP[$TEST_NAME]}" >> "$GITHUB_OUTPUT"
echo "full_suite=true" >> "$GITHUB_OUTPUT"
{
echo "test_type=${MAP[$TEST_NAME]}"
echo "test_scope=direct"
echo "full_suite=false"
} >> "$GITHUB_OUTPUT"
fi
- name: Get PR details
@@ -145,14 +155,41 @@ jobs:
});
- name: Trigger Buildkite
uses: buildkite/trigger-pipeline-action@v2.4.1
with:
buildkite_api_access_token: ${{ secrets.BUILDKITE_API_TOKEN }}
pipeline: "${{ vars.BUILDKITE_ORG_SLUG }}/${{ vars.BUILDKITE_PIPELINE_SLUG }}"
branch: "${{ needs.parse-command.outputs.pr_branch }}"
commit: "${{ needs.parse-command.outputs.pr_sha }}"
message: "/test ${{ needs.parse-command.outputs.test_type }} on PR #${{ github.event.issue.number }}"
build_env_vars: '{"FULL_SUITE": "${{ needs.parse-command.outputs.full_suite }}", "TEST_TYPE": "${{ needs.parse-command.outputs.test_type }}", "PR_NUMBER": "${{ github.event.issue.number }}"}'
send_pull_request: "true"
pull_request_id: "${{ github.event.issue.number }}"
pull_request_base_branch: "main"
env:
BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
PR_SHA: ${{ needs.parse-command.outputs.pr_sha }}
PR_BRANCH: ${{ needs.parse-command.outputs.pr_branch }}
PR_NUMBER: ${{ github.event.issue.number }}
TEST_SCOPE: ${{ needs.parse-command.outputs.test_scope }}
FULL_SUITE: ${{ needs.parse-command.outputs.full_suite }}
TEST_TYPE: ${{ needs.parse-command.outputs.test_type }}
BK_ORG: ${{ vars.BUILDKITE_ORG_SLUG }}
BK_PIPELINE: ${{ vars.BUILDKITE_PIPELINE_SLUG }}
run: |
curl -sS --fail-with-body -X POST \
"https://api.buildkite.com/v2/organizations/${BK_ORG}/pipelines/${BK_PIPELINE}/builds" \
-H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
-H "Content-Type: application/json" \
--data-raw "$(jq -n \
--arg commit "$PR_SHA" \
--arg branch "$PR_BRANCH" \
--arg message "/test ${TEST_TYPE} on PR #${PR_NUMBER}" \
--argjson pr_id "$PR_NUMBER" \
--arg test_scope "$TEST_SCOPE" \
--arg full_suite "$FULL_SUITE" \
--arg test_type "$TEST_TYPE" \
--arg pr_number "$PR_NUMBER" \
'{
commit: $commit,
branch: $branch,
message: $message,
ignore_pipeline_branch_filters: true,
pull_request_id: $pr_id,
pull_request_base_branch: "main",
env: {
TEST_SCOPE: $test_scope,
FULL_SUITE: $full_suite,
TEST_TYPE: $test_type,
PR_NUMBER: $pr_number
}
}')"
.github/workflows/ci-trigger-full-suite.yml
+32 -13
View File
@@ -39,10 +39,10 @@ jobs:
BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
PR_BRANCH: ${{ github.event.pull_request.head.ref }}
run: |
# Find running builds for this branch with FULL_SUITE=true and cancel them
# Find running builds for this branch with TEST_SCOPE=full and cancel them
builds=$(curl -sS -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
"https://api.buildkite.com/v2/organizations/${{ vars.BUILDKITE_ORG_SLUG }}/pipelines/${{ vars.BUILDKITE_PIPELINE_SLUG }}/builds?branch=${PR_BRANCH}&state=running,scheduled" \
| jq -r '.[] | select(.env.FULL_SUITE == "true") | .number')
| jq -r '.[] | select(.env.TEST_SCOPE == "full") | .number')
for build_num in $builds; do
echo "Cancelling Buildkite build #$build_num"
curl -sS -X PUT -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
@@ -51,14 +51,33 @@ jobs:
- name: Trigger Buildkite Full Suite
if: steps.check.outputs.has_ready == 'true'
uses: buildkite/trigger-pipeline-action@v2.4.1
with:
buildkite_api_access_token: ${{ secrets.BUILDKITE_API_TOKEN }}
pipeline: "${{ vars.BUILDKITE_ORG_SLUG }}/${{ vars.BUILDKITE_PIPELINE_SLUG }}"
branch: "${{ github.event.pull_request.head.ref }}"
commit: "${{ github.event.pull_request.head.sha }}"
message: "Full Suite for PR #${{ github.event.pull_request.number }}"
build_env_vars: '{"FULL_SUITE": "true", "PR_NUMBER": "${{ github.event.pull_request.number }}"}'
send_pull_request: "true"
pull_request_id: "${{ github.event.pull_request.number }}"
pull_request_base_branch: "main"
env:
BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
PR_SHA: ${{ github.event.pull_request.head.sha }}
PR_BRANCH: ${{ github.event.pull_request.head.ref }}
PR_NUMBER: ${{ github.event.pull_request.number }}
BK_ORG: ${{ vars.BUILDKITE_ORG_SLUG }}
BK_PIPELINE: ${{ vars.BUILDKITE_PIPELINE_SLUG }}
run: |
curl -sS --fail-with-body -X POST \
"https://api.buildkite.com/v2/organizations/${BK_ORG}/pipelines/${BK_PIPELINE}/builds" \
-H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
-H "Content-Type: application/json" \
--data-raw "$(jq -n \
--arg commit "$PR_SHA" \
--arg branch "$PR_BRANCH" \
--arg message "Full Suite for PR #${PR_NUMBER}" \
--argjson pr_id "$PR_NUMBER" \
'{
commit: $commit,
branch: $branch,
message: $message,
ignore_pipeline_branch_filters: true,
pull_request_id: $pr_id,
pull_request_base_branch: "main",
env: {
TEST_SCOPE: "full",
FULL_SUITE: "true",
PR_NUMBER: ($pr_id | tostring)
}
}')"
docs/contributing/ci_architecture.md
+2 -2
View File
@@ -277,8 +277,8 @@ Triggers a specific Buildkite test or suite on the current PR branch.
| `/test vmoba` | VMoBA inference tests | `inference_vmoba` |
| `/test performance` | Performance benchmarks | `performance` |
| `/test api` | API server integration tests | `api_server` |
| `/test full` | Entire Full Suite | all (with `FULL_SUITE=true`) |
| `/test fastcheck` | Entire Fastcheck suite | fastcheck (with `FULL_SUITE=false`) |
| `/test full` | Entire Full Suite | all (with `TEST_SCOPE=full`) |
| `/test fastcheck` | Entire Fastcheck suite | fastcheck (with `TEST_SCOPE=fastcheck`) |
---
fastvideo/tests/ssim/inference_similarity_utils.py
+7 -2
View File
@@ -79,8 +79,13 @@ def _assert_similarity(
attention_backend_name: str,
) -> None:
if not os.path.exists(reference_folder):
logger.error("Reference folder missing")
raise FileNotFoundError(f"Reference video folder does not exist: {reference_folder}")
logger.error("Reference folder missing: %s", reference_folder)
error_msg = (
f"Reference video folder does not exist: {reference_folder}\n"
f"To download reference videos, run:\n"
f" python fastvideo/tests/ssim/reference_videos_cli.py download"
)
raise FileNotFoundError(error_msg)
try:
reference_video_path = _find_reference_video(reference_folder, prompt)
fastvideo/tests/ssim/reference_videos_cli.py
+57 -88
View File
@@ -51,11 +51,7 @@ def _iter_video_files(root: Path) -> Iterable[Path]:
def _discover_reference_dirs(base_dir: Path) -> list[Path]:
if not base_dir.exists():
return []
dirs = [
p
for p in base_dir.iterdir()
if p.is_dir() and p.name.endswith("_reference_videos")
]
dirs = [p for p in base_dir.iterdir() if p.is_dir() and p.name.endswith("_reference_videos")]
return sorted(dirs)
@@ -106,10 +102,7 @@ def _resolve_device_reference_folder(
) -> str:
if explicit_device_folder and explicit_device_folder.strip():
return explicit_device_folder.strip()
if (
reference_dir is not None
and reference_dir.name.endswith("_reference_videos")
):
if reference_dir is not None and reference_dir.name.endswith("_reference_videos"):
return reference_dir.name
return DEFAULT_DEVICE_REFERENCE_FOLDER
@@ -141,6 +134,11 @@ def _discover_reference_dirs_for_tier(
def _has_local_reference_videos(base_dir: Path, quality_tier: str) -> bool:
# Check for completion marker file first
marker_path = base_dir / REFERENCE_VIDEOS_DIRNAME / quality_tier / f".download_complete_{quality_tier}"
if not marker_path.exists():
return False
# Also verify at least one .mp4 exists
for ref_dir in _discover_reference_dirs_for_tier(base_dir, quality_tier):
for _ in _iter_video_files(ref_dir):
return True
@@ -161,8 +159,7 @@ def _load_hf_sdk():
from huggingface_hub import HfApi, snapshot_download
except ImportError as exc:
raise RuntimeError(
"huggingface_hub is required for download/upload.\n"
"Install with: pip install huggingface_hub"
"huggingface_hub is required for download/upload.\nInstall with: pip install huggingface_hub"
) from exc
return HfApi, snapshot_download
@@ -210,9 +207,7 @@ def download_reference_videos(
for quality_tier in tiers:
tier_prefix = f"{REFERENCE_VIDEOS_DIRNAME}/{quality_tier}"
if device_folders:
allow_patterns.extend(
f"{tier_prefix}/{folder}/**" for folder in device_folders
)
allow_patterns.extend(f"{tier_prefix}/{folder}/**" for folder in device_folders)
else:
allow_patterns.append(f"{tier_prefix}/*_reference_videos/**")
@@ -242,9 +237,12 @@ def download_reference_videos(
)
if copied_roots == 0:
raise RuntimeError(
"HF download completed but no reference_videos content "
"was found in downloaded artifacts."
"HF download completed but no reference_videos content was found in downloaded artifacts."
)
for quality_tier in tiers:
marker_path = local_dir / REFERENCE_VIDEOS_DIRNAME / quality_tier / f".download_complete_{quality_tier}"
marker_path.parent.mkdir(parents=True, exist_ok=True)
marker_path.touch()
def upload_reference_videos(
@@ -266,12 +264,8 @@ def upload_reference_videos(
for quality_tier, reference_dir in reference_dirs_by_tier:
if not reference_dir.exists():
raise FileNotFoundError(
f"Reference directory not found: {reference_dir}"
)
path_in_repo = (
f"{REFERENCE_VIDEOS_DIRNAME}/{quality_tier}/{reference_dir.name}"
)
raise FileNotFoundError(f"Reference directory not found: {reference_dir}")
path_in_repo = f"{REFERENCE_VIDEOS_DIRNAME}/{quality_tier}/{reference_dir.name}"
print(f"Uploading {reference_dir.name} ({quality_tier}) to {repo_id} ...")
api.upload_folder(
repo_id=repo_id,
@@ -294,20 +288,14 @@ def _resolve_upload_reference_dirs(
if explicit_reference_dirs:
if len(tiers) != 1:
raise RuntimeError(
"--reference-dir requires a single --quality-tier "
"(default or full_quality)."
)
raise RuntimeError("--reference-dir requires a single --quality-tier (default or full_quality).")
resolved_tier = tiers[0]
return [(resolved_tier, Path(p).resolve()) for p in explicit_reference_dirs]
resolved: list[tuple[str, Path]] = []
for tier in tiers:
for reference_dir in _discover_reference_dirs_for_tier(base_dir, tier):
if (
selected_device_folders
and reference_dir.name not in selected_device_folders
):
if selected_device_folders and reference_dir.name not in selected_device_folders:
continue
resolved.append((tier, reference_dir.resolve()))
return resolved
@@ -327,32 +315,38 @@ def ensure_reference_videos_available(
lock_path = target_dir / ".reference_videos_download.lock"
with _exclusive_download_lock(lock_path):
if _has_local_reference_videos(target_dir, quality_tier):
print(f"Reference videos ({quality_tier}) already available at {target_dir}")
return False
resolved_repo_id = repo_id or _default_repo_id()
resolved_repo_type = repo_type or _default_repo_type()
if not resolved_repo_id:
raise RuntimeError(
"No local reference videos found and no HF repo configured.\n"
f"Set {HF_REPO_ENV_KEY} or pass --repo-id."
f"No local reference videos found and no HF repo configured.\nSet {HF_REPO_ENV_KEY} or pass --repo-id."
)
print(
f"No local {quality_tier} reference videos found under {target_dir}. "
f"Downloading from HF repo {resolved_repo_id} ..."
)
download_reference_videos(
repo_id=resolved_repo_id,
repo_type=resolved_repo_type,
local_dir=target_dir,
quality_tiers=[quality_tier],
)
print(f"Repo ID: {resolved_repo_id}")
print(f"Quality tier: {quality_tier}")
print(f"No local {quality_tier} reference videos found under {target_dir}. Starting download...")
try:
download_reference_videos(
repo_id=resolved_repo_id,
repo_type=resolved_repo_type,
local_dir=target_dir,
quality_tiers=[quality_tier],
)
print(f"Download completed for {quality_tier} reference videos.")
except Exception as exc:
print(f"ERROR: Failed to download {quality_tier} reference videos from {resolved_repo_id}.")
print(
f"Suggested command to retry: "
f"python fastvideo/tests/ssim/reference_videos_cli.py download "
f"--quality-tier {quality_tier}"
)
raise
if not _has_local_reference_videos(target_dir, quality_tier):
raise RuntimeError(
f"HF download completed but no {quality_tier} *_reference_videos "
"content found."
)
raise RuntimeError(f"HF download completed but no {quality_tier} *_reference_videos content found.")
return True
@@ -397,18 +391,12 @@ def _build_parser() -> argparse.ArgumentParser:
"--generated-dir",
type=Path,
default=None,
help=(
"Source generated directory. "
"Default: <ssim_dir>/generated_videos/<quality-tier>/<device-folder>"
),
help=("Source generated directory. Default: <ssim_dir>/generated_videos/<quality-tier>/<device-folder>"),
)
copy_parser.add_argument(
"--device-folder",
default=None,
help=(
"GPU folder name (e.g., H200_reference_videos) used to build "
"default generated/reference paths."
),
help=("GPU folder name (e.g., H200_reference_videos) used to build default generated/reference paths."),
)
copy_parser.add_argument(
"--reference-dir",
@@ -500,19 +488,13 @@ def _build_parser() -> argparse.ArgumentParser:
"--device-folder",
action="append",
default=[],
help=(
"Specific GPU reference folder to upload (repeatable), "
"e.g. H200_reference_videos."
),
help=("Specific GPU reference folder to upload (repeatable), e.g. H200_reference_videos."),
)
upload_parser.add_argument(
"--base-dir",
type=Path,
default=_ssim_dir(),
help=(
"Base SSIM directory that contains "
"reference_videos/<quality-tier>/<GPU>_reference_videos."
),
help=("Base SSIM directory that contains reference_videos/<quality-tier>/<GPU>_reference_videos."),
)
upload_parser.add_argument(
"--private",
@@ -538,10 +520,7 @@ def _build_parser() -> argparse.ArgumentParser:
"--local-dir",
type=Path,
default=_ssim_dir(),
help=(
"Local SSIM directory that should contain "
"reference_videos/<quality-tier>/<GPU>_reference_videos."
),
help=("Local SSIM directory that should contain reference_videos/<quality-tier>/<GPU>_reference_videos."),
)
ensure_parser.add_argument(
"--quality-tier",
@@ -577,21 +556,20 @@ def main(argv: Sequence[str] | None = None) -> int:
reference_dir = (
args.reference_dir
if args.reference_dir is not None
else (
_ssim_dir()
/ REFERENCE_VIDEOS_DIRNAME
/ args.quality_tier
/ device_folder
)
else (_ssim_dir() / REFERENCE_VIDEOS_DIRNAME / args.quality_tier / device_folder)
)
copied = copy_generated_to_reference(
generated_dir=generated_dir,
reference_dir=reference_dir,
dry_run=args.dry_run,
)
print(
f"Done. {'Would copy' if args.dry_run else 'Copied'} {copied} video files."
)
print(f"Done. {'Would copy' if args.dry_run else 'Copied'} {copied} video files.")
if not args.dry_run and copied > 0:
marker = (
_ssim_dir() / REFERENCE_VIDEOS_DIRNAME / args.quality_tier / f".download_complete_{args.quality_tier}"
)
marker.parent.mkdir(parents=True, exist_ok=True)
marker.touch()
return 0
if args.command == "download":
@@ -603,19 +581,14 @@ def main(argv: Sequence[str] | None = None) -> int:
quality_tiers=quality_tiers,
device_folders=args.device_folder or None,
)
print(
"Download complete for quality tier(s): "
+ ", ".join(quality_tiers)
+ "."
)
print("Download complete for quality tier(s): " + ", ".join(quality_tiers) + ".")
return 0
if args.command == "upload":
token = _get_hf_token()
if token is None:
raise RuntimeError(
"Hugging Face API key is required for upload. Set "
"HF_API_KEY, HUGGINGFACE_HUB_TOKEN, or HF_TOKEN."
"Hugging Face API key is required for upload. Set HF_API_KEY, HUGGINGFACE_HUB_TOKEN, or HF_TOKEN."
)
reference_dirs_by_tier = _resolve_upload_reference_dirs(
@@ -626,8 +599,7 @@ def main(argv: Sequence[str] | None = None) -> int:
)
if not reference_dirs_by_tier:
raise RuntimeError(
"No *_reference_videos directories found for the selected "
f"quality tier(s) under {args.base_dir}"
f"No *_reference_videos directories found for the selected quality tier(s) under {args.base_dir}"
)
upload_reference_videos(
@@ -648,10 +620,7 @@ def main(argv: Sequence[str] | None = None) -> int:
quality_tier=args.quality_tier,
)
if downloaded:
print(
f"{args.quality_tier} reference videos were missing and "
"have been downloaded."
)
print(f"{args.quality_tier} reference videos were missing and have been downloaded.")
else:
print(f"{args.quality_tier} reference videos already exist locally.")
return 0