[ci]: tighten perf software identity

This commit is contained in:
Mac Lee
2026-07-15 13:34:46 -07:00
committed by SolitaryThinker
parent 4e38406e0b
commit ca0ad47841
4 changed files with 26 additions and 9 deletions
@@ -434,6 +434,7 @@ FlashInfer, Cutlass DSL, SageAttention, Triton, and xFormers when installed.
| `FASTVIDEO_FA4` | `0` | `test_inference_performance.py` | FlashAttention-4 toggle included in `software_profile_id`. |
| `FASTVIDEO_PERFORMANCE_PROFILE_VERSION` | unset | `test_inference_performance.py` | Optional explicit software cohort/profile version included in `software_profile_id`. |
| `IMAGE_VERSION` | unset | `test_inference_performance.py` | CI container image/profile version included in `software_profile_id` when available. |
| `FASTVIDEO_CONTAINER_IMAGE_REF` | unset | `pr_test.py`, `launch_l40s_job.py`, `test_inference_performance.py` | Resolved CI container image ref or digest included in `software_profile_id` when available. |
| `FASTVIDEO_STAGE_LOGGING` | set by the pytest test | `test_inference_performance.py` | Enables pipeline stage timing capture for component metrics during benchmark runs. |
## CI integration
+2
View File
@@ -95,6 +95,8 @@ image = (
"PATH": "/root/.cargo/bin:$PATH",
"HF_HOME": "/root/data/.cache",
"TOKENIZERS_PARALLELISM": "false",
"IMAGE_VERSION": IMAGE_VERSION,
"FASTVIDEO_CONTAINER_IMAGE_REF": IMAGE_REF,
**({"UV_TORCH_BACKEND": uv_torch_backend_override} if uv_torch_backend_override else {}),
"FASTVIDEO_ATTENTION_BACKEND": os.environ.get("FASTVIDEO_ATTENTION_BACKEND", "FLASH_ATTN"),
# FA4 is opt-in (FASTVIDEO_FA4). Generic ad hoc jobs should follow the
@@ -11,7 +11,6 @@ import hashlib
import json
import os
import platform
import re
import time
from collections.abc import Mapping
from datetime import datetime, timezone
@@ -91,7 +91,7 @@ def test_performance_producer_emits_v2_identity_from_raw_result_shape(monkeypatc
assert "output_path" not in record["recipe"]["generation_kwargs"]
def test_software_profile_tracks_attention_kernel_and_container_versions(monkeypatch):
def test_software_profile_tracks_exact_runtime_attention_kernel_and_container_versions(monkeypatch):
package_versions = {
"triton": "3.2.1",
"flash-attn": "2.8.1",
@@ -107,32 +107,40 @@ def test_software_profile_tracks_attention_kernel_and_container_versions(monkeyp
return package_versions[package_name]
monkeypatch.setattr(perf.importlib_metadata, "version", fake_version)
monkeypatch.setattr(perf.platform, "python_version", lambda: "3.12.11")
monkeypatch.setattr(perf.torch, "__version__", "2.7.1+cu128")
monkeypatch.setattr(perf.torch.version, "cuda", "12.8.1", raising=False)
monkeypatch.setenv("FASTVIDEO_ATTENTION_BACKEND", "SAGE_ATTN")
monkeypatch.setenv("FASTVIDEO_FA4", "1")
monkeypatch.setenv("FASTVIDEO_PERFORMANCE_PROFILE_VERSION", "perf-profile-v2")
monkeypatch.setenv("IMAGE_VERSION", "py3.12-cuda13.0")
monkeypatch.setenv("FASTVIDEO_CONTAINER_IMAGE_REF", "ghcr.io/hao-ai-lab/fastvideo/fastvideo-dev@sha256:abc")
profile = perf._software_profile()
changed_profile = {
**profile,
"packages": {
**profile["packages"],
"triton": "3.3",
"triton": "3.2.2",
},
}
assert profile["profile_schema_version"] == perf.SOFTWARE_PROFILE_SCHEMA_VERSION
assert profile["python"] == "3.12.11"
assert profile["pytorch"] == "2.7.1+cu128"
assert profile["cuda"] == "12.8.1"
assert profile["attention_backend"] == "SAGE_ATTN"
assert profile["flash_attention_4_enabled"] is True
assert profile["performance_profile_version"] == "perf-profile-v2"
assert profile["container_image_version"] == "py3.12-cuda13.0"
assert profile["container_image_ref"] == "ghcr.io/hao-ai-lab/fastvideo/fastvideo-dev@sha256:abc"
assert profile["packages"] == {
"triton": "3.2",
"flash-attn": "2.8",
"sageattention": "2.1",
"xformers": "0.0",
"fastvideo-kernel": "0.3",
"flashinfer-python": "0.2",
"triton": "3.2.1",
"flash-attn": "2.8.1",
"sageattention": "2.1.1",
"xformers": "0.0.31",
"fastvideo-kernel": "0.3.2",
"flashinfer-python": "0.2.3",
}
assert perf._profile_id("sw", profile) != perf._profile_id("sw", changed_profile)
assert perf._profile_id("sw", profile) != perf._profile_id(
@@ -142,3 +150,10 @@ def test_software_profile_tracks_attention_kernel_and_container_versions(monkeyp
"flash_attention_4_enabled": False,
},
)
assert perf._profile_id("sw", profile) != perf._profile_id(
"sw",
{
**profile,
"container_image_ref": "ghcr.io/hao-ai-lab/fastvideo/fastvideo-dev@sha256:def",
},
)