From ca0ad47841aa4f2d45e593dce3c4c87d42401268 Mon Sep 17 00:00:00 2001 From: Mac Lee Date: Sun, 5 Jul 2026 16:00:15 +0000 Subject: [PATCH] [ci]: tighten perf software identity --- docs/contributing/performance_benchmarks.md | 1 + fastvideo/tests/modal/launch_l40s_job.py | 2 ++ .../performance/test_inference_performance.py | 1 - .../test_inference_performance_identity.py | 31 ++++++++++++++----- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/docs/contributing/performance_benchmarks.md b/docs/contributing/performance_benchmarks.md index 790e8b1ce..739546bdb 100644 --- a/docs/contributing/performance_benchmarks.md +++ b/docs/contributing/performance_benchmarks.md @@ -434,6 +434,7 @@ FlashInfer, Cutlass DSL, SageAttention, Triton, and xFormers when installed. | `FASTVIDEO_FA4` | `0` | `test_inference_performance.py` | FlashAttention-4 toggle included in `software_profile_id`. | | `FASTVIDEO_PERFORMANCE_PROFILE_VERSION` | unset | `test_inference_performance.py` | Optional explicit software cohort/profile version included in `software_profile_id`. | | `IMAGE_VERSION` | unset | `test_inference_performance.py` | CI container image/profile version included in `software_profile_id` when available. | +| `FASTVIDEO_CONTAINER_IMAGE_REF` | unset | `pr_test.py`, `launch_l40s_job.py`, `test_inference_performance.py` | Resolved CI container image ref or digest included in `software_profile_id` when available. | | `FASTVIDEO_STAGE_LOGGING` | set by the pytest test | `test_inference_performance.py` | Enables pipeline stage timing capture for component metrics during benchmark runs. | ## CI integration diff --git a/fastvideo/tests/modal/launch_l40s_job.py b/fastvideo/tests/modal/launch_l40s_job.py index 99106125d..36749b315 100644 --- a/fastvideo/tests/modal/launch_l40s_job.py +++ b/fastvideo/tests/modal/launch_l40s_job.py @@ -95,6 +95,8 @@ image = ( "PATH": "/root/.cargo/bin:$PATH", "HF_HOME": "/root/data/.cache", "TOKENIZERS_PARALLELISM": "false", + "IMAGE_VERSION": IMAGE_VERSION, + "FASTVIDEO_CONTAINER_IMAGE_REF": IMAGE_REF, **({"UV_TORCH_BACKEND": uv_torch_backend_override} if uv_torch_backend_override else {}), "FASTVIDEO_ATTENTION_BACKEND": os.environ.get("FASTVIDEO_ATTENTION_BACKEND", "FLASH_ATTN"), # FA4 is opt-in (FASTVIDEO_FA4). Generic ad hoc jobs should follow the diff --git a/fastvideo/tests/performance/test_inference_performance.py b/fastvideo/tests/performance/test_inference_performance.py index a2c2978ff..95689ef1a 100644 --- a/fastvideo/tests/performance/test_inference_performance.py +++ b/fastvideo/tests/performance/test_inference_performance.py @@ -11,7 +11,6 @@ import hashlib import json import os import platform -import re import time from collections.abc import Mapping from datetime import datetime, timezone diff --git a/fastvideo/tests/performance/test_inference_performance_identity.py b/fastvideo/tests/performance/test_inference_performance_identity.py index 56b663d88..13c7d6d07 100644 --- a/fastvideo/tests/performance/test_inference_performance_identity.py +++ b/fastvideo/tests/performance/test_inference_performance_identity.py @@ -91,7 +91,7 @@ def test_performance_producer_emits_v2_identity_from_raw_result_shape(monkeypatc assert "output_path" not in record["recipe"]["generation_kwargs"] -def test_software_profile_tracks_attention_kernel_and_container_versions(monkeypatch): +def test_software_profile_tracks_exact_runtime_attention_kernel_and_container_versions(monkeypatch): package_versions = { "triton": "3.2.1", "flash-attn": "2.8.1", @@ -107,32 +107,40 @@ def test_software_profile_tracks_attention_kernel_and_container_versions(monkeyp return package_versions[package_name] monkeypatch.setattr(perf.importlib_metadata, "version", fake_version) + monkeypatch.setattr(perf.platform, "python_version", lambda: "3.12.11") + monkeypatch.setattr(perf.torch, "__version__", "2.7.1+cu128") + monkeypatch.setattr(perf.torch.version, "cuda", "12.8.1", raising=False) monkeypatch.setenv("FASTVIDEO_ATTENTION_BACKEND", "SAGE_ATTN") monkeypatch.setenv("FASTVIDEO_FA4", "1") monkeypatch.setenv("FASTVIDEO_PERFORMANCE_PROFILE_VERSION", "perf-profile-v2") monkeypatch.setenv("IMAGE_VERSION", "py3.12-cuda13.0") + monkeypatch.setenv("FASTVIDEO_CONTAINER_IMAGE_REF", "ghcr.io/hao-ai-lab/fastvideo/fastvideo-dev@sha256:abc") profile = perf._software_profile() changed_profile = { **profile, "packages": { **profile["packages"], - "triton": "3.3", + "triton": "3.2.2", }, } assert profile["profile_schema_version"] == perf.SOFTWARE_PROFILE_SCHEMA_VERSION + assert profile["python"] == "3.12.11" + assert profile["pytorch"] == "2.7.1+cu128" + assert profile["cuda"] == "12.8.1" assert profile["attention_backend"] == "SAGE_ATTN" assert profile["flash_attention_4_enabled"] is True assert profile["performance_profile_version"] == "perf-profile-v2" assert profile["container_image_version"] == "py3.12-cuda13.0" + assert profile["container_image_ref"] == "ghcr.io/hao-ai-lab/fastvideo/fastvideo-dev@sha256:abc" assert profile["packages"] == { - "triton": "3.2", - "flash-attn": "2.8", - "sageattention": "2.1", - "xformers": "0.0", - "fastvideo-kernel": "0.3", - "flashinfer-python": "0.2", + "triton": "3.2.1", + "flash-attn": "2.8.1", + "sageattention": "2.1.1", + "xformers": "0.0.31", + "fastvideo-kernel": "0.3.2", + "flashinfer-python": "0.2.3", } assert perf._profile_id("sw", profile) != perf._profile_id("sw", changed_profile) assert perf._profile_id("sw", profile) != perf._profile_id( @@ -142,3 +150,10 @@ def test_software_profile_tracks_attention_kernel_and_container_versions(monkeyp "flash_attention_4_enabled": False, }, ) + assert perf._profile_id("sw", profile) != perf._profile_id( + "sw", + { + **profile, + "container_image_ref": "ghcr.io/hao-ai-lab/fastvideo/fastvideo-dev@sha256:def", + }, + )