nodes/hao-ai-lab-FastVideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[ci]: include FA4 in perf identity

Browse Source
This commit is contained in:
Mac Lee
2026-07-15 13:34:45 -07:00
committed by SolitaryThinker
parent df71595ae7
commit 4e38406e0b
4 changed files with 77 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/contributing/performance_benchmarks.md
+6 -1
View File
@@ -197,12 +197,16 @@ Comparator summaries and normalized artifacts include an explicit
| `PASS` | Comparable baseline exists and no gated metric regressed. Legacy records with no baseline also keep the historical initialization behavior. | Passes |
| `REGRESSION` | Comparable baseline exists and at least one gated metric regressed. | Fails |
| `CALIBRATION_NEEDED` | A v2 record has no exact comparable baseline. | Passes, may upload when `PERF_UPLOAD_POLICY=pass`, never seeds a baseline |
| `RECIPE_MISMATCH` | The same workload, variant, benchmark version, hardware profile, and software profile has existing successful records under another recipe fingerprint. | Fails |
| `RECIPE_MISMATCH` | The same workload, variant, benchmark version, hardware profile, and software profile has trusted successful records under another recipe fingerprint. | Fails |
| `INFRA_ERROR` | The comparator cannot safely classify the record, such as a v2 record missing required identity fields. | Fails |
`QUALITY_BLOCKED` is reserved for a future variant-promotion workflow and is
not emitted by normal rolling-baseline comparison.
For `RECIPE_MISMATCH`, trusted records are scheduled-main records or records
already marked `baseline_eligible`. PR and local calibration uploads remain
visible but do not authorize future CI-gating recipe mismatch failures.
When the baseline shifts for a legitimate reason (torch upgrade, kernel
change, etc.) and CI starts failing, use the
[`reseed-performance-baseline`](https://github.com/hao-ai-lab/FastVideo/blob/main/.agents/skills/reseed-performance-baseline/SKILL.md)
@@ -427,6 +431,7 @@ FlashInfer, Cutlass DSL, SageAttention, Triton, and xFormers when installed.
| `DASHBOARD_DAYS` | `30` | `dashboard.py` | Lookback window for the Plotly trend pages. |
| `PERFORMANCE_TRACKING_SYNC_REUSE_TTL_SECONDS` | `3600` | `fastvideo/performance/hf_store.py` | Freshness window for reusing an existing HF sync when requested by dashboard consumers. |
| `FASTVIDEO_ATTENTION_BACKEND` | `auto` | `test_inference_performance.py` | Requested attention backend included in `software_profile_id`. |
| `FASTVIDEO_FA4` | `0` | `test_inference_performance.py` | FlashAttention-4 toggle included in `software_profile_id`. |
| `FASTVIDEO_PERFORMANCE_PROFILE_VERSION` | unset | `test_inference_performance.py` | Optional explicit software cohort/profile version included in `software_profile_id`. |
| `IMAGE_VERSION` | unset | `test_inference_performance.py` | CI container image/profile version included in `software_profile_id` when available. |
| `FASTVIDEO_STAGE_LOGGING` | set by the pytest test | `test_inference_performance.py` | Enables pipeline stage timing capture for component metrics during benchmark runs. |
fastvideo/tests/performance/compare_baseline.py
+6 -1
View File
@@ -220,11 +220,16 @@ def _recipe_mismatch_records(
current_recipe = str(record.get("recipe_fingerprint"))
return [
item for item in cohort_records
if _none_if_empty(item.get("recipe_fingerprint")) is not None
if _record_can_authorize_recipe_mismatch(item)
and _none_if_empty(item.get("recipe_fingerprint")) is not None
and str(item.get("recipe_fingerprint")) != current_recipe
]
def _record_can_authorize_recipe_mismatch(record: dict[str, Any]) -> bool:
return record.get("run_source") == "scheduled_main" or record.get("baseline_eligible") is True
def _format_identity_filters(filters: dict[str, str]) -> str:
if not filters:
return ""
fastvideo/tests/performance/test_compare_baseline_policy.py
+55 -2
View File
@@ -581,11 +581,14 @@ def test_same_recipe_calibration_record_does_not_report_recipe_mismatch(monkeypa
assert compare_baseline._recipe_mismatch_records(
record,
[{"recipe_fingerprint": "recipe-1"}],
[{
"recipe_fingerprint": "recipe-1",
"run_source": "scheduled_main",
}],
) == []
def test_prior_uploaded_calibration_record_can_report_recipe_mismatch(
def test_prior_scheduled_main_calibration_record_can_report_recipe_mismatch(
monkeypatch,
tmp_path,
):
@@ -603,6 +606,7 @@ def test_prior_uploaded_calibration_record_can_report_recipe_mismatch(
"timestamp": "2026-06-15T00:00:00+00:00",
"success": True,
"baseline_eligible": False,
"run_source": "scheduled_main",
"comparison_status": compare_baseline.STATUS_CALIBRATION_NEEDED,
"latency": 10.0,
"workload_id": "wan-t2v",
@@ -634,6 +638,55 @@ def test_prior_uploaded_calibration_record_can_report_recipe_mismatch(
assert "recipe-1" in normalized["comparison_status_reason"]
def test_prior_pr_calibration_record_does_not_report_recipe_mismatch(
monkeypatch,
tmp_path,
):
tracking_root = tmp_path / "tracking"
results_dir = tmp_path / "results"
reports_dir = tmp_path / "reports"
results_dir.mkdir()
_write_record(
tracking_root,
"renamed-benchmark-id",
"prior-pr-calibration.json",
{
"model_id": "renamed-benchmark-id",
"gpu_type": "NVIDIA L40S PCIe",
"timestamp": "2026-06-15T00:00:00+00:00",
"success": True,
"baseline_eligible": False,
"run_source": "pr",
"comparison_status": compare_baseline.STATUS_CALIBRATION_NEEDED,
"latency": 10.0,
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"recipe_fingerprint": "recipe-1",
"hardware_profile_id": "hw-l40s-2",
"software_profile_id": "sw-cuda",
},
)
with open(results_dir / "perf_current.json", "w", encoding="utf-8") as f:
json.dump(_v2_raw_result(recipe_fingerprint="recipe-2"), f)
monkeypatch.setattr(compare_baseline, "TRACKING_ROOT", str(tracking_root))
monkeypatch.setattr(compare_baseline, "RESULTS_DIR", str(results_dir))
monkeypatch.setattr(compare_baseline, "PERF_REPORTS_DIR", str(reports_dir))
monkeypatch.setattr(compare_baseline, "UPLOAD_POLICY", "never")
monkeypatch.setattr(compare_baseline, "sync_from_hf", lambda local_dir, strict=False: local_dir)
monkeypatch.delenv("PERF_PYTEST_RC", raising=False)
assert compare_baseline.main() == 0
artifacts = list((reports_dir / "results").glob("normalized_perf_*.json"))
assert len(artifacts) == 1
with open(artifacts[0], encoding="utf-8") as f:
normalized = json.load(f)
assert normalized["comparison_status"] == compare_baseline.STATUS_CALIBRATION_NEEDED
def test_v2_records_do_not_compare_against_v1_records_by_default(tmp_path):
model_id = "wan-t2v-1.3b-2gpu"
model_dir = tmp_path / sanitize(model_id)
fastvideo/tests/performance/test_inference_performance_identity.py
+10
View File
@@ -7,6 +7,7 @@ from fastvideo.tests.performance import test_inference_performance as perf
def _benchmark_config():
return {
"benchmark_id": "wan-t2v-1.3b-2gpu",
"config_schema_version": perf.V2_CONFIG_SCHEMA_VERSION,
"workload_id": "wan-t2v-1.3b",
"variant_id": "canonical",
"benchmark_version": 1,
@@ -107,6 +108,7 @@ def test_software_profile_tracks_attention_kernel_and_container_versions(monkeyp
monkeypatch.setattr(perf.importlib_metadata, "version", fake_version)
monkeypatch.setenv("FASTVIDEO_ATTENTION_BACKEND", "SAGE_ATTN")
monkeypatch.setenv("FASTVIDEO_FA4", "1")
monkeypatch.setenv("FASTVIDEO_PERFORMANCE_PROFILE_VERSION", "perf-profile-v2")
monkeypatch.setenv("IMAGE_VERSION", "py3.12-cuda13.0")
@@ -121,6 +123,7 @@ def test_software_profile_tracks_attention_kernel_and_container_versions(monkeyp
assert profile["profile_schema_version"] == perf.SOFTWARE_PROFILE_SCHEMA_VERSION
assert profile["attention_backend"] == "SAGE_ATTN"
assert profile["flash_attention_4_enabled"] is True
assert profile["performance_profile_version"] == "perf-profile-v2"
assert profile["container_image_version"] == "py3.12-cuda13.0"
assert profile["packages"] == {
@@ -132,3 +135,10 @@ def test_software_profile_tracks_attention_kernel_and_container_versions(monkeyp
"flashinfer-python": "0.2",
}
assert perf._profile_id("sw", profile) != perf._profile_id("sw", changed_profile)
assert perf._profile_id("sw", profile) != perf._profile_id(
"sw",
{
**profile,
"flash_attention_4_enabled": False,
},
)