diff --git a/docs/contributing/performance_benchmarks.md b/docs/contributing/performance_benchmarks.md index 96f4b288b..790e8b1ce 100644 --- a/docs/contributing/performance_benchmarks.md +++ b/docs/contributing/performance_benchmarks.md @@ -197,12 +197,16 @@ Comparator summaries and normalized artifacts include an explicit | `PASS` | Comparable baseline exists and no gated metric regressed. Legacy records with no baseline also keep the historical initialization behavior. | Passes | | `REGRESSION` | Comparable baseline exists and at least one gated metric regressed. | Fails | | `CALIBRATION_NEEDED` | A v2 record has no exact comparable baseline. | Passes, may upload when `PERF_UPLOAD_POLICY=pass`, never seeds a baseline | -| `RECIPE_MISMATCH` | The same workload, variant, benchmark version, hardware profile, and software profile has existing successful records under another recipe fingerprint. | Fails | +| `RECIPE_MISMATCH` | The same workload, variant, benchmark version, hardware profile, and software profile has trusted successful records under another recipe fingerprint. | Fails | | `INFRA_ERROR` | The comparator cannot safely classify the record, such as a v2 record missing required identity fields. | Fails | `QUALITY_BLOCKED` is reserved for a future variant-promotion workflow and is not emitted by normal rolling-baseline comparison. +For `RECIPE_MISMATCH`, trusted records are scheduled-main records or records +already marked `baseline_eligible`. PR and local calibration uploads remain +visible but do not authorize future CI-gating recipe mismatch failures. + When the baseline shifts for a legitimate reason (torch upgrade, kernel change, etc.) and CI starts failing, use the [`reseed-performance-baseline`](https://github.com/hao-ai-lab/FastVideo/blob/main/.agents/skills/reseed-performance-baseline/SKILL.md) @@ -427,6 +431,7 @@ FlashInfer, Cutlass DSL, SageAttention, Triton, and xFormers when installed. | `DASHBOARD_DAYS` | `30` | `dashboard.py` | Lookback window for the Plotly trend pages. | | `PERFORMANCE_TRACKING_SYNC_REUSE_TTL_SECONDS` | `3600` | `fastvideo/performance/hf_store.py` | Freshness window for reusing an existing HF sync when requested by dashboard consumers. | | `FASTVIDEO_ATTENTION_BACKEND` | `auto` | `test_inference_performance.py` | Requested attention backend included in `software_profile_id`. | +| `FASTVIDEO_FA4` | `0` | `test_inference_performance.py` | FlashAttention-4 toggle included in `software_profile_id`. | | `FASTVIDEO_PERFORMANCE_PROFILE_VERSION` | unset | `test_inference_performance.py` | Optional explicit software cohort/profile version included in `software_profile_id`. | | `IMAGE_VERSION` | unset | `test_inference_performance.py` | CI container image/profile version included in `software_profile_id` when available. | | `FASTVIDEO_STAGE_LOGGING` | set by the pytest test | `test_inference_performance.py` | Enables pipeline stage timing capture for component metrics during benchmark runs. | diff --git a/fastvideo/tests/performance/compare_baseline.py b/fastvideo/tests/performance/compare_baseline.py index e3462c15e..e2ae2d8fd 100644 --- a/fastvideo/tests/performance/compare_baseline.py +++ b/fastvideo/tests/performance/compare_baseline.py @@ -220,11 +220,16 @@ def _recipe_mismatch_records( current_recipe = str(record.get("recipe_fingerprint")) return [ item for item in cohort_records - if _none_if_empty(item.get("recipe_fingerprint")) is not None + if _record_can_authorize_recipe_mismatch(item) + and _none_if_empty(item.get("recipe_fingerprint")) is not None and str(item.get("recipe_fingerprint")) != current_recipe ] +def _record_can_authorize_recipe_mismatch(record: dict[str, Any]) -> bool: + return record.get("run_source") == "scheduled_main" or record.get("baseline_eligible") is True + + def _format_identity_filters(filters: dict[str, str]) -> str: if not filters: return "" diff --git a/fastvideo/tests/performance/test_compare_baseline_policy.py b/fastvideo/tests/performance/test_compare_baseline_policy.py index df7ef7053..e1509af34 100644 --- a/fastvideo/tests/performance/test_compare_baseline_policy.py +++ b/fastvideo/tests/performance/test_compare_baseline_policy.py @@ -581,11 +581,14 @@ def test_same_recipe_calibration_record_does_not_report_recipe_mismatch(monkeypa assert compare_baseline._recipe_mismatch_records( record, - [{"recipe_fingerprint": "recipe-1"}], + [{ + "recipe_fingerprint": "recipe-1", + "run_source": "scheduled_main", + }], ) == [] -def test_prior_uploaded_calibration_record_can_report_recipe_mismatch( +def test_prior_scheduled_main_calibration_record_can_report_recipe_mismatch( monkeypatch, tmp_path, ): @@ -603,6 +606,7 @@ def test_prior_uploaded_calibration_record_can_report_recipe_mismatch( "timestamp": "2026-06-15T00:00:00+00:00", "success": True, "baseline_eligible": False, + "run_source": "scheduled_main", "comparison_status": compare_baseline.STATUS_CALIBRATION_NEEDED, "latency": 10.0, "workload_id": "wan-t2v", @@ -634,6 +638,55 @@ def test_prior_uploaded_calibration_record_can_report_recipe_mismatch( assert "recipe-1" in normalized["comparison_status_reason"] +def test_prior_pr_calibration_record_does_not_report_recipe_mismatch( + monkeypatch, + tmp_path, +): + tracking_root = tmp_path / "tracking" + results_dir = tmp_path / "results" + reports_dir = tmp_path / "reports" + results_dir.mkdir() + _write_record( + tracking_root, + "renamed-benchmark-id", + "prior-pr-calibration.json", + { + "model_id": "renamed-benchmark-id", + "gpu_type": "NVIDIA L40S PCIe", + "timestamp": "2026-06-15T00:00:00+00:00", + "success": True, + "baseline_eligible": False, + "run_source": "pr", + "comparison_status": compare_baseline.STATUS_CALIBRATION_NEEDED, + "latency": 10.0, + "workload_id": "wan-t2v", + "variant_id": "1.3b-sp2", + "benchmark_version": 2, + "recipe_fingerprint": "recipe-1", + "hardware_profile_id": "hw-l40s-2", + "software_profile_id": "sw-cuda", + }, + ) + with open(results_dir / "perf_current.json", "w", encoding="utf-8") as f: + json.dump(_v2_raw_result(recipe_fingerprint="recipe-2"), f) + + monkeypatch.setattr(compare_baseline, "TRACKING_ROOT", str(tracking_root)) + monkeypatch.setattr(compare_baseline, "RESULTS_DIR", str(results_dir)) + monkeypatch.setattr(compare_baseline, "PERF_REPORTS_DIR", str(reports_dir)) + monkeypatch.setattr(compare_baseline, "UPLOAD_POLICY", "never") + monkeypatch.setattr(compare_baseline, "sync_from_hf", lambda local_dir, strict=False: local_dir) + monkeypatch.delenv("PERF_PYTEST_RC", raising=False) + + assert compare_baseline.main() == 0 + + artifacts = list((reports_dir / "results").glob("normalized_perf_*.json")) + assert len(artifacts) == 1 + with open(artifacts[0], encoding="utf-8") as f: + normalized = json.load(f) + + assert normalized["comparison_status"] == compare_baseline.STATUS_CALIBRATION_NEEDED + + def test_v2_records_do_not_compare_against_v1_records_by_default(tmp_path): model_id = "wan-t2v-1.3b-2gpu" model_dir = tmp_path / sanitize(model_id) diff --git a/fastvideo/tests/performance/test_inference_performance_identity.py b/fastvideo/tests/performance/test_inference_performance_identity.py index f22420a8d..56b663d88 100644 --- a/fastvideo/tests/performance/test_inference_performance_identity.py +++ b/fastvideo/tests/performance/test_inference_performance_identity.py @@ -7,6 +7,7 @@ from fastvideo.tests.performance import test_inference_performance as perf def _benchmark_config(): return { "benchmark_id": "wan-t2v-1.3b-2gpu", + "config_schema_version": perf.V2_CONFIG_SCHEMA_VERSION, "workload_id": "wan-t2v-1.3b", "variant_id": "canonical", "benchmark_version": 1, @@ -107,6 +108,7 @@ def test_software_profile_tracks_attention_kernel_and_container_versions(monkeyp monkeypatch.setattr(perf.importlib_metadata, "version", fake_version) monkeypatch.setenv("FASTVIDEO_ATTENTION_BACKEND", "SAGE_ATTN") + monkeypatch.setenv("FASTVIDEO_FA4", "1") monkeypatch.setenv("FASTVIDEO_PERFORMANCE_PROFILE_VERSION", "perf-profile-v2") monkeypatch.setenv("IMAGE_VERSION", "py3.12-cuda13.0") @@ -121,6 +123,7 @@ def test_software_profile_tracks_attention_kernel_and_container_versions(monkeyp assert profile["profile_schema_version"] == perf.SOFTWARE_PROFILE_SCHEMA_VERSION assert profile["attention_backend"] == "SAGE_ATTN" + assert profile["flash_attention_4_enabled"] is True assert profile["performance_profile_version"] == "perf-profile-v2" assert profile["container_image_version"] == "py3.12-cuda13.0" assert profile["packages"] == { @@ -132,3 +135,10 @@ def test_software_profile_tracks_attention_kernel_and_container_versions(monkeyp "flashinfer-python": "0.2", } assert perf._profile_id("sw", profile) != perf._profile_id("sw", changed_profile) + assert perf._profile_id("sw", profile) != perf._profile_id( + "sw", + { + **profile, + "flash_attention_4_enabled": False, + }, + )