diff --git a/docs/contributing/performance_benchmarks.md b/docs/contributing/performance_benchmarks.md index 0c13b565d..06e7dbf59 100644 --- a/docs/contributing/performance_benchmarks.md +++ b/docs/contributing/performance_benchmarks.md @@ -79,10 +79,11 @@ fastvideo/performance/ ``` The HF dataset (`FastVideo/performance-tracking` by default) holds one -normalized JSON per `(model_id, gpu_type, run)` tuple. The rolling baseline is -the median of the last 5 successful, baseline-eligible records for that -model+GPU. PR and local records are visible in the dashboard but are not -baseline eligible. +normalized JSON per run. For v2 records, the rolling baseline is the median of +the last 5 successful, baseline-eligible records in the same comparison cohort: +`model_id`, `gpu_type`, `workload_id`, `variant_id`, `benchmark_version`, +`recipe_fingerprint`, `hardware_profile_id`, and `software_profile_id`. PR and +local records are visible in the dashboard but are not baseline eligible. ## Planned Coverage @@ -158,13 +159,16 @@ unrealistic memory growth, and optionally large component-specific slowdowns even when the rolling baseline is empty. They are hand-set with generous headroom and almost never need touching. -### Rolling baseline (per `(model_id, gpu_type)`) +### Rolling baseline (per comparison cohort) `compare_baseline.py` loads the last 5 successful, baseline-eligible records -for the same `(model_id, gpu_type)` from the HF dataset, computes the median -for each available metric, and evaluates the current run with the metric's -rolling regression policy. For latency, memory, and component times, higher -values are regressions. For throughput, lower values are regressions. +for the same comparison cohort from the HF dataset, computes the median for +each available metric, and evaluates the current run with the metric's +rolling regression policy. For v2 records, that cohort is `model_id`, +`gpu_type`, `workload_id`, `variant_id`, `benchmark_version`, +`recipe_fingerprint`, `hardware_profile_id`, and `software_profile_id`. For +latency, memory, and component times, higher values are regressions. For +throughput, lower values are regressions. A metric exceeds its rolling threshold when both of these are true: @@ -424,7 +428,7 @@ When the rolling-baseline phase runs, it emits: per-benchmark row with current vs. baseline values for latency, throughput, memory, text encoder time, DiT time, and VAE decode time. * **Plotly dashboard** — `dashboard__.html` showing time-series for - each metric grouped by `(model_id, gpu_type)`. + each metric grouped by comparison cohort. * **Normalized records** — `normalized_perf_*.json`, one per benchmark. Useful as input to the [`reseed-performance-baseline`](https://github.com/hao-ai-lab/FastVideo/blob/main/.agents/skills/reseed-performance-baseline/SKILL.md) @@ -498,8 +502,8 @@ When the rolling-baseline phase runs, it emits: ## Troubleshooting -**"No baseline for ... Initializing"** — first run for this `(model_id, -gpu_type)`. Run will pass and (if persisting) seed the first record. +**"No baseline for ... Initializing"** — first run for this comparison cohort. +Run will pass and (if persisting) seed the first record. **Persistent failure right after a torch / kernel / image upgrade** — genuine regression *or* baseline drift. Compare the failing normalized record diff --git a/fastvideo/performance_dashboard/service.py b/fastvideo/performance_dashboard/service.py index fb070d277..8b6ef7cbf 100644 --- a/fastvideo/performance_dashboard/service.py +++ b/fastvideo/performance_dashboard/service.py @@ -86,20 +86,32 @@ def record_metadata(record: Record) -> Record: } +def _cohort_metadata_value(value: Any) -> Any: + if value is None: + return "" + if isinstance(value, str) and not value.strip(): + return "" + return value + + +def _cohort_key_value(value: Any) -> str: + return str(_cohort_metadata_value(value)) + + def record_comparison_metadata(record: Record) -> Record: - return {key: record.get(key) or "" for key in COMPARISON_COHORT_KEYS} + return {key: _cohort_metadata_value(record.get(key)) for key in COMPARISON_COHORT_KEYS} def comparison_cohort_key(record: Record) -> CohortKey: return ( str(record.get("model_id") or "unknown"), str(record.get("gpu_type") or "unknown"), - str(record.get("workload_id") or ""), - str(record.get("variant_id") or ""), - str(record.get("benchmark_version") or ""), - str(record.get("recipe_fingerprint") or ""), - str(record.get("hardware_profile_id") or ""), - str(record.get("software_profile_id") or ""), + _cohort_key_value(record.get("workload_id")), + _cohort_key_value(record.get("variant_id")), + _cohort_key_value(record.get("benchmark_version")), + _cohort_key_value(record.get("recipe_fingerprint")), + _cohort_key_value(record.get("hardware_profile_id")), + _cohort_key_value(record.get("software_profile_id")), ) @@ -146,9 +158,10 @@ def build_latest_summary(records: list[Record], continue latest = latest_candidates[-1] + latest_index = next(index for index, record in enumerate(group) if record is latest) baseline_pool = [ - record for record in group - if record is not latest and record.get("success", True) and is_baseline_eligible_record(record) + record for record in group[:latest_index] + if record.get("success", True) and is_baseline_eligible_record(record) ] baseline_records = baseline_pool[-baseline_window:] metric_policies = resolve_metric_policies(latest.get("regression_thresholds")) diff --git a/fastvideo/tests/performance/test_dashboard_api.py b/fastvideo/tests/performance/test_dashboard_api.py index 41a25bccf..d585b0b7e 100644 --- a/fastvideo/tests/performance/test_dashboard_api.py +++ b/fastvideo/tests/performance/test_dashboard_api.py @@ -154,8 +154,8 @@ def test_dashboard_endpoints_filter_and_return_run_source_metadata(): assert summary["count"] == 1 assert summary["rows"][0]["run_source"] == "pr" assert summary["rows"][0]["pr_number"] == "123" - assert summary["rows"][0]["baseline_n"] == 1 - assert summary["rows"][0]["metrics"]["latency"]["baseline"] == 11.0 + assert summary["rows"][0]["baseline_n"] == 0 + assert summary["rows"][0]["metrics"]["latency"]["baseline"] is None assert summary["filters"]["run_source"] == "pr" assert trends["count"] == 1 assert trends["groups"][0]["points"][0]["run_source"] == "pr" diff --git a/fastvideo/tests/performance/test_dashboard_service.py b/fastvideo/tests/performance/test_dashboard_service.py index 8fa3dfbcd..6ac52ff9b 100644 --- a/fastvideo/tests/performance/test_dashboard_service.py +++ b/fastvideo/tests/performance/test_dashboard_service.py @@ -142,6 +142,47 @@ def test_build_latest_summary_separates_informational_threshold_crossing(): assert rows[0]["threshold_exceeded_metrics"] == ["latency"] assert rows[0]["failing_metrics"] == [] assert rows[0]["computed_regression_status"] == "pass" + + +def test_build_latest_summary_run_source_filter_excludes_future_baselines(): + records = [ + _record( + "2026-01-01T00:00:00+00:00", + "a" * 40, + 10.0, + 10.0, + run_source="scheduled_main", + baseline_eligible=True, + ), + _record( + "2026-01-02T00:00:00+00:00", + "b" * 40, + 11.0, + 9.0, + run_source="pr", + baseline_eligible=False, + pr_number="123", + ), + _record( + "2026-01-03T00:00:00+00:00", + "c" * 40, + 30.0, + 3.0, + run_source="scheduled_main", + baseline_eligible=True, + ), + ] + + rows = build_latest_summary(records, run_source="pr") + + assert len(rows) == 1 + assert rows[0]["run_source"] == "pr" + assert rows[0]["baseline_n"] == 1 + assert rows[0]["metrics"]["latency"]["baseline"] == 10.0 + assert rows[0]["metrics"]["throughput"]["baseline"] == 10.0 + + + def test_build_latest_summary_keeps_identity_cohorts_separate(): records = [ _record( @@ -186,7 +227,7 @@ def test_build_latest_summary_keeps_identity_cohorts_separate(): ), ] - rows = build_latest_summary(records, max_regression=0.05) + rows = build_latest_summary(records) recipe_b_row = next(row for row in rows if row["recipe_fingerprint"] == "recipe-b") assert len(rows) == 2 @@ -241,7 +282,7 @@ def test_build_latest_summary_keeps_variant_versions_separate(): ), ] - rows = build_latest_summary(records, max_regression=0.05) + rows = build_latest_summary(records) version_2_row = next(row for row in rows if row["benchmark_version"] == 2) assert len(rows) == 2 @@ -249,6 +290,58 @@ def test_build_latest_summary_keeps_variant_versions_separate(): assert version_2_row["metrics"]["latency"]["baseline"] == 20.0 +def test_dashboard_identity_preserves_zero_benchmark_version(): + records = [ + _record( + "2026-01-01T00:00:00+00:00", + "a" * 40, + 10.0, + 10.0, + workload_id="wan-t2v", + variant_id="1.3b-sp2", + benchmark_version=0, + recipe_fingerprint="recipe-a", + hardware_profile_id="hw-l40s", + software_profile_id="sw-cu130", + run_source="scheduled_main", + baseline_eligible=True, + ), + _record( + "2026-01-02T00:00:00+00:00", + "b" * 40, + 11.0, + 9.0, + workload_id="wan-t2v", + variant_id="1.3b-sp2", + benchmark_version=0, + recipe_fingerprint="recipe-a", + hardware_profile_id="hw-l40s", + software_profile_id="sw-cu130", + ), + _record( + "2026-01-03T00:00:00+00:00", + "c" * 40, + 20.0, + 5.0, + ), + ] + + rows = build_latest_summary(records) + version_zero_row = next(row for row in rows if row["benchmark_version"] == 0) + legacy_row = next(row for row in rows if row["benchmark_version"] == "") + trends = build_trends(records) + version_zero_trend = next(trend for trend in trends if trend["benchmark_version"] == 0) + legacy_trend = next(trend for trend in trends if trend["benchmark_version"] == "") + + assert len(rows) == 2 + assert version_zero_row["baseline_n"] == 1 + assert version_zero_row["metrics"]["latency"]["baseline"] == 10.0 + assert legacy_row["baseline_n"] == 0 + assert len(trends) == 2 + assert version_zero_trend["points"][0]["benchmark_version"] == 0 + assert legacy_trend["points"][0]["benchmark_version"] == "" + + def test_filter_records_and_trends_preserve_metric_points(): records = [ _record("2026-01-01T00:00:00+00:00", "a" * 40, 10.0, 10.0),