nodes/hao-ai-lab-FastVideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[ci]: fix v2 dashboard cohort handling (#1531)

Browse Source
This commit is contained in:
Mac Lee
2026-07-05 16:02:56 -07:00
committed by SolitaryThinker
parent f43bcdefae
commit bc51cd6d37
4 changed files with 135 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/contributing/performance_benchmarks.md
+16 -12
View File
@@ -79,10 +79,11 @@ fastvideo/performance/
```
The HF dataset (`FastVideo/performance-tracking` by default) holds one
normalized JSON per `(model_id, gpu_type, run)` tuple. The rolling baseline is
the median of the last 5 successful, baseline-eligible records for that
model+GPU. PR and local records are visible in the dashboard but are not
baseline eligible.
normalized JSON per run. For v2 records, the rolling baseline is the median of
the last 5 successful, baseline-eligible records in the same comparison cohort:
`model_id`, `gpu_type`, `workload_id`, `variant_id`, `benchmark_version`,
`recipe_fingerprint`, `hardware_profile_id`, and `software_profile_id`. PR and
local records are visible in the dashboard but are not baseline eligible.
## Planned Coverage
@@ -158,13 +159,16 @@ unrealistic memory growth, and optionally large component-specific slowdowns
even when the rolling baseline is empty. They are hand-set with generous
headroom and almost never need touching.
### Rolling baseline (per `(model_id, gpu_type)`)
### Rolling baseline (per comparison cohort)
`compare_baseline.py` loads the last 5 successful, baseline-eligible records
for the same `(model_id, gpu_type)` from the HF dataset, computes the median
for each available metric, and evaluates the current run with the metric's
rolling regression policy. For latency, memory, and component times, higher
values are regressions. For throughput, lower values are regressions.
for the same comparison cohort from the HF dataset, computes the median for
each available metric, and evaluates the current run with the metric's
rolling regression policy. For v2 records, that cohort is `model_id`,
`gpu_type`, `workload_id`, `variant_id`, `benchmark_version`,
`recipe_fingerprint`, `hardware_profile_id`, and `software_profile_id`. For
latency, memory, and component times, higher values are regressions. For
throughput, lower values are regressions.
A metric exceeds its rolling threshold when both of these are true:
@@ -424,7 +428,7 @@ When the rolling-baseline phase runs, it emits:
per-benchmark row with current vs. baseline values for latency, throughput,
memory, text encoder time, DiT time, and VAE decode time.
* **Plotly dashboard** — `dashboard_<sha>_<ts>.html` showing time-series for
each metric grouped by `(model_id, gpu_type)`.
each metric grouped by comparison cohort.
* **Normalized records** — `normalized_perf_*.json`, one per benchmark.
Useful as input to the
[`reseed-performance-baseline`](https://github.com/hao-ai-lab/FastVideo/blob/main/.agents/skills/reseed-performance-baseline/SKILL.md)
@@ -498,8 +502,8 @@ When the rolling-baseline phase runs, it emits:
## Troubleshooting
**"No baseline for ... Initializing"** — first run for this `(model_id,
gpu_type)`. Run will pass and (if persisting) seed the first record.
**"No baseline for ... Initializing"** — first run for this comparison cohort.
Run will pass and (if persisting) seed the first record.
**Persistent failure right after a torch / kernel / image upgrade** —
genuine regression *or* baseline drift. Compare the failing normalized record
fastvideo/performance_dashboard/service.py
+22 -9
View File
@@ -86,20 +86,32 @@ def record_metadata(record: Record) -> Record:
}
def _cohort_metadata_value(value: Any) -> Any:
if value is None:
return ""
if isinstance(value, str) and not value.strip():
return ""
return value
def _cohort_key_value(value: Any) -> str:
return str(_cohort_metadata_value(value))
def record_comparison_metadata(record: Record) -> Record:
return {key: record.get(key) or "" for key in COMPARISON_COHORT_KEYS}
return {key: _cohort_metadata_value(record.get(key)) for key in COMPARISON_COHORT_KEYS}
def comparison_cohort_key(record: Record) -> CohortKey:
return (
str(record.get("model_id") or "unknown"),
str(record.get("gpu_type") or "unknown"),
str(record.get("workload_id") or ""),
str(record.get("variant_id") or ""),
str(record.get("benchmark_version") or ""),
str(record.get("recipe_fingerprint") or ""),
str(record.get("hardware_profile_id") or ""),
str(record.get("software_profile_id") or ""),
_cohort_key_value(record.get("workload_id")),
_cohort_key_value(record.get("variant_id")),
_cohort_key_value(record.get("benchmark_version")),
_cohort_key_value(record.get("recipe_fingerprint")),
_cohort_key_value(record.get("hardware_profile_id")),
_cohort_key_value(record.get("software_profile_id")),
)
@@ -146,9 +158,10 @@ def build_latest_summary(records: list[Record],
continue
latest = latest_candidates[-1]
latest_index = next(index for index, record in enumerate(group) if record is latest)
baseline_pool = [
record for record in group
if record is not latest and record.get("success", True) and is_baseline_eligible_record(record)
record for record in group[:latest_index]
if record.get("success", True) and is_baseline_eligible_record(record)
]
baseline_records = baseline_pool[-baseline_window:]
metric_policies = resolve_metric_policies(latest.get("regression_thresholds"))
fastvideo/tests/performance/test_dashboard_api.py
+2 -2
View File
@@ -154,8 +154,8 @@ def test_dashboard_endpoints_filter_and_return_run_source_metadata():
assert summary["count"] == 1
assert summary["rows"][0]["run_source"] == "pr"
assert summary["rows"][0]["pr_number"] == "123"
assert summary["rows"][0]["baseline_n"] == 1
assert summary["rows"][0]["metrics"]["latency"]["baseline"] == 11.0
assert summary["rows"][0]["baseline_n"] == 0
assert summary["rows"][0]["metrics"]["latency"]["baseline"] is None
assert summary["filters"]["run_source"] == "pr"
assert trends["count"] == 1
assert trends["groups"][0]["points"][0]["run_source"] == "pr"
fastvideo/tests/performance/test_dashboard_service.py
+95 -2
View File
@@ -142,6 +142,47 @@ def test_build_latest_summary_separates_informational_threshold_crossing():
assert rows[0]["threshold_exceeded_metrics"] == ["latency"]
assert rows[0]["failing_metrics"] == []
assert rows[0]["computed_regression_status"] == "pass"
def test_build_latest_summary_run_source_filter_excludes_future_baselines():
records = [
_record(
"2026-01-01T00:00:00+00:00",
"a" * 40,
10.0,
10.0,
run_source="scheduled_main",
baseline_eligible=True,
),
_record(
"2026-01-02T00:00:00+00:00",
"b" * 40,
11.0,
9.0,
run_source="pr",
baseline_eligible=False,
pr_number="123",
),
_record(
"2026-01-03T00:00:00+00:00",
"c" * 40,
30.0,
3.0,
run_source="scheduled_main",
baseline_eligible=True,
),
]
rows = build_latest_summary(records, run_source="pr")
assert len(rows) == 1
assert rows[0]["run_source"] == "pr"
assert rows[0]["baseline_n"] == 1
assert rows[0]["metrics"]["latency"]["baseline"] == 10.0
assert rows[0]["metrics"]["throughput"]["baseline"] == 10.0
def test_build_latest_summary_keeps_identity_cohorts_separate():
records = [
_record(
@@ -186,7 +227,7 @@ def test_build_latest_summary_keeps_identity_cohorts_separate():
),
]
rows = build_latest_summary(records, max_regression=0.05)
rows = build_latest_summary(records)
recipe_b_row = next(row for row in rows if row["recipe_fingerprint"] == "recipe-b")
assert len(rows) == 2
@@ -241,7 +282,7 @@ def test_build_latest_summary_keeps_variant_versions_separate():
),
]
rows = build_latest_summary(records, max_regression=0.05)
rows = build_latest_summary(records)
version_2_row = next(row for row in rows if row["benchmark_version"] == 2)
assert len(rows) == 2
@@ -249,6 +290,58 @@ def test_build_latest_summary_keeps_variant_versions_separate():
assert version_2_row["metrics"]["latency"]["baseline"] == 20.0
def test_dashboard_identity_preserves_zero_benchmark_version():
records = [
_record(
"2026-01-01T00:00:00+00:00",
"a" * 40,
10.0,
10.0,
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=0,
recipe_fingerprint="recipe-a",
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
run_source="scheduled_main",
baseline_eligible=True,
),
_record(
"2026-01-02T00:00:00+00:00",
"b" * 40,
11.0,
9.0,
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=0,
recipe_fingerprint="recipe-a",
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
),
_record(
"2026-01-03T00:00:00+00:00",
"c" * 40,
20.0,
5.0,
),
]
rows = build_latest_summary(records)
version_zero_row = next(row for row in rows if row["benchmark_version"] == 0)
legacy_row = next(row for row in rows if row["benchmark_version"] == "")
trends = build_trends(records)
version_zero_trend = next(trend for trend in trends if trend["benchmark_version"] == 0)
legacy_trend = next(trend for trend in trends if trend["benchmark_version"] == "")
assert len(rows) == 2
assert version_zero_row["baseline_n"] == 1
assert version_zero_row["metrics"]["latency"]["baseline"] == 10.0
assert legacy_row["baseline_n"] == 0
assert len(trends) == 2
assert version_zero_trend["points"][0]["benchmark_version"] == 0
assert legacy_trend["points"][0]["benchmark_version"] == ""
def test_filter_records_and_trends_preserve_metric_points():
records = [
_record("2026-01-01T00:00:00+00:00", "a" * 40, 10.0, 10.0),