nodes/hao-ai-lab-FastVideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[ci]: complete performance cohort identity

Browse Source
This commit is contained in:
Mac Lee
2026-07-05 14:39:57 -07:00
committed by SolitaryThinker
parent 752248e466
commit 7b74b94ad4
14 changed files with 450 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.buildkite/performance-benchmarks/tests/wan-t2v-1.3b.json
+3 -3
View File
@@ -1,9 +1,9 @@
{
"benchmark_id": "wan-t2v-1.3b-2gpu",
"config_schema_version": 2,
"workload_id": "wan-t2v-1.3b",
"variant_id": "canonical",
"benchmark_version": 1,
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"description": "Wan2.1 T2V 1.3B inference performance",
"model": {
"model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
apps/performance_dashboard/frontend/src/App.tsx
+72 -5
View File
@@ -1,6 +1,7 @@
import { useEffect, useMemo, useState } from "react";
import { fetchSummary, fetchTrends, refreshData, RunSource, SummaryResponse, TrendGroup, TrendPoint } from "./api";
import { fetchSummary, fetchTrends, refreshData } from "./api";
import type { CohortValue, RunSource, SummaryResponse, TrendGroup, TrendPoint } from "./api";
const METRIC_KEYS = ["latency", "throughput", "memory", "text_encoder_time_s", "dit_time_s", "vae_decode_time_s"];
const RUN_SOURCES: Array<{ value: "" | RunSource; label: string }> = [
@@ -109,6 +110,61 @@ function metricLabel(metricKey: string) {
return METRIC_DEFINITIONS[metricKey]?.label ?? metricKey;
}
type CohortFields = {
model_id: string;
gpu_type: string;
workload_id: CohortValue;
variant_id: CohortValue;
benchmark_version: CohortValue;
recipe_fingerprint: CohortValue;
hardware_profile_id: CohortValue;
software_profile_id: CohortValue;
};
function cohortValue(value: CohortValue) {
if (value === null || value === undefined || value === "") {
return "legacy";
}
return String(value);
}
function shortCohortValue(value: CohortValue) {
const text = cohortValue(value);
if (text === "legacy" || text.length <= 14) {
return text;
}
return text.slice(0, 12);
}
function cohortKey(cohort: CohortFields) {
return [
cohort.model_id,
cohort.gpu_type,
cohortValue(cohort.workload_id),
cohortValue(cohort.variant_id),
cohortValue(cohort.benchmark_version),
cohortValue(cohort.recipe_fingerprint),
cohortValue(cohort.hardware_profile_id),
cohortValue(cohort.software_profile_id)
].join("|");
}
function cohortTitle(cohort: CohortFields) {
const workload = cohortValue(cohort.workload_id);
const variant = cohortValue(cohort.variant_id);
const version = cohortValue(cohort.benchmark_version);
const versionLabel = version === "legacy" ? version : `v${version}`;
return `${workload} / ${variant} / ${versionLabel}`;
}
function cohortDetail(cohort: CohortFields) {
return [
`recipe ${shortCohortValue(cohort.recipe_fingerprint)}`,
shortCohortValue(cohort.hardware_profile_id),
shortCohortValue(cohort.software_profile_id)
].join(" | ");
}
function formatMetricValue(metricKey: string, value: number | null | undefined, tooltip = false) {
const definition = METRIC_DEFINITIONS[metricKey];
if (!definition) {
@@ -171,7 +227,9 @@ function TrendChart({ group, metricKey }: { group: TrendGroup; metricKey: string
top: `${(activePoint.y / height) * 100}%`
}
: undefined;
const ariaLabel = `${metricLabel(metricKey)} trend for ${group.model_id} on ${group.gpu_type}`;
const ariaLabel = `${metricLabel(metricKey)} trend for ${group.model_id} on ${group.gpu_type}, ${cohortTitle(
group
)}`;
return (
<div className="chart-shell">
@@ -419,7 +477,7 @@ export default function App() {
<section className="panel">
<div className="panel-header">
<h2>Latest Status</h2>
<span>{latestRows.length} model/GPU groups</span>
<span>{latestRows.length} comparison cohorts</span>
</div>
{latestRows.length === 0 ? (
<div className="empty">No records match the selected filters.</div>
@@ -432,6 +490,7 @@ export default function App() {
<th>Recomputed</th>
<th>Model</th>
<th>GPU</th>
<th>Cohort</th>
<th>Commit</th>
<th>Source</th>
<th>Baseline</th>
@@ -446,7 +505,7 @@ export default function App() {
</thead>
<tbody>
{latestRows.map((row) => (
<tr key={`${row.model_id}-${row.gpu_type}`}>
<tr key={cohortKey(row)}>
<td>
<span className={`badge ${row.status}`}>{row.status}</span>
</td>
@@ -457,6 +516,12 @@ export default function App() {
</td>
<td>{row.model_id}</td>
<td>{row.gpu_type}</td>
<td>
<div className="cohort-cell">
<strong>{cohortTitle(row)}</strong>
<span>{cohortDetail(row)}</span>
</div>
</td>
<td>{shortSha(row.commit_sha)}</td>
<td>
<span className={`source-badge source-${row.run_source}`}>{runSourceLabel(row.run_source)}</span>
@@ -495,11 +560,13 @@ export default function App() {
) : (
trends.map((group) =>
METRIC_KEYS.map((metricKey) => (
<article className="trend-card" key={`${group.model_id}-${group.gpu_type}-${metricKey}`}>
<article className="trend-card" key={`${cohortKey(group)}-${metricKey}`}>
<div>
<h3>{metricLabel(metricKey)}</h3>
<p>
{group.model_id} | {group.gpu_type}
<span>{cohortTitle(group)}</span>
<span>{cohortDetail(group)}</span>
</p>
</div>
<TrendChart group={group} metricKey={metricKey} />
apps/performance_dashboard/frontend/src/api.ts
+14 -3
View File
@@ -13,6 +13,17 @@ export type MetricValue = {
precision: number;
};
export type CohortValue = string | number | null;
export type ComparisonCohort = {
workload_id: CohortValue;
variant_id: CohortValue;
benchmark_version: CohortValue;
recipe_fingerprint: CohortValue;
hardware_profile_id: CohortValue;
software_profile_id: CohortValue;
};
export type SummaryRow = {
model_id: string;
gpu_type: string;
@@ -34,7 +45,7 @@ export type SummaryRow = {
build_id: string;
job_id: string;
metrics: Record<string, MetricValue>;
};
} & ComparisonCohort;
export type RunSource = "pr" | "local" | "scheduled_main" | "unknown";
@@ -68,13 +79,13 @@ export type TrendPoint = {
build_id: string;
job_id: string;
metrics: Record<string, number | null>;
};
} & ComparisonCohort;
export type TrendGroup = {
model_id: string;
gpu_type: string;
points: TrendPoint[];
};
} & ComparisonCohort;
export type TrendsResponse = {
groups: TrendGroup[];
apps/performance_dashboard/frontend/src/styles.css
+25 -1
View File
@@ -149,6 +149,11 @@ h3 {
font-size: 0.82rem;
}
.trend-card p {
display: grid;
gap: 2px;
}
.stat strong {
display: block;
margin-top: 8px;
@@ -186,7 +191,7 @@ h3 {
table {
width: 100%;
min-width: 1120px;
min-width: 1260px;
border-collapse: collapse;
}
@@ -209,6 +214,25 @@ td {
font-size: 0.9rem;
}
.cohort-cell {
display: grid;
gap: 2px;
}
.cohort-cell strong,
.trend-card p span {
color: #1b2836;
font-size: 0.78rem;
font-weight: 700;
}
.cohort-cell span,
.trend-card p span + span {
color: #607080;
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace;
font-size: 0.72rem;
}
.badge {
display: inline-flex;
align-items: center;
docs/contributing/performance_benchmarks.md
+11 -5
View File
@@ -273,7 +273,12 @@ Written by `test_inference_performance.py`. One file per benchmark run.
"vae_decode_time_s": 3.208,
"recipe": {
"recipe_schema_version": 1,
"benchmark": { "benchmark_id": "wan-t2v-1.3b-2gpu" },
"benchmark": {
"benchmark_id": "wan-t2v-1.3b-2gpu",
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2
},
"model": { "model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" },
"init_kwargs": { "num_gpus": 2, "sp_size": 2, "tp_size": 1 },
"generation_kwargs": { "height": 480, "width": 832, "num_frames": 45 },
@@ -292,7 +297,7 @@ Written by `test_inference_performance.py`. One file per benchmark run.
"python": "3.12",
"pytorch": "2.12",
"cuda": "13.0",
"packages": { "fastvideo_kernel": "0.3", "triton": "3.4" }
"packages": { "fastvideo_kernel": "0.3.2", "triton": "3.4.1" }
},
"software_profile_id": "sw-<sha256-prefix>",
"environment_metadata": { "env": { "IMAGE_VERSION": "py3.12-cuda13.0.0" } },
@@ -344,7 +349,8 @@ Records missing both `run_source` and `baseline_eligible` are treated as legacy
successful main/full-suite uploads and remain eligible for rolling baselines.
New records compare only against the same `model_id`, `gpu_type`,
`recipe_fingerprint`, `hardware_profile_id`, and `software_profile_id` cohort.
`workload_id`, `variant_id`, `benchmark_version`, `recipe_fingerprint`,
`hardware_profile_id`, and `software_profile_id` cohort.
`environment_metadata` and `environment_fingerprint` are audit data and are not
part of the comparison key.
The recipe prompt digests describe the prompts actually measured by the
@@ -404,8 +410,8 @@ When the rolling-baseline phase runs, it emits:
{
"benchmark_id": "<unique-id>",
"config_schema_version": 2,
"workload_id": "<stable-workload-id>",
"variant_id": "canonical",
"workload_id": "<workload-family>",
"variant_id": "<model-or-runtime-variant>",
"benchmark_version": 1,
"model": { "model_path": "...", "model_short_name": "..." },
"init_kwargs": { "num_gpus": 1, ... },
fastvideo/performance/hf_store.py
+10 -1
View File
@@ -268,6 +268,9 @@ def load_records_for_model(
model_id: str,
gpu_type: str | None = None,
*,
workload_id: str | None = None,
variant_id: str | None = None,
benchmark_version: str | None = None,
recipe_fingerprint: str | None = None,
hardware_profile_id: str | None = None,
software_profile_id: str | None = None,
@@ -281,6 +284,9 @@ def load_records_for_model(
local_dir: Root directory previously populated by :func:`sync_from_hf`.
model_id: Matches the ``model_id`` field inside each JSON record.
gpu_type: When set, only records whose ``gpu_type`` matches are returned.
workload_id: When set, only records from the same workload are returned.
variant_id: When set, only records from the same workload variant are returned.
benchmark_version: When set, only records from the same benchmark version are returned.
recipe_fingerprint: When set, only records from the same benchmark
recipe are returned.
hardware_profile_id: When set, only records from the same hardware
@@ -309,13 +315,16 @@ def load_records_for_model(
records = [r for r in records if r.get("gpu_type") == gpu_type]
identity_filters = {
"workload_id": workload_id,
"variant_id": variant_id,
"benchmark_version": benchmark_version,
"recipe_fingerprint": recipe_fingerprint,
"hardware_profile_id": hardware_profile_id,
"software_profile_id": software_profile_id,
}
for key, expected in identity_filters.items():
if expected is not None:
records = [r for r in records if r.get(key) == expected]
records = [r for r in records if str(r.get(key)) == str(expected)]
if last_n is not None:
records = records[-last_n:]
fastvideo/performance_dashboard/service.py
+40 -16
View File
@@ -17,7 +17,11 @@ from fastvideo.performance.hf_store import is_baseline_eligible_record, safe_flo
from fastvideo.performance.metric_policy import regression_delta, resolve_metric_policies
Record = dict[str, Any]
CohortKey = tuple[str, str, str, str, str, str, str, str]
COMPARISON_COHORT_KEYS = (
"workload_id",
"variant_id",
"benchmark_version",
"recipe_fingerprint",
"hardware_profile_id",
"software_profile_id",
@@ -86,18 +90,40 @@ def record_comparison_metadata(record: Record) -> Record:
return {key: record.get(key) or "" for key in COMPARISON_COHORT_KEYS}
def group_by_comparison_cohort(records: list[Record]) -> dict[tuple[str, str, str, str, str], list[Record]]:
groups: dict[tuple[str, str, str, str, str], list[Record]] = defaultdict(list)
def comparison_cohort_key(record: Record) -> CohortKey:
return (
str(record.get("model_id") or "unknown"),
str(record.get("gpu_type") or "unknown"),
str(record.get("workload_id") or ""),
str(record.get("variant_id") or ""),
str(record.get("benchmark_version") or ""),
str(record.get("recipe_fingerprint") or ""),
str(record.get("hardware_profile_id") or ""),
str(record.get("software_profile_id") or ""),
)
def group_by_comparison_cohort(records: list[Record]) -> dict[CohortKey, list[Record]]:
groups: dict[CohortKey, list[Record]] = defaultdict(list)
for record in records:
model_id = str(record.get("model_id") or "unknown")
gpu_type = str(record.get("gpu_type") or "unknown")
recipe_fingerprint = str(record.get("recipe_fingerprint") or "")
hardware_profile = str(record.get("hardware_profile_id") or "")
software_profile = str(record.get("software_profile_id") or "")
groups[(model_id, gpu_type, recipe_fingerprint, hardware_profile, software_profile)].append(record)
groups[comparison_cohort_key(record)].append(record)
return {key: sorted(value, key=record_sort_key) for key, value in groups.items()}
def comparison_sort_key(record: Record) -> CohortKey:
return comparison_cohort_key(record)
def latest_row_sort_key(row: Record) -> tuple[Any, ...]:
return (row["status"] != "fail", *comparison_sort_key(row))
def group_identity(key: CohortKey) -> tuple[str, str]:
model_id = key[0]
gpu_type = key[1]
return model_id, gpu_type
def baseline_value(records: list[Record], metric_key: str) -> float | None:
values = [safe_float(record.get(metric_key)) for record in records]
values = [value for value in values if value is not None]
@@ -111,7 +137,8 @@ def build_latest_summary(records: list[Record],
baseline_window: int = 5,
run_source: str | None = None) -> list[Record]:
rows: list[Record] = []
for (model_id, gpu_type, _recipe, _hardware, _software), group in group_by_comparison_cohort(records).items():
for key, group in group_by_comparison_cohort(records).items():
model_id, gpu_type = group_identity(key)
latest_candidates = group
if run_source:
latest_candidates = [record for record in group if record_run_source(record) == run_source]
@@ -179,14 +206,13 @@ def build_latest_summary(records: list[Record],
"metrics": metrics,
})
return sorted(rows,
key=lambda row: (row["status"] != "fail", row["model_id"], row["gpu_type"], row["recipe_fingerprint"],
row["hardware_profile_id"], row["software_profile_id"]))
return sorted(rows, key=latest_row_sort_key)
def build_trends(records: list[Record]) -> list[Record]:
trends: list[Record] = []
for (model_id, gpu_type, _recipe, _hardware, _software), group in group_by_comparison_cohort(records).items():
for key, group in group_by_comparison_cohort(records).items():
model_id, gpu_type = group_identity(key)
points = []
for record in group:
metric_policies = resolve_metric_policies(record.get("regression_thresholds"))
@@ -208,6 +234,4 @@ def build_trends(records: list[Record]) -> list[Record]:
**record_comparison_metadata(group[-1]),
"points": points,
})
return sorted(trends,
key=lambda trend: (trend["model_id"], trend["gpu_type"], trend["recipe_fingerprint"], trend[
"hardware_profile_id"], trend["software_profile_id"]))
return sorted(trends, key=comparison_sort_key)
fastvideo/tests/performance/compare_baseline.py
+5 -2
View File
@@ -5,8 +5,8 @@ This script:
1) reads current benchmark results from fastvideo/tests/performance/results,
2) syncs the canonical baseline from the configured HF dataset repo,
3) compares each current record against the median of up to 5 prior
baseline-eligible successful records in the same GPU/recipe/hardware/software
cohort,
baseline-eligible successful records in the same workload/variant/version,
GPU, recipe, hardware, and software cohort,
4) writes normalized records back to the HF dataset repo according to
PERF_UPLOAD_POLICY,
5) exits non-zero if any gated metric exceeds both its percent and absolute
@@ -79,6 +79,9 @@ IDENTITY_KEYS = (
"environment_fingerprint",
)
COMPARISON_IDENTITY_KEYS = (
"workload_id",
"variant_id",
"benchmark_version",
"recipe_fingerprint",
"hardware_profile_id",
"software_profile_id",
fastvideo/tests/performance/identity.py
+14 -3
View File
@@ -186,7 +186,11 @@ def hardware_profile(
gpu_count = int(num_gpus if num_gpus is not None else torch.cuda.device_count())
devices = []
cuda_device_count = torch.cuda.device_count()
for device_id in range(gpu_count):
if device_id >= cuda_device_count:
devices.append(_normalize_gpu_device({}))
continue
props = torch.cuda.get_device_properties(device_id)
capability = torch.cuda.get_device_capability(device_id)
devices.append(
@@ -211,7 +215,11 @@ def software_profile(
cuda_version: str | None = None,
package_versions: Mapping[str, str | None] | None = None,
) -> dict[str, Any]:
"""Return normalized software cohort metadata using major/minor versions."""
"""Return normalized software cohort metadata.
Python, PyTorch, and CUDA use major/minor cohorts. Attention and kernel
packages keep exact versions because patch releases can change performance.
"""
versions = dict(package_versions) if package_versions is not None else _installed_package_versions()
return {
@@ -219,7 +227,7 @@ def software_profile(
"pytorch": _major_minor(torch_version or torch.__version__),
"cuda": _major_minor(cuda_version or torch.version.cuda),
"packages": {
name: _major_minor(version)
name: str(version)
for name, version in sorted(versions.items())
if version
},
@@ -276,6 +284,8 @@ def _canonicalize(value: Any) -> Any:
str(key): _canonicalize(value[key])
for key in sorted(value, key=lambda item: str(item))
}
if isinstance(value, (set, frozenset)):
return [_canonicalize(item) for item in sorted(value, key=lambda item: str(item))]
if isinstance(value, tuple):
return [_canonicalize(item) for item in value]
if isinstance(value, list):
@@ -311,7 +321,8 @@ def _looks_like_local_path(value: str) -> bool:
if value.startswith(("/", "./", "../", "~")):
return True
looks_like_hf_repo = re.match(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$", value)
return os.sep in value and looks_like_hf_repo is None
has_separator = "/" in value or "\\" in value or os.sep in value
return has_separator and looks_like_hf_repo is None
def _normalize_gpu_device(device: Mapping[str, Any]) -> dict[str, Any]:
fastvideo/tests/performance/test_compare_baseline_policy.py
+20
View File
@@ -179,6 +179,26 @@ def test_normalized_record_reads_identity_labels_from_recipe(monkeypatch):
assert record["benchmark_version"] == 2
def test_comparison_identity_filters_use_full_issue_key():
record = {
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"recipe_fingerprint": "recipe-1",
"hardware_profile_id": "hw-1",
"software_profile_id": "sw-1",
}
assert compare_baseline._comparison_identity_filters(record) == {
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": "2",
"recipe_fingerprint": "recipe-1",
"hardware_profile_id": "hw-1",
"software_profile_id": "sw-1",
}
def test_baseline_eligibility_only_for_successful_scheduled_main():
assert compare_baseline._is_baseline_eligible("scheduled_main", True) is True
assert compare_baseline._is_baseline_eligible("scheduled_main", False) is False
fastvideo/tests/performance/test_dashboard_api.py
+32 -2
View File
@@ -56,8 +56,34 @@ def _record(model_id, gpu_type, ts, commit, latency, throughput, success=True, *
def test_summary_endpoint_returns_latest_group_status():
app = create_app(FakeStore([
_record("wan", "NVIDIA L40S", "2026-01-01T00:00:00+00:00", "a" * 40, 10.0, 10.0),
_record("wan", "NVIDIA L40S", "2026-01-02T00:00:00+00:00", "b" * 40, 11.0, 9.0),
_record(
"wan",
"NVIDIA L40S",
"2026-01-01T00:00:00+00:00",
"a" * 40,
10.0,
10.0,
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=2,
recipe_fingerprint="recipe-a",
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
),
_record(
"wan",
"NVIDIA L40S",
"2026-01-02T00:00:00+00:00",
"b" * 40,
11.0,
9.0,
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=2,
recipe_fingerprint="recipe-a",
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
),
]))
client = TestClient(app)
@@ -71,6 +97,10 @@ def test_summary_endpoint_returns_latest_group_status():
assert body["rows"][0]["metrics"]["latency"]["threshold_exceeded"] is True
assert body["rows"][0]["threshold_exceeded_metrics"] == ["latency", "throughput"]
assert body["rows"][0]["computed_regression_status"] == "fail"
assert body["rows"][0]["workload_id"] == "wan-t2v"
assert body["rows"][0]["variant_id"] == "1.3b-sp2"
assert body["rows"][0]["benchmark_version"] == 2
assert body["rows"][0]["recipe_fingerprint"] == "recipe-a"
def test_summary_status_is_independent_of_days_window():
fastvideo/tests/performance/test_dashboard_service.py
+91
View File
@@ -150,6 +150,9 @@ def test_build_latest_summary_keeps_identity_cohorts_separate():
10.0,
10.0,
recipe_fingerprint="recipe-a",
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=2,
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
run_source="scheduled_main",
@@ -161,6 +164,9 @@ def test_build_latest_summary_keeps_identity_cohorts_separate():
20.0,
5.0,
recipe_fingerprint="recipe-b",
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=2,
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
run_source="scheduled_main",
@@ -172,6 +178,9 @@ def test_build_latest_summary_keeps_identity_cohorts_separate():
22.0,
4.5,
recipe_fingerprint="recipe-b",
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=2,
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
),
@@ -183,6 +192,61 @@ def test_build_latest_summary_keeps_identity_cohorts_separate():
assert len(rows) == 2
assert recipe_b_row["baseline_n"] == 1
assert recipe_b_row["metrics"]["latency"]["baseline"] == 20.0
assert recipe_b_row["workload_id"] == "wan-t2v"
assert recipe_b_row["variant_id"] == "1.3b-sp2"
assert recipe_b_row["benchmark_version"] == 2
def test_build_latest_summary_keeps_variant_versions_separate():
records = [
_record(
"2026-01-01T00:00:00+00:00",
"a" * 40,
10.0,
10.0,
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=1,
recipe_fingerprint="recipe-a",
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
run_source="scheduled_main",
baseline_eligible=True,
),
_record(
"2026-01-02T00:00:00+00:00",
"b" * 40,
20.0,
5.0,
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=2,
recipe_fingerprint="recipe-a",
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
run_source="scheduled_main",
baseline_eligible=True,
),
_record(
"2026-01-03T00:00:00+00:00",
"c" * 40,
22.0,
4.5,
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version=2,
recipe_fingerprint="recipe-a",
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
),
]
rows = build_latest_summary(records, max_regression=0.05)
version_2_row = next(row for row in rows if row["benchmark_version"] == 2)
assert len(rows) == 2
assert version_2_row["baseline_n"] == 1
assert version_2_row["metrics"]["latency"]["baseline"] == 20.0
def test_filter_records_and_trends_preserve_metric_points():
@@ -272,6 +336,9 @@ def test_load_records_for_model_filters_identity_cohort(tmp_path):
"timestamp": "2026-01-01T00:00:00+00:00",
"success": true,
"baseline_eligible": true,
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"recipe_fingerprint": "recipe-a",
"hardware_profile_id": "hw-l40s",
"software_profile_id": "sw-cu130"
@@ -287,6 +354,9 @@ def test_load_records_for_model_filters_identity_cohort(tmp_path):
"timestamp": "2026-01-02T00:00:00+00:00",
"success": true,
"baseline_eligible": true,
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"recipe_fingerprint": "recipe-b",
"hardware_profile_id": "hw-l40s",
"software_profile_id": "sw-cu130"
@@ -294,11 +364,32 @@ def test_load_records_for_model_filters_identity_cohort(tmp_path):
""",
encoding="utf-8",
)
(model_dir / "other_version.json").write_text(
"""
{
"model_id": "wan",
"gpu_type": "NVIDIA L40S",
"timestamp": "2026-01-03T00:00:00+00:00",
"success": true,
"baseline_eligible": true,
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 3,
"recipe_fingerprint": "recipe-a",
"hardware_profile_id": "hw-l40s",
"software_profile_id": "sw-cu130"
}
""",
encoding="utf-8",
)
records = hf_store.load_records_for_model(
str(tmp_path),
"wan",
"NVIDIA L40S",
workload_id="wan-t2v",
variant_id="1.3b-sp2",
benchmark_version="2",
recipe_fingerprint="recipe-a",
hardware_profile_id="hw-l40s",
software_profile_id="sw-cu130",
fastvideo/tests/performance/test_identity.py
+97 -4
View File
@@ -2,6 +2,9 @@
from copy import deepcopy
import torch
from fastvideo.tests.performance import identity as identity_module
from fastvideo.tests.performance.identity import (
build_recipe_from_benchmark_config,
canonical_json,
@@ -19,6 +22,9 @@ from fastvideo.tests.performance.identity import (
def _benchmark_config():
return {
"benchmark_id": "wan-t2v-1.3b-2gpu",
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
"model": {
"model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
"model_short_name": "Wan2.1-T2V-1.3B",
@@ -58,12 +64,28 @@ def test_canonical_json_is_stable_for_mapping_order():
assert canonical_json(left) == canonical_json(right)
def test_canonical_json_handles_sets_deterministically():
assert canonical_json({"values": {"b", "a"}}) == '{"values":["a","b"]}'
assert canonical_json({"values": frozenset(("b", "a"))}) == '{"values":["a","b"]}'
def test_same_config_produces_same_recipe_fingerprint():
cfg = _benchmark_config()
assert _fingerprint(cfg) == _fingerprint(deepcopy(cfg))
def test_recipe_includes_first_class_benchmark_identity():
recipe = build_recipe_from_benchmark_config(_benchmark_config())
assert recipe["benchmark"] == {
"benchmark_id": "wan-t2v-1.3b-2gpu",
"workload_id": "wan-t2v",
"variant_id": "1.3b-sp2",
"benchmark_version": 2,
}
def test_semantically_equivalent_sequence_forms_match():
cfg = _benchmark_config()
equivalent = deepcopy(cfg)
@@ -103,6 +125,14 @@ def test_num_inference_steps_changes_recipe_fingerprint():
assert _fingerprint(cfg) != _fingerprint(changed)
def test_benchmark_version_changes_recipe_fingerprint():
cfg = _benchmark_config()
changed = deepcopy(cfg)
changed["benchmark_version"] = 3
assert _fingerprint(cfg) != _fingerprint(changed)
def test_attention_backend_changes_recipe_fingerprint():
cfg = _benchmark_config()
@@ -138,7 +168,7 @@ def test_distributed_layout_changes_recipe_fingerprint():
assert _fingerprint(cfg) != _fingerprint(changed)
def test_software_profile_uses_major_minor_versions_for_id():
def test_software_profile_uses_exact_attention_kernel_package_versions_for_id():
profile_a = software_profile(
python_version="3.12.4",
torch_version="2.12.0+cu130",
@@ -163,11 +193,14 @@ def test_software_profile_uses_major_minor_versions_for_id():
"pytorch": "2.12",
"cuda": "13.0",
"packages": {
"fastvideo_kernel": "0.3",
"triton": "3.4",
"fastvideo_kernel": "0.3.2",
"triton": "3.4.1",
},
}
assert software_profile_id(profile_a) == software_profile_id(profile_b)
assert profile_b["python"] == "3.12"
assert profile_b["pytorch"] == "2.12"
assert profile_b["cuda"] == "13.0"
assert software_profile_id(profile_a) != software_profile_id(profile_b)
def test_hardware_profile_id_uses_normalized_gpu_cohort():
@@ -208,6 +241,41 @@ def test_hardware_profile_id_uses_normalized_gpu_cohort():
assert hardware_profile_id(profile).startswith("hw-")
def test_hardware_profile_pads_missing_requested_cuda_devices(monkeypatch):
class Props:
name = "NVIDIA L40S"
total_memory = 48 * 1024**3
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
monkeypatch.setattr(torch.cuda, "get_device_properties", lambda device_id: Props())
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device_id: (8, 9))
profile = hardware_profile(num_gpus=2, interconnect="unknown")
assert profile["gpu_count"] == 2
assert profile["gpus"] == [
{
"name": "NVIDIA L40S",
"memory_gb": 48,
"compute_capability": "8.9",
},
{
"name": "unknown",
"memory_gb": None,
"compute_capability": None,
},
]
def test_local_path_detection_checks_both_path_separators(monkeypatch):
monkeypatch.setattr(identity_module.os, "sep", "\\")
assert identity_module._looks_like_local_path("models/local/checkpoint") is True
assert identity_module._looks_like_local_path(r"C:\models\checkpoint") is True
assert identity_module._looks_like_local_path("Wan-AI/Wan2.1-T2V-1.3B-Diffusers") is False
def test_environment_metadata_is_separate_audit_fingerprint():
cfg = _benchmark_config()
recipe_hash = _fingerprint(cfg)
@@ -261,3 +329,28 @@ def test_runtime_identity_from_generator_summarizes_worker_records():
"resolved_attention_backend": "FLASH_ATTN",
"resolved_model_revision": revision,
}
def test_collect_worker_identity_handles_torch_module_pipeline():
from fastvideo.tests.performance.test_inference_performance import _collect_worker_identity
class Backend:
name = "FLASH_ATTN"
class Leaf(torch.nn.Module):
backend = Backend()
class Pipeline(torch.nn.Module):
model_path = "/models/local"
def __init__(self):
super().__init__()
self.leaf = Leaf()
class Worker:
pipeline = Pipeline()
assert _collect_worker_identity(Worker()) == {
"resolved_attention_backends": ["FLASH_ATTN"],
"resolved_model_path": "/models/local",
}
fastvideo/tests/performance/test_inference_performance.py
+16 -6
View File
@@ -252,14 +252,24 @@ def _backend_name(value) -> str:
def _collect_worker_identity(worker) -> dict[str, Any]:
pipeline = getattr(worker, "pipeline", None)
model_path = getattr(pipeline, "model_path", None)
modules = getattr(pipeline, "modules", {}) or {}
backends: set[str] = set()
for module in modules.values():
if not isinstance(module, torch.nn.Module):
continue
for submodule in module.modules():
backend = getattr(submodule, "backend", None)
modules = getattr(pipeline, "modules", None)
if isinstance(modules, Mapping):
module_iter = modules.values()
elif isinstance(pipeline, torch.nn.Module):
module_iter = (pipeline,)
else:
module_iter = ()
for module in module_iter:
if isinstance(module, torch.nn.Module):
for submodule in module.modules():
backend = getattr(submodule, "backend", None)
if backend is not None:
backends.add(_backend_name(backend))
else:
backend = getattr(module, "backend", None)
if backend is not None:
backends.add(_backend_name(backend))