From 7b74b94ad4c1b7e4aedb8b72eee5e4aaa661442e Mon Sep 17 00:00:00 2001 From: Mac Lee Date: Sat, 4 Jul 2026 15:49:03 +0000 Subject: [PATCH] [ci]: complete performance cohort identity --- .../tests/wan-t2v-1.3b.json | 6 +- .../frontend/src/App.tsx | 77 ++++++++++++- .../performance_dashboard/frontend/src/api.ts | 17 ++- .../frontend/src/styles.css | 26 ++++- docs/contributing/performance_benchmarks.md | 16 ++- fastvideo/performance/hf_store.py | 11 +- fastvideo/performance_dashboard/service.py | 56 +++++++--- .../tests/performance/compare_baseline.py | 7 +- fastvideo/tests/performance/identity.py | 17 ++- .../test_compare_baseline_policy.py | 20 ++++ .../tests/performance/test_dashboard_api.py | 34 +++++- .../performance/test_dashboard_service.py | 91 ++++++++++++++++ fastvideo/tests/performance/test_identity.py | 101 +++++++++++++++++- .../performance/test_inference_performance.py | 22 ++-- 14 files changed, 450 insertions(+), 51 deletions(-) diff --git a/.buildkite/performance-benchmarks/tests/wan-t2v-1.3b.json b/.buildkite/performance-benchmarks/tests/wan-t2v-1.3b.json index 8a568ef6b..4af21b6ab 100644 --- a/.buildkite/performance-benchmarks/tests/wan-t2v-1.3b.json +++ b/.buildkite/performance-benchmarks/tests/wan-t2v-1.3b.json @@ -1,9 +1,9 @@ { "benchmark_id": "wan-t2v-1.3b-2gpu", "config_schema_version": 2, - "workload_id": "wan-t2v-1.3b", - "variant_id": "canonical", - "benchmark_version": 1, + "workload_id": "wan-t2v", + "variant_id": "1.3b-sp2", + "benchmark_version": 2, "description": "Wan2.1 T2V 1.3B inference performance", "model": { "model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", diff --git a/apps/performance_dashboard/frontend/src/App.tsx b/apps/performance_dashboard/frontend/src/App.tsx index 5c85fc643..47173d3ce 100644 --- a/apps/performance_dashboard/frontend/src/App.tsx +++ b/apps/performance_dashboard/frontend/src/App.tsx @@ -1,6 +1,7 @@ import { useEffect, useMemo, useState } from "react"; -import { fetchSummary, fetchTrends, refreshData, RunSource, SummaryResponse, TrendGroup, TrendPoint } from "./api"; +import { fetchSummary, fetchTrends, refreshData } from "./api"; +import type { CohortValue, RunSource, SummaryResponse, TrendGroup, TrendPoint } from "./api"; const METRIC_KEYS = ["latency", "throughput", "memory", "text_encoder_time_s", "dit_time_s", "vae_decode_time_s"]; const RUN_SOURCES: Array<{ value: "" | RunSource; label: string }> = [ @@ -109,6 +110,61 @@ function metricLabel(metricKey: string) { return METRIC_DEFINITIONS[metricKey]?.label ?? metricKey; } +type CohortFields = { + model_id: string; + gpu_type: string; + workload_id: CohortValue; + variant_id: CohortValue; + benchmark_version: CohortValue; + recipe_fingerprint: CohortValue; + hardware_profile_id: CohortValue; + software_profile_id: CohortValue; +}; + +function cohortValue(value: CohortValue) { + if (value === null || value === undefined || value === "") { + return "legacy"; + } + return String(value); +} + +function shortCohortValue(value: CohortValue) { + const text = cohortValue(value); + if (text === "legacy" || text.length <= 14) { + return text; + } + return text.slice(0, 12); +} + +function cohortKey(cohort: CohortFields) { + return [ + cohort.model_id, + cohort.gpu_type, + cohortValue(cohort.workload_id), + cohortValue(cohort.variant_id), + cohortValue(cohort.benchmark_version), + cohortValue(cohort.recipe_fingerprint), + cohortValue(cohort.hardware_profile_id), + cohortValue(cohort.software_profile_id) + ].join("|"); +} + +function cohortTitle(cohort: CohortFields) { + const workload = cohortValue(cohort.workload_id); + const variant = cohortValue(cohort.variant_id); + const version = cohortValue(cohort.benchmark_version); + const versionLabel = version === "legacy" ? version : `v${version}`; + return `${workload} / ${variant} / ${versionLabel}`; +} + +function cohortDetail(cohort: CohortFields) { + return [ + `recipe ${shortCohortValue(cohort.recipe_fingerprint)}`, + shortCohortValue(cohort.hardware_profile_id), + shortCohortValue(cohort.software_profile_id) + ].join(" | "); +} + function formatMetricValue(metricKey: string, value: number | null | undefined, tooltip = false) { const definition = METRIC_DEFINITIONS[metricKey]; if (!definition) { @@ -171,7 +227,9 @@ function TrendChart({ group, metricKey }: { group: TrendGroup; metricKey: string top: `${(activePoint.y / height) * 100}%` } : undefined; - const ariaLabel = `${metricLabel(metricKey)} trend for ${group.model_id} on ${group.gpu_type}`; + const ariaLabel = `${metricLabel(metricKey)} trend for ${group.model_id} on ${group.gpu_type}, ${cohortTitle( + group + )}`; return (
@@ -419,7 +477,7 @@ export default function App() {

Latest Status

- {latestRows.length} model/GPU groups + {latestRows.length} comparison cohorts
{latestRows.length === 0 ? (
No records match the selected filters.
@@ -432,6 +490,7 @@ export default function App() { Recomputed Model GPU + Cohort Commit Source Baseline @@ -446,7 +505,7 @@ export default function App() { {latestRows.map((row) => ( - + {row.status} @@ -457,6 +516,12 @@ export default function App() { {row.model_id} {row.gpu_type} + +
+ {cohortTitle(row)} + {cohortDetail(row)} +
+ {shortSha(row.commit_sha)} {runSourceLabel(row.run_source)} @@ -495,11 +560,13 @@ export default function App() { ) : ( trends.map((group) => METRIC_KEYS.map((metricKey) => ( -
+

{metricLabel(metricKey)}

{group.model_id} | {group.gpu_type} + {cohortTitle(group)} + {cohortDetail(group)}

diff --git a/apps/performance_dashboard/frontend/src/api.ts b/apps/performance_dashboard/frontend/src/api.ts index 2b00c6e37..8a474e160 100644 --- a/apps/performance_dashboard/frontend/src/api.ts +++ b/apps/performance_dashboard/frontend/src/api.ts @@ -13,6 +13,17 @@ export type MetricValue = { precision: number; }; +export type CohortValue = string | number | null; + +export type ComparisonCohort = { + workload_id: CohortValue; + variant_id: CohortValue; + benchmark_version: CohortValue; + recipe_fingerprint: CohortValue; + hardware_profile_id: CohortValue; + software_profile_id: CohortValue; +}; + export type SummaryRow = { model_id: string; gpu_type: string; @@ -34,7 +45,7 @@ export type SummaryRow = { build_id: string; job_id: string; metrics: Record; -}; +} & ComparisonCohort; export type RunSource = "pr" | "local" | "scheduled_main" | "unknown"; @@ -68,13 +79,13 @@ export type TrendPoint = { build_id: string; job_id: string; metrics: Record; -}; +} & ComparisonCohort; export type TrendGroup = { model_id: string; gpu_type: string; points: TrendPoint[]; -}; +} & ComparisonCohort; export type TrendsResponse = { groups: TrendGroup[]; diff --git a/apps/performance_dashboard/frontend/src/styles.css b/apps/performance_dashboard/frontend/src/styles.css index bd3ac0c5c..fcfcafc8f 100644 --- a/apps/performance_dashboard/frontend/src/styles.css +++ b/apps/performance_dashboard/frontend/src/styles.css @@ -149,6 +149,11 @@ h3 { font-size: 0.82rem; } +.trend-card p { + display: grid; + gap: 2px; +} + .stat strong { display: block; margin-top: 8px; @@ -186,7 +191,7 @@ h3 { table { width: 100%; - min-width: 1120px; + min-width: 1260px; border-collapse: collapse; } @@ -209,6 +214,25 @@ td { font-size: 0.9rem; } +.cohort-cell { + display: grid; + gap: 2px; +} + +.cohort-cell strong, +.trend-card p span { + color: #1b2836; + font-size: 0.78rem; + font-weight: 700; +} + +.cohort-cell span, +.trend-card p span + span { + color: #607080; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace; + font-size: 0.72rem; +} + .badge { display: inline-flex; align-items: center; diff --git a/docs/contributing/performance_benchmarks.md b/docs/contributing/performance_benchmarks.md index f5dacfdc5..fc74a2c69 100644 --- a/docs/contributing/performance_benchmarks.md +++ b/docs/contributing/performance_benchmarks.md @@ -273,7 +273,12 @@ Written by `test_inference_performance.py`. One file per benchmark run. "vae_decode_time_s": 3.208, "recipe": { "recipe_schema_version": 1, - "benchmark": { "benchmark_id": "wan-t2v-1.3b-2gpu" }, + "benchmark": { + "benchmark_id": "wan-t2v-1.3b-2gpu", + "workload_id": "wan-t2v", + "variant_id": "1.3b-sp2", + "benchmark_version": 2 + }, "model": { "model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" }, "init_kwargs": { "num_gpus": 2, "sp_size": 2, "tp_size": 1 }, "generation_kwargs": { "height": 480, "width": 832, "num_frames": 45 }, @@ -292,7 +297,7 @@ Written by `test_inference_performance.py`. One file per benchmark run. "python": "3.12", "pytorch": "2.12", "cuda": "13.0", - "packages": { "fastvideo_kernel": "0.3", "triton": "3.4" } + "packages": { "fastvideo_kernel": "0.3.2", "triton": "3.4.1" } }, "software_profile_id": "sw-", "environment_metadata": { "env": { "IMAGE_VERSION": "py3.12-cuda13.0.0" } }, @@ -344,7 +349,8 @@ Records missing both `run_source` and `baseline_eligible` are treated as legacy successful main/full-suite uploads and remain eligible for rolling baselines. New records compare only against the same `model_id`, `gpu_type`, -`recipe_fingerprint`, `hardware_profile_id`, and `software_profile_id` cohort. +`workload_id`, `variant_id`, `benchmark_version`, `recipe_fingerprint`, +`hardware_profile_id`, and `software_profile_id` cohort. `environment_metadata` and `environment_fingerprint` are audit data and are not part of the comparison key. The recipe prompt digests describe the prompts actually measured by the @@ -404,8 +410,8 @@ When the rolling-baseline phase runs, it emits: { "benchmark_id": "", "config_schema_version": 2, - "workload_id": "", - "variant_id": "canonical", + "workload_id": "", + "variant_id": "", "benchmark_version": 1, "model": { "model_path": "...", "model_short_name": "..." }, "init_kwargs": { "num_gpus": 1, ... }, diff --git a/fastvideo/performance/hf_store.py b/fastvideo/performance/hf_store.py index 31cb20bbf..1fe475cd5 100644 --- a/fastvideo/performance/hf_store.py +++ b/fastvideo/performance/hf_store.py @@ -268,6 +268,9 @@ def load_records_for_model( model_id: str, gpu_type: str | None = None, *, + workload_id: str | None = None, + variant_id: str | None = None, + benchmark_version: str | None = None, recipe_fingerprint: str | None = None, hardware_profile_id: str | None = None, software_profile_id: str | None = None, @@ -281,6 +284,9 @@ def load_records_for_model( local_dir: Root directory previously populated by :func:`sync_from_hf`. model_id: Matches the ``model_id`` field inside each JSON record. gpu_type: When set, only records whose ``gpu_type`` matches are returned. + workload_id: When set, only records from the same workload are returned. + variant_id: When set, only records from the same workload variant are returned. + benchmark_version: When set, only records from the same benchmark version are returned. recipe_fingerprint: When set, only records from the same benchmark recipe are returned. hardware_profile_id: When set, only records from the same hardware @@ -309,13 +315,16 @@ def load_records_for_model( records = [r for r in records if r.get("gpu_type") == gpu_type] identity_filters = { + "workload_id": workload_id, + "variant_id": variant_id, + "benchmark_version": benchmark_version, "recipe_fingerprint": recipe_fingerprint, "hardware_profile_id": hardware_profile_id, "software_profile_id": software_profile_id, } for key, expected in identity_filters.items(): if expected is not None: - records = [r for r in records if r.get(key) == expected] + records = [r for r in records if str(r.get(key)) == str(expected)] if last_n is not None: records = records[-last_n:] diff --git a/fastvideo/performance_dashboard/service.py b/fastvideo/performance_dashboard/service.py index 8b9bcdbc1..fb070d277 100644 --- a/fastvideo/performance_dashboard/service.py +++ b/fastvideo/performance_dashboard/service.py @@ -17,7 +17,11 @@ from fastvideo.performance.hf_store import is_baseline_eligible_record, safe_flo from fastvideo.performance.metric_policy import regression_delta, resolve_metric_policies Record = dict[str, Any] +CohortKey = tuple[str, str, str, str, str, str, str, str] COMPARISON_COHORT_KEYS = ( + "workload_id", + "variant_id", + "benchmark_version", "recipe_fingerprint", "hardware_profile_id", "software_profile_id", @@ -86,18 +90,40 @@ def record_comparison_metadata(record: Record) -> Record: return {key: record.get(key) or "" for key in COMPARISON_COHORT_KEYS} -def group_by_comparison_cohort(records: list[Record]) -> dict[tuple[str, str, str, str, str], list[Record]]: - groups: dict[tuple[str, str, str, str, str], list[Record]] = defaultdict(list) +def comparison_cohort_key(record: Record) -> CohortKey: + return ( + str(record.get("model_id") or "unknown"), + str(record.get("gpu_type") or "unknown"), + str(record.get("workload_id") or ""), + str(record.get("variant_id") or ""), + str(record.get("benchmark_version") or ""), + str(record.get("recipe_fingerprint") or ""), + str(record.get("hardware_profile_id") or ""), + str(record.get("software_profile_id") or ""), + ) + + +def group_by_comparison_cohort(records: list[Record]) -> dict[CohortKey, list[Record]]: + groups: dict[CohortKey, list[Record]] = defaultdict(list) for record in records: - model_id = str(record.get("model_id") or "unknown") - gpu_type = str(record.get("gpu_type") or "unknown") - recipe_fingerprint = str(record.get("recipe_fingerprint") or "") - hardware_profile = str(record.get("hardware_profile_id") or "") - software_profile = str(record.get("software_profile_id") or "") - groups[(model_id, gpu_type, recipe_fingerprint, hardware_profile, software_profile)].append(record) + groups[comparison_cohort_key(record)].append(record) return {key: sorted(value, key=record_sort_key) for key, value in groups.items()} +def comparison_sort_key(record: Record) -> CohortKey: + return comparison_cohort_key(record) + + +def latest_row_sort_key(row: Record) -> tuple[Any, ...]: + return (row["status"] != "fail", *comparison_sort_key(row)) + + +def group_identity(key: CohortKey) -> tuple[str, str]: + model_id = key[0] + gpu_type = key[1] + return model_id, gpu_type + + def baseline_value(records: list[Record], metric_key: str) -> float | None: values = [safe_float(record.get(metric_key)) for record in records] values = [value for value in values if value is not None] @@ -111,7 +137,8 @@ def build_latest_summary(records: list[Record], baseline_window: int = 5, run_source: str | None = None) -> list[Record]: rows: list[Record] = [] - for (model_id, gpu_type, _recipe, _hardware, _software), group in group_by_comparison_cohort(records).items(): + for key, group in group_by_comparison_cohort(records).items(): + model_id, gpu_type = group_identity(key) latest_candidates = group if run_source: latest_candidates = [record for record in group if record_run_source(record) == run_source] @@ -179,14 +206,13 @@ def build_latest_summary(records: list[Record], "metrics": metrics, }) - return sorted(rows, - key=lambda row: (row["status"] != "fail", row["model_id"], row["gpu_type"], row["recipe_fingerprint"], - row["hardware_profile_id"], row["software_profile_id"])) + return sorted(rows, key=latest_row_sort_key) def build_trends(records: list[Record]) -> list[Record]: trends: list[Record] = [] - for (model_id, gpu_type, _recipe, _hardware, _software), group in group_by_comparison_cohort(records).items(): + for key, group in group_by_comparison_cohort(records).items(): + model_id, gpu_type = group_identity(key) points = [] for record in group: metric_policies = resolve_metric_policies(record.get("regression_thresholds")) @@ -208,6 +234,4 @@ def build_trends(records: list[Record]) -> list[Record]: **record_comparison_metadata(group[-1]), "points": points, }) - return sorted(trends, - key=lambda trend: (trend["model_id"], trend["gpu_type"], trend["recipe_fingerprint"], trend[ - "hardware_profile_id"], trend["software_profile_id"])) + return sorted(trends, key=comparison_sort_key) diff --git a/fastvideo/tests/performance/compare_baseline.py b/fastvideo/tests/performance/compare_baseline.py index dbc674ce9..253f50cee 100644 --- a/fastvideo/tests/performance/compare_baseline.py +++ b/fastvideo/tests/performance/compare_baseline.py @@ -5,8 +5,8 @@ This script: 1) reads current benchmark results from fastvideo/tests/performance/results, 2) syncs the canonical baseline from the configured HF dataset repo, 3) compares each current record against the median of up to 5 prior - baseline-eligible successful records in the same GPU/recipe/hardware/software - cohort, + baseline-eligible successful records in the same workload/variant/version, + GPU, recipe, hardware, and software cohort, 4) writes normalized records back to the HF dataset repo according to PERF_UPLOAD_POLICY, 5) exits non-zero if any gated metric exceeds both its percent and absolute @@ -79,6 +79,9 @@ IDENTITY_KEYS = ( "environment_fingerprint", ) COMPARISON_IDENTITY_KEYS = ( + "workload_id", + "variant_id", + "benchmark_version", "recipe_fingerprint", "hardware_profile_id", "software_profile_id", diff --git a/fastvideo/tests/performance/identity.py b/fastvideo/tests/performance/identity.py index 4f0581ebd..82e107ed3 100644 --- a/fastvideo/tests/performance/identity.py +++ b/fastvideo/tests/performance/identity.py @@ -186,7 +186,11 @@ def hardware_profile( gpu_count = int(num_gpus if num_gpus is not None else torch.cuda.device_count()) devices = [] + cuda_device_count = torch.cuda.device_count() for device_id in range(gpu_count): + if device_id >= cuda_device_count: + devices.append(_normalize_gpu_device({})) + continue props = torch.cuda.get_device_properties(device_id) capability = torch.cuda.get_device_capability(device_id) devices.append( @@ -211,7 +215,11 @@ def software_profile( cuda_version: str | None = None, package_versions: Mapping[str, str | None] | None = None, ) -> dict[str, Any]: - """Return normalized software cohort metadata using major/minor versions.""" + """Return normalized software cohort metadata. + + Python, PyTorch, and CUDA use major/minor cohorts. Attention and kernel + packages keep exact versions because patch releases can change performance. + """ versions = dict(package_versions) if package_versions is not None else _installed_package_versions() return { @@ -219,7 +227,7 @@ def software_profile( "pytorch": _major_minor(torch_version or torch.__version__), "cuda": _major_minor(cuda_version or torch.version.cuda), "packages": { - name: _major_minor(version) + name: str(version) for name, version in sorted(versions.items()) if version }, @@ -276,6 +284,8 @@ def _canonicalize(value: Any) -> Any: str(key): _canonicalize(value[key]) for key in sorted(value, key=lambda item: str(item)) } + if isinstance(value, (set, frozenset)): + return [_canonicalize(item) for item in sorted(value, key=lambda item: str(item))] if isinstance(value, tuple): return [_canonicalize(item) for item in value] if isinstance(value, list): @@ -311,7 +321,8 @@ def _looks_like_local_path(value: str) -> bool: if value.startswith(("/", "./", "../", "~")): return True looks_like_hf_repo = re.match(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$", value) - return os.sep in value and looks_like_hf_repo is None + has_separator = "/" in value or "\\" in value or os.sep in value + return has_separator and looks_like_hf_repo is None def _normalize_gpu_device(device: Mapping[str, Any]) -> dict[str, Any]: diff --git a/fastvideo/tests/performance/test_compare_baseline_policy.py b/fastvideo/tests/performance/test_compare_baseline_policy.py index 27b2d968a..ad24d7f06 100644 --- a/fastvideo/tests/performance/test_compare_baseline_policy.py +++ b/fastvideo/tests/performance/test_compare_baseline_policy.py @@ -179,6 +179,26 @@ def test_normalized_record_reads_identity_labels_from_recipe(monkeypatch): assert record["benchmark_version"] == 2 +def test_comparison_identity_filters_use_full_issue_key(): + record = { + "workload_id": "wan-t2v", + "variant_id": "1.3b-sp2", + "benchmark_version": 2, + "recipe_fingerprint": "recipe-1", + "hardware_profile_id": "hw-1", + "software_profile_id": "sw-1", + } + + assert compare_baseline._comparison_identity_filters(record) == { + "workload_id": "wan-t2v", + "variant_id": "1.3b-sp2", + "benchmark_version": "2", + "recipe_fingerprint": "recipe-1", + "hardware_profile_id": "hw-1", + "software_profile_id": "sw-1", + } + + def test_baseline_eligibility_only_for_successful_scheduled_main(): assert compare_baseline._is_baseline_eligible("scheduled_main", True) is True assert compare_baseline._is_baseline_eligible("scheduled_main", False) is False diff --git a/fastvideo/tests/performance/test_dashboard_api.py b/fastvideo/tests/performance/test_dashboard_api.py index 0098b6834..41a25bccf 100644 --- a/fastvideo/tests/performance/test_dashboard_api.py +++ b/fastvideo/tests/performance/test_dashboard_api.py @@ -56,8 +56,34 @@ def _record(model_id, gpu_type, ts, commit, latency, throughput, success=True, * def test_summary_endpoint_returns_latest_group_status(): app = create_app(FakeStore([ - _record("wan", "NVIDIA L40S", "2026-01-01T00:00:00+00:00", "a" * 40, 10.0, 10.0), - _record("wan", "NVIDIA L40S", "2026-01-02T00:00:00+00:00", "b" * 40, 11.0, 9.0), + _record( + "wan", + "NVIDIA L40S", + "2026-01-01T00:00:00+00:00", + "a" * 40, + 10.0, + 10.0, + workload_id="wan-t2v", + variant_id="1.3b-sp2", + benchmark_version=2, + recipe_fingerprint="recipe-a", + hardware_profile_id="hw-l40s", + software_profile_id="sw-cu130", + ), + _record( + "wan", + "NVIDIA L40S", + "2026-01-02T00:00:00+00:00", + "b" * 40, + 11.0, + 9.0, + workload_id="wan-t2v", + variant_id="1.3b-sp2", + benchmark_version=2, + recipe_fingerprint="recipe-a", + hardware_profile_id="hw-l40s", + software_profile_id="sw-cu130", + ), ])) client = TestClient(app) @@ -71,6 +97,10 @@ def test_summary_endpoint_returns_latest_group_status(): assert body["rows"][0]["metrics"]["latency"]["threshold_exceeded"] is True assert body["rows"][0]["threshold_exceeded_metrics"] == ["latency", "throughput"] assert body["rows"][0]["computed_regression_status"] == "fail" + assert body["rows"][0]["workload_id"] == "wan-t2v" + assert body["rows"][0]["variant_id"] == "1.3b-sp2" + assert body["rows"][0]["benchmark_version"] == 2 + assert body["rows"][0]["recipe_fingerprint"] == "recipe-a" def test_summary_status_is_independent_of_days_window(): diff --git a/fastvideo/tests/performance/test_dashboard_service.py b/fastvideo/tests/performance/test_dashboard_service.py index d6c3d0328..8fa3dfbcd 100644 --- a/fastvideo/tests/performance/test_dashboard_service.py +++ b/fastvideo/tests/performance/test_dashboard_service.py @@ -150,6 +150,9 @@ def test_build_latest_summary_keeps_identity_cohorts_separate(): 10.0, 10.0, recipe_fingerprint="recipe-a", + workload_id="wan-t2v", + variant_id="1.3b-sp2", + benchmark_version=2, hardware_profile_id="hw-l40s", software_profile_id="sw-cu130", run_source="scheduled_main", @@ -161,6 +164,9 @@ def test_build_latest_summary_keeps_identity_cohorts_separate(): 20.0, 5.0, recipe_fingerprint="recipe-b", + workload_id="wan-t2v", + variant_id="1.3b-sp2", + benchmark_version=2, hardware_profile_id="hw-l40s", software_profile_id="sw-cu130", run_source="scheduled_main", @@ -172,6 +178,9 @@ def test_build_latest_summary_keeps_identity_cohorts_separate(): 22.0, 4.5, recipe_fingerprint="recipe-b", + workload_id="wan-t2v", + variant_id="1.3b-sp2", + benchmark_version=2, hardware_profile_id="hw-l40s", software_profile_id="sw-cu130", ), @@ -183,6 +192,61 @@ def test_build_latest_summary_keeps_identity_cohorts_separate(): assert len(rows) == 2 assert recipe_b_row["baseline_n"] == 1 assert recipe_b_row["metrics"]["latency"]["baseline"] == 20.0 + assert recipe_b_row["workload_id"] == "wan-t2v" + assert recipe_b_row["variant_id"] == "1.3b-sp2" + assert recipe_b_row["benchmark_version"] == 2 + + +def test_build_latest_summary_keeps_variant_versions_separate(): + records = [ + _record( + "2026-01-01T00:00:00+00:00", + "a" * 40, + 10.0, + 10.0, + workload_id="wan-t2v", + variant_id="1.3b-sp2", + benchmark_version=1, + recipe_fingerprint="recipe-a", + hardware_profile_id="hw-l40s", + software_profile_id="sw-cu130", + run_source="scheduled_main", + baseline_eligible=True, + ), + _record( + "2026-01-02T00:00:00+00:00", + "b" * 40, + 20.0, + 5.0, + workload_id="wan-t2v", + variant_id="1.3b-sp2", + benchmark_version=2, + recipe_fingerprint="recipe-a", + hardware_profile_id="hw-l40s", + software_profile_id="sw-cu130", + run_source="scheduled_main", + baseline_eligible=True, + ), + _record( + "2026-01-03T00:00:00+00:00", + "c" * 40, + 22.0, + 4.5, + workload_id="wan-t2v", + variant_id="1.3b-sp2", + benchmark_version=2, + recipe_fingerprint="recipe-a", + hardware_profile_id="hw-l40s", + software_profile_id="sw-cu130", + ), + ] + + rows = build_latest_summary(records, max_regression=0.05) + version_2_row = next(row for row in rows if row["benchmark_version"] == 2) + + assert len(rows) == 2 + assert version_2_row["baseline_n"] == 1 + assert version_2_row["metrics"]["latency"]["baseline"] == 20.0 def test_filter_records_and_trends_preserve_metric_points(): @@ -272,6 +336,9 @@ def test_load_records_for_model_filters_identity_cohort(tmp_path): "timestamp": "2026-01-01T00:00:00+00:00", "success": true, "baseline_eligible": true, + "workload_id": "wan-t2v", + "variant_id": "1.3b-sp2", + "benchmark_version": 2, "recipe_fingerprint": "recipe-a", "hardware_profile_id": "hw-l40s", "software_profile_id": "sw-cu130" @@ -287,6 +354,9 @@ def test_load_records_for_model_filters_identity_cohort(tmp_path): "timestamp": "2026-01-02T00:00:00+00:00", "success": true, "baseline_eligible": true, + "workload_id": "wan-t2v", + "variant_id": "1.3b-sp2", + "benchmark_version": 2, "recipe_fingerprint": "recipe-b", "hardware_profile_id": "hw-l40s", "software_profile_id": "sw-cu130" @@ -294,11 +364,32 @@ def test_load_records_for_model_filters_identity_cohort(tmp_path): """, encoding="utf-8", ) + (model_dir / "other_version.json").write_text( + """ + { + "model_id": "wan", + "gpu_type": "NVIDIA L40S", + "timestamp": "2026-01-03T00:00:00+00:00", + "success": true, + "baseline_eligible": true, + "workload_id": "wan-t2v", + "variant_id": "1.3b-sp2", + "benchmark_version": 3, + "recipe_fingerprint": "recipe-a", + "hardware_profile_id": "hw-l40s", + "software_profile_id": "sw-cu130" + } + """, + encoding="utf-8", + ) records = hf_store.load_records_for_model( str(tmp_path), "wan", "NVIDIA L40S", + workload_id="wan-t2v", + variant_id="1.3b-sp2", + benchmark_version="2", recipe_fingerprint="recipe-a", hardware_profile_id="hw-l40s", software_profile_id="sw-cu130", diff --git a/fastvideo/tests/performance/test_identity.py b/fastvideo/tests/performance/test_identity.py index 180d25f2c..54da6dfc7 100644 --- a/fastvideo/tests/performance/test_identity.py +++ b/fastvideo/tests/performance/test_identity.py @@ -2,6 +2,9 @@ from copy import deepcopy +import torch + +from fastvideo.tests.performance import identity as identity_module from fastvideo.tests.performance.identity import ( build_recipe_from_benchmark_config, canonical_json, @@ -19,6 +22,9 @@ from fastvideo.tests.performance.identity import ( def _benchmark_config(): return { "benchmark_id": "wan-t2v-1.3b-2gpu", + "workload_id": "wan-t2v", + "variant_id": "1.3b-sp2", + "benchmark_version": 2, "model": { "model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", "model_short_name": "Wan2.1-T2V-1.3B", @@ -58,12 +64,28 @@ def test_canonical_json_is_stable_for_mapping_order(): assert canonical_json(left) == canonical_json(right) +def test_canonical_json_handles_sets_deterministically(): + assert canonical_json({"values": {"b", "a"}}) == '{"values":["a","b"]}' + assert canonical_json({"values": frozenset(("b", "a"))}) == '{"values":["a","b"]}' + + def test_same_config_produces_same_recipe_fingerprint(): cfg = _benchmark_config() assert _fingerprint(cfg) == _fingerprint(deepcopy(cfg)) +def test_recipe_includes_first_class_benchmark_identity(): + recipe = build_recipe_from_benchmark_config(_benchmark_config()) + + assert recipe["benchmark"] == { + "benchmark_id": "wan-t2v-1.3b-2gpu", + "workload_id": "wan-t2v", + "variant_id": "1.3b-sp2", + "benchmark_version": 2, + } + + def test_semantically_equivalent_sequence_forms_match(): cfg = _benchmark_config() equivalent = deepcopy(cfg) @@ -103,6 +125,14 @@ def test_num_inference_steps_changes_recipe_fingerprint(): assert _fingerprint(cfg) != _fingerprint(changed) +def test_benchmark_version_changes_recipe_fingerprint(): + cfg = _benchmark_config() + changed = deepcopy(cfg) + changed["benchmark_version"] = 3 + + assert _fingerprint(cfg) != _fingerprint(changed) + + def test_attention_backend_changes_recipe_fingerprint(): cfg = _benchmark_config() @@ -138,7 +168,7 @@ def test_distributed_layout_changes_recipe_fingerprint(): assert _fingerprint(cfg) != _fingerprint(changed) -def test_software_profile_uses_major_minor_versions_for_id(): +def test_software_profile_uses_exact_attention_kernel_package_versions_for_id(): profile_a = software_profile( python_version="3.12.4", torch_version="2.12.0+cu130", @@ -163,11 +193,14 @@ def test_software_profile_uses_major_minor_versions_for_id(): "pytorch": "2.12", "cuda": "13.0", "packages": { - "fastvideo_kernel": "0.3", - "triton": "3.4", + "fastvideo_kernel": "0.3.2", + "triton": "3.4.1", }, } - assert software_profile_id(profile_a) == software_profile_id(profile_b) + assert profile_b["python"] == "3.12" + assert profile_b["pytorch"] == "2.12" + assert profile_b["cuda"] == "13.0" + assert software_profile_id(profile_a) != software_profile_id(profile_b) def test_hardware_profile_id_uses_normalized_gpu_cohort(): @@ -208,6 +241,41 @@ def test_hardware_profile_id_uses_normalized_gpu_cohort(): assert hardware_profile_id(profile).startswith("hw-") +def test_hardware_profile_pads_missing_requested_cuda_devices(monkeypatch): + class Props: + name = "NVIDIA L40S" + total_memory = 48 * 1024**3 + + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "device_count", lambda: 1) + monkeypatch.setattr(torch.cuda, "get_device_properties", lambda device_id: Props()) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device_id: (8, 9)) + + profile = hardware_profile(num_gpus=2, interconnect="unknown") + + assert profile["gpu_count"] == 2 + assert profile["gpus"] == [ + { + "name": "NVIDIA L40S", + "memory_gb": 48, + "compute_capability": "8.9", + }, + { + "name": "unknown", + "memory_gb": None, + "compute_capability": None, + }, + ] + + +def test_local_path_detection_checks_both_path_separators(monkeypatch): + monkeypatch.setattr(identity_module.os, "sep", "\\") + + assert identity_module._looks_like_local_path("models/local/checkpoint") is True + assert identity_module._looks_like_local_path(r"C:\models\checkpoint") is True + assert identity_module._looks_like_local_path("Wan-AI/Wan2.1-T2V-1.3B-Diffusers") is False + + def test_environment_metadata_is_separate_audit_fingerprint(): cfg = _benchmark_config() recipe_hash = _fingerprint(cfg) @@ -261,3 +329,28 @@ def test_runtime_identity_from_generator_summarizes_worker_records(): "resolved_attention_backend": "FLASH_ATTN", "resolved_model_revision": revision, } + + +def test_collect_worker_identity_handles_torch_module_pipeline(): + from fastvideo.tests.performance.test_inference_performance import _collect_worker_identity + + class Backend: + name = "FLASH_ATTN" + + class Leaf(torch.nn.Module): + backend = Backend() + + class Pipeline(torch.nn.Module): + model_path = "/models/local" + + def __init__(self): + super().__init__() + self.leaf = Leaf() + + class Worker: + pipeline = Pipeline() + + assert _collect_worker_identity(Worker()) == { + "resolved_attention_backends": ["FLASH_ATTN"], + "resolved_model_path": "/models/local", + } diff --git a/fastvideo/tests/performance/test_inference_performance.py b/fastvideo/tests/performance/test_inference_performance.py index 8d6e6c5ba..ac5cc3f49 100644 --- a/fastvideo/tests/performance/test_inference_performance.py +++ b/fastvideo/tests/performance/test_inference_performance.py @@ -252,14 +252,24 @@ def _backend_name(value) -> str: def _collect_worker_identity(worker) -> dict[str, Any]: pipeline = getattr(worker, "pipeline", None) model_path = getattr(pipeline, "model_path", None) - modules = getattr(pipeline, "modules", {}) or {} backends: set[str] = set() - for module in modules.values(): - if not isinstance(module, torch.nn.Module): - continue - for submodule in module.modules(): - backend = getattr(submodule, "backend", None) + modules = getattr(pipeline, "modules", None) + if isinstance(modules, Mapping): + module_iter = modules.values() + elif isinstance(pipeline, torch.nn.Module): + module_iter = (pipeline,) + else: + module_iter = () + + for module in module_iter: + if isinstance(module, torch.nn.Module): + for submodule in module.modules(): + backend = getattr(submodule, "backend", None) + if backend is not None: + backends.add(_backend_name(backend)) + else: + backend = getattr(module, "backend", None) if backend is not None: backends.add(_backend_name(backend))