[ci]: add performance fingerprint cohorts
This commit is contained in:
Mac Lee
@@ -269,7 +269,33 @@ Written by `test_inference_performance.py`. One file per benchmark run.
|
||||
"timestamp": "2026-05-08T22:00:00+00:00",
|
||||
"text_encoder_time_s": 2.141,
|
||||
"dit_time_s": 8.437,
|
||||
"vae_decode_time_s": 3.208
|
||||
"vae_decode_time_s": 3.208,
|
||||
"recipe": {
|
||||
"recipe_schema_version": 1,
|
||||
"benchmark": { "benchmark_id": "wan-t2v-1.3b-2gpu" },
|
||||
"model": { "model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" },
|
||||
"init_kwargs": { "num_gpus": 2, "sp_size": 2, "tp_size": 1 },
|
||||
"generation_kwargs": { "height": 480, "width": 832, "num_frames": 45 },
|
||||
"inputs": { "prompt_count": 1, "prompt_sha256": ["<sha256>"] },
|
||||
"attention": { "requested_backend": "FLASH_ATTN" }
|
||||
},
|
||||
"recipe_fingerprint": "<sha256>",
|
||||
"hardware_profile": {
|
||||
"device_type": "cuda",
|
||||
"gpu_count": 2,
|
||||
"gpus": [{ "name": "NVIDIA L40S", "memory_gb": 48, "compute_capability": "8.9" }],
|
||||
"interconnect": "none_or_partial"
|
||||
},
|
||||
"hardware_profile_id": "hw-<sha256-prefix>",
|
||||
"software_profile": {
|
||||
"python": "3.12",
|
||||
"pytorch": "2.12",
|
||||
"cuda": "13.0",
|
||||
"packages": { "fastvideo_kernel": "0.3", "triton": "3.4" }
|
||||
},
|
||||
"software_profile_id": "sw-<sha256-prefix>",
|
||||
"environment_metadata": { "env": { "IMAGE_VERSION": "py3.12-cuda13.0.0" } },
|
||||
"environment_fingerprint": "env-<sha256-prefix>"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -297,6 +323,10 @@ result, used as the rolling-baseline source of truth.
|
||||
"gated": true
|
||||
}
|
||||
},
|
||||
"recipe_fingerprint": "<sha256>",
|
||||
"hardware_profile_id": "hw-<sha256-prefix>",
|
||||
"software_profile_id": "sw-<sha256-prefix>",
|
||||
"environment_fingerprint": "env-<sha256-prefix>",
|
||||
"success": true
|
||||
}
|
||||
```
|
||||
@@ -309,6 +339,11 @@ dashboard lists skipped plots for metric series that have no non-null values.
|
||||
Records missing both `run_source` and `baseline_eligible` are treated as legacy
|
||||
successful main/full-suite uploads and remain eligible for rolling baselines.
|
||||
|
||||
New records compare only against the same `model_id`, `gpu_type`,
|
||||
`recipe_fingerprint`, `hardware_profile_id`, and `software_profile_id` cohort.
|
||||
`environment_metadata` and `environment_fingerprint` are audit data and are not
|
||||
part of the comparison key.
|
||||
|
||||
## Environment variable reference
|
||||
|
||||
| Variable | Default | Used by | Purpose |
|
||||
|
||||
@@ -268,16 +268,25 @@ def load_records_for_model(
|
||||
model_id: str,
|
||||
gpu_type: str | None = None,
|
||||
*,
|
||||
recipe_fingerprint: str | None = None,
|
||||
hardware_profile_id: str | None = None,
|
||||
software_profile_id: str | None = None,
|
||||
last_n: int | None = None,
|
||||
successful_only: bool = True,
|
||||
baseline_eligible_only: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return records for a specific *model_id*, optionally filtered by GPU.
|
||||
"""Return records for a specific *model_id*, optionally filtered by cohort.
|
||||
|
||||
Args:
|
||||
local_dir: Root directory previously populated by :func:`sync_from_hf`.
|
||||
model_id: Matches the ``model_id`` field inside each JSON record.
|
||||
gpu_type: When set, only records whose ``gpu_type`` matches are returned.
|
||||
recipe_fingerprint: When set, only records from the same benchmark
|
||||
recipe are returned.
|
||||
hardware_profile_id: When set, only records from the same hardware
|
||||
cohort are returned.
|
||||
software_profile_id: When set, only records from the same software
|
||||
cohort are returned.
|
||||
last_n: When set, return only the most recent *n* records (after all
|
||||
other filters). Useful for sliding-window baseline calculations.
|
||||
successful_only: Passed through to :func:`load_records`.
|
||||
@@ -299,6 +308,15 @@ def load_records_for_model(
|
||||
if gpu_type is not None:
|
||||
records = [r for r in records if r.get("gpu_type") == gpu_type]
|
||||
|
||||
identity_filters = {
|
||||
"recipe_fingerprint": recipe_fingerprint,
|
||||
"hardware_profile_id": hardware_profile_id,
|
||||
"software_profile_id": software_profile_id,
|
||||
}
|
||||
for key, expected in identity_filters.items():
|
||||
if expected is not None:
|
||||
records = [r for r in records if r.get(key) == expected]
|
||||
|
||||
if last_n is not None:
|
||||
records = records[-last_n:]
|
||||
|
||||
|
||||
@@ -17,6 +17,11 @@ from fastvideo.performance.hf_store import is_baseline_eligible_record, safe_flo
|
||||
from fastvideo.performance.metric_policy import regression_delta, resolve_metric_policies
|
||||
|
||||
Record = dict[str, Any]
|
||||
COMPARISON_COHORT_KEYS = (
|
||||
"recipe_fingerprint",
|
||||
"hardware_profile_id",
|
||||
"software_profile_id",
|
||||
)
|
||||
|
||||
|
||||
def parse_timestamp(value: Any) -> datetime | None:
|
||||
@@ -77,12 +82,19 @@ def record_metadata(record: Record) -> Record:
|
||||
}
|
||||
|
||||
|
||||
def group_by_model_gpu(records: list[Record]) -> dict[tuple[str, str], list[Record]]:
|
||||
groups: dict[tuple[str, str], list[Record]] = defaultdict(list)
|
||||
def record_comparison_metadata(record: Record) -> Record:
|
||||
return {key: record.get(key) or "" for key in COMPARISON_COHORT_KEYS}
|
||||
|
||||
|
||||
def group_by_comparison_cohort(records: list[Record]) -> dict[tuple[str, str, str, str, str], list[Record]]:
|
||||
groups: dict[tuple[str, str, str, str, str], list[Record]] = defaultdict(list)
|
||||
for record in records:
|
||||
model_id = str(record.get("model_id") or "unknown")
|
||||
gpu_type = str(record.get("gpu_type") or "unknown")
|
||||
groups[(model_id, gpu_type)].append(record)
|
||||
recipe_fingerprint = str(record.get("recipe_fingerprint") or "")
|
||||
hardware_profile = str(record.get("hardware_profile_id") or "")
|
||||
software_profile = str(record.get("software_profile_id") or "")
|
||||
groups[(model_id, gpu_type, recipe_fingerprint, hardware_profile, software_profile)].append(record)
|
||||
return {key: sorted(value, key=record_sort_key) for key, value in groups.items()}
|
||||
|
||||
|
||||
@@ -99,7 +111,7 @@ def build_latest_summary(records: list[Record],
|
||||
baseline_window: int = 5,
|
||||
run_source: str | None = None) -> list[Record]:
|
||||
rows: list[Record] = []
|
||||
for (model_id, gpu_type), group in group_by_model_gpu(records).items():
|
||||
for (model_id, gpu_type, _recipe, _hardware, _software), group in group_by_comparison_cohort(records).items():
|
||||
latest_candidates = group
|
||||
if run_source:
|
||||
latest_candidates = [record for record in group if record_run_source(record) == run_source]
|
||||
@@ -156,6 +168,7 @@ def build_latest_summary(records: list[Record],
|
||||
"timestamp": latest.get("timestamp"),
|
||||
"commit_sha": latest.get("commit_sha"),
|
||||
**record_metadata(latest),
|
||||
**record_comparison_metadata(latest),
|
||||
"success": success,
|
||||
"baseline_n": len(baseline_records),
|
||||
"worst_regression_pct": worst_regression,
|
||||
@@ -166,12 +179,14 @@ def build_latest_summary(records: list[Record],
|
||||
"metrics": metrics,
|
||||
})
|
||||
|
||||
return sorted(rows, key=lambda row: (row["status"] != "fail", row["model_id"], row["gpu_type"]))
|
||||
return sorted(rows,
|
||||
key=lambda row: (row["status"] != "fail", row["model_id"], row["gpu_type"], row["recipe_fingerprint"],
|
||||
row["hardware_profile_id"], row["software_profile_id"]))
|
||||
|
||||
|
||||
def build_trends(records: list[Record]) -> list[Record]:
|
||||
trends: list[Record] = []
|
||||
for (model_id, gpu_type), group in group_by_model_gpu(records).items():
|
||||
for (model_id, gpu_type, _recipe, _hardware, _software), group in group_by_comparison_cohort(records).items():
|
||||
points = []
|
||||
for record in group:
|
||||
metric_policies = resolve_metric_policies(record.get("regression_thresholds"))
|
||||
@@ -179,6 +194,7 @@ def build_trends(records: list[Record]) -> list[Record]:
|
||||
"timestamp": record.get("timestamp"),
|
||||
"commit_sha": record.get("commit_sha"),
|
||||
**record_metadata(record),
|
||||
**record_comparison_metadata(record),
|
||||
"success": bool(record.get("success", True)),
|
||||
"metrics": {
|
||||
policy.key: safe_float(record.get(policy.key))
|
||||
@@ -189,6 +205,9 @@ def build_trends(records: list[Record]) -> list[Record]:
|
||||
trends.append({
|
||||
"model_id": model_id,
|
||||
"gpu_type": gpu_type,
|
||||
**record_comparison_metadata(group[-1]),
|
||||
"points": points,
|
||||
})
|
||||
return sorted(trends, key=lambda trend: (trend["model_id"], trend["gpu_type"]))
|
||||
return sorted(trends,
|
||||
key=lambda trend: (trend["model_id"], trend["gpu_type"], trend["recipe_fingerprint"], trend[
|
||||
"hardware_profile_id"], trend["software_profile_id"]))
|
||||
|
||||
@@ -5,7 +5,8 @@ This script:
|
||||
1) reads current benchmark results from fastvideo/tests/performance/results,
|
||||
2) syncs the canonical baseline from the configured HF dataset repo,
|
||||
3) compares each current record against the median of up to 5 prior
|
||||
baseline-eligible successful records (filtered by gpu_type),
|
||||
baseline-eligible successful records in the same GPU/recipe/hardware/software
|
||||
cohort,
|
||||
4) writes normalized records back to the HF dataset repo according to
|
||||
PERF_UPLOAD_POLICY,
|
||||
5) exits non-zero if any gated metric exceeds both its percent and absolute
|
||||
@@ -64,6 +65,21 @@ PERF_REPORTS_DIR = os.environ.get("PERF_REPORTS_DIR", "/root/data/perf_reports")
|
||||
UPLOAD_POLICY = os.environ.get("PERF_UPLOAD_POLICY", "never").strip().lower()
|
||||
VALID_UPLOAD_POLICIES = {"never", "pass", "always"}
|
||||
VALID_RUN_SOURCES = {"pr", "local", "scheduled_main", "unknown"}
|
||||
IDENTITY_KEYS = (
|
||||
"recipe",
|
||||
"recipe_fingerprint",
|
||||
"hardware_profile",
|
||||
"hardware_profile_id",
|
||||
"software_profile",
|
||||
"software_profile_id",
|
||||
"environment_metadata",
|
||||
"environment_fingerprint",
|
||||
)
|
||||
COMPARISON_IDENTITY_KEYS = (
|
||||
"recipe_fingerprint",
|
||||
"hardware_profile_id",
|
||||
"software_profile_id",
|
||||
)
|
||||
|
||||
|
||||
def _should_persist_tracking() -> bool:
|
||||
@@ -136,6 +152,29 @@ def _record_metadata(run_source: str, result: dict[str, Any]) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _identity_metadata(result: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
key: result[key]
|
||||
for key in IDENTITY_KEYS
|
||||
if key in result and result[key] is not None
|
||||
}
|
||||
|
||||
|
||||
def _comparison_identity_filters(record: dict[str, Any]) -> dict[str, str]:
|
||||
return {
|
||||
key: str(record[key])
|
||||
for key in COMPARISON_IDENTITY_KEYS
|
||||
if record.get(key)
|
||||
}
|
||||
|
||||
|
||||
def _format_identity_filters(filters: dict[str, str]) -> str:
|
||||
if not filters:
|
||||
return ""
|
||||
compact = ", ".join(f"{key}={value}" for key, value in filters.items())
|
||||
return f" ({compact})"
|
||||
|
||||
|
||||
def _load_current_results() -> list[dict[str, Any]]:
|
||||
pattern = os.path.join(RESULTS_DIR, "perf_*.json")
|
||||
records: list[dict[str, Any]] = []
|
||||
@@ -169,7 +208,7 @@ def normalize_performance_result(result: dict[str, Any]) -> dict[str, Any]:
|
||||
vae_decode_time = safe_float(result.get("vae_decode_time_s"))
|
||||
metric_policies = resolve_metric_policies(result.get("regression_thresholds"))
|
||||
|
||||
return {
|
||||
record = {
|
||||
"model_id": model_id,
|
||||
"timestamp": timestamp,
|
||||
"commit_sha": commit_sha,
|
||||
@@ -184,6 +223,8 @@ def normalize_performance_result(result: dict[str, Any]) -> dict[str, Any]:
|
||||
"success": True,
|
||||
**_record_metadata(_detect_run_source(), result),
|
||||
}
|
||||
record.update(_identity_metadata(result))
|
||||
return record
|
||||
|
||||
|
||||
def _normalize_record(result: dict[str, Any]) -> dict[str, Any]:
|
||||
@@ -422,6 +463,7 @@ def main() -> int:
|
||||
TRACKING_ROOT,
|
||||
record["model_id"],
|
||||
record["gpu_type"],
|
||||
**_comparison_identity_filters(record),
|
||||
last_n=5,
|
||||
successful_only=True,
|
||||
baseline_eligible_only=True,
|
||||
@@ -429,7 +471,8 @@ def main() -> int:
|
||||
|
||||
if not baseline_records:
|
||||
print(f"No baseline for {record['model_id']} on "
|
||||
f"{record['gpu_type']}. Initializing...")
|
||||
f"{record['gpu_type']}"
|
||||
f"{_format_identity_filters(_comparison_identity_filters(record))}. Initializing...")
|
||||
failures: list[str] = []
|
||||
record["success"] = True
|
||||
else:
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Comparable identity helpers for performance benchmark records."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import enum
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
RECIPE_SCHEMA_VERSION = 1
|
||||
PROFILE_ID_LENGTH = 16
|
||||
_PATH_EXCLUDED_GENERATION_KEYS = {"output_path", "output_video_name"}
|
||||
_PROMPT_KEYS = {"prompt", "negative_prompt", "neg_prompt"}
|
||||
_PROFILE_ENV_VARS = (
|
||||
"CUDA_VISIBLE_DEVICES",
|
||||
"FASTVIDEO_ATTENTION_BACKEND",
|
||||
"IMAGE_VERSION",
|
||||
"UV_TORCH_BACKEND",
|
||||
)
|
||||
_PACKAGE_DISTRIBUTIONS = {
|
||||
"fastvideo_kernel": ("fastvideo-kernel", "fastvideo_kernel"),
|
||||
"flash_attn": ("flash-attn", "flash_attn"),
|
||||
"sageattention": ("sageattention",),
|
||||
"sageattn3": ("sageattn3",),
|
||||
"triton": ("triton",),
|
||||
"xformers": ("xformers",),
|
||||
}
|
||||
|
||||
|
||||
def canonical_json(payload: Any) -> str:
|
||||
"""Return deterministic, compact JSON for a recipe/profile payload."""
|
||||
|
||||
return json.dumps(
|
||||
_canonicalize(payload),
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
)
|
||||
|
||||
|
||||
def sha256_hexdigest(payload: str | bytes) -> str:
|
||||
if isinstance(payload, str):
|
||||
payload = payload.encode("utf-8")
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def recipe_fingerprint(recipe: Mapping[str, Any]) -> str:
|
||||
return sha256_hexdigest(canonical_json(recipe))
|
||||
|
||||
|
||||
def profile_id(prefix: str, profile: Mapping[str, Any]) -> str:
|
||||
return f"{prefix}-{sha256_hexdigest(canonical_json(profile))[:PROFILE_ID_LENGTH]}"
|
||||
|
||||
|
||||
def hardware_profile_id(profile: Mapping[str, Any]) -> str:
|
||||
return profile_id("hw", profile)
|
||||
|
||||
|
||||
def software_profile_id(profile: Mapping[str, Any]) -> str:
|
||||
return profile_id("sw", profile)
|
||||
|
||||
|
||||
def environment_fingerprint(metadata: Mapping[str, Any]) -> str:
|
||||
return profile_id("env", metadata)
|
||||
|
||||
|
||||
def build_recipe_from_benchmark_config(
|
||||
cfg: Mapping[str, Any],
|
||||
*,
|
||||
attention_backend: str | None = None,
|
||||
resolved_attention_backend: str | None = None,
|
||||
resolved_model_revision: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a deterministic recipe document from a benchmark config.
|
||||
|
||||
The recipe captures fields that describe what benchmark workload was run. It
|
||||
intentionally excludes timings, timestamps, commit metadata, and output paths.
|
||||
"""
|
||||
|
||||
model = dict(cfg.get("model") or {})
|
||||
init_kwargs = dict(cfg.get("init_kwargs") or {})
|
||||
generation_kwargs = dict(cfg.get("generation_kwargs") or {})
|
||||
prompts = list(cfg.get("test_prompts") or ["A cinematic video."])
|
||||
if attention_backend is None:
|
||||
attention_backend = os.environ.get("FASTVIDEO_ATTENTION_BACKEND")
|
||||
|
||||
negative_prompt = generation_kwargs.pop("negative_prompt", generation_kwargs.pop("neg_prompt", None))
|
||||
generation_recipe = {
|
||||
key: value
|
||||
for key, value in generation_kwargs.items()
|
||||
if key not in _PATH_EXCLUDED_GENERATION_KEYS and key not in _PROMPT_KEYS
|
||||
}
|
||||
|
||||
return {
|
||||
"recipe_schema_version": RECIPE_SCHEMA_VERSION,
|
||||
"benchmark": {
|
||||
"benchmark_id": cfg.get("benchmark_id"),
|
||||
"workload_id": cfg.get("workload_id"),
|
||||
"variant_id": cfg.get("variant_id"),
|
||||
"benchmark_version": cfg.get("benchmark_version"),
|
||||
},
|
||||
"model": {
|
||||
"model_path": normalize_model_path(model.get("model_path")),
|
||||
"revision": _none_if_empty(model.get("revision") or cfg.get("revision")),
|
||||
"resolved_revision": _none_if_empty(resolved_model_revision),
|
||||
},
|
||||
"init_kwargs": init_kwargs,
|
||||
"generation_kwargs": generation_recipe,
|
||||
"inputs": {
|
||||
"prompt_count": len(prompts),
|
||||
"prompt_sha256": [_optional_digest(prompt) for prompt in prompts],
|
||||
"negative_prompt_sha256": _optional_digest(negative_prompt),
|
||||
},
|
||||
"attention": {
|
||||
"requested_backend": _none_if_empty(attention_backend) or "auto",
|
||||
"resolved_backend": _none_if_empty(resolved_attention_backend),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def normalize_model_path(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = os.fspath(value) if isinstance(value, os.PathLike) else str(value)
|
||||
text = text.strip()
|
||||
if not text:
|
||||
return None
|
||||
if _looks_like_local_path(text):
|
||||
return Path(text).expanduser().as_posix()
|
||||
return re.sub(r"/+", "/", text)
|
||||
|
||||
|
||||
def hardware_profile(
|
||||
*,
|
||||
num_gpus: int | None = None,
|
||||
gpu_devices: Sequence[Mapping[str, Any]] | None = None,
|
||||
device_type: str | None = None,
|
||||
interconnect: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Return normalized hardware cohort metadata.
|
||||
|
||||
``gpu_devices`` is an injection point for tests and non-CUDA callers.
|
||||
"""
|
||||
|
||||
if gpu_devices is not None:
|
||||
devices = [_normalize_gpu_device(device) for device in gpu_devices]
|
||||
return {
|
||||
"device_type": device_type or "cuda",
|
||||
"gpu_count": num_gpus if num_gpus is not None else len(devices),
|
||||
"gpus": devices,
|
||||
"interconnect": interconnect or "unknown",
|
||||
}
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
return {
|
||||
"device_type": device_type or "cpu",
|
||||
"gpu_count": 0,
|
||||
"gpus": [],
|
||||
"interconnect": interconnect or "none",
|
||||
}
|
||||
|
||||
gpu_count = int(num_gpus if num_gpus is not None else torch.cuda.device_count())
|
||||
devices = []
|
||||
for device_id in range(gpu_count):
|
||||
props = torch.cuda.get_device_properties(device_id)
|
||||
capability = torch.cuda.get_device_capability(device_id)
|
||||
devices.append(
|
||||
_normalize_gpu_device({
|
||||
"name": props.name,
|
||||
"memory_bytes": props.total_memory,
|
||||
"compute_capability": f"{capability[0]}.{capability[1]}",
|
||||
}))
|
||||
|
||||
return {
|
||||
"device_type": device_type or "cuda",
|
||||
"gpu_count": gpu_count,
|
||||
"gpus": devices,
|
||||
"interconnect": interconnect or _detect_cuda_interconnect(gpu_count),
|
||||
}
|
||||
|
||||
|
||||
def software_profile(
|
||||
*,
|
||||
python_version: str | None = None,
|
||||
torch_version: str | None = None,
|
||||
cuda_version: str | None = None,
|
||||
package_versions: Mapping[str, str | None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Return normalized software cohort metadata using major/minor versions."""
|
||||
|
||||
versions = dict(package_versions) if package_versions is not None else _installed_package_versions()
|
||||
return {
|
||||
"python": _major_minor(python_version or platform.python_version()),
|
||||
"pytorch": _major_minor(torch_version or torch.__version__),
|
||||
"cuda": _major_minor(cuda_version or torch.version.cuda),
|
||||
"packages": {
|
||||
name: _major_minor(version)
|
||||
for name, version in sorted(versions.items())
|
||||
if version
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def environment_metadata(
|
||||
*,
|
||||
env: Mapping[str, str] | None = None,
|
||||
package_versions: Mapping[str, str | None] | None = None,
|
||||
hardware: Mapping[str, Any] | None = None,
|
||||
software: Mapping[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Return audit metadata kept separate from comparable identity keys."""
|
||||
|
||||
source_env = env if env is not None else os.environ
|
||||
full_package_versions = dict(package_versions) if package_versions is not None else _installed_package_versions()
|
||||
return {
|
||||
"python": {
|
||||
"version": platform.python_version(),
|
||||
"implementation": platform.python_implementation(),
|
||||
},
|
||||
"platform": {
|
||||
"system": platform.system(),
|
||||
"machine": platform.machine(),
|
||||
"release": platform.release(),
|
||||
},
|
||||
"torch": {
|
||||
"version": torch.__version__,
|
||||
"cuda": torch.version.cuda,
|
||||
"cudnn": torch.backends.cudnn.version(),
|
||||
},
|
||||
"env": {
|
||||
key: source_env.get(key)
|
||||
for key in _PROFILE_ENV_VARS
|
||||
if source_env.get(key) is not None
|
||||
},
|
||||
"packages": {
|
||||
key: value
|
||||
for key, value in sorted(full_package_versions.items())
|
||||
if value
|
||||
},
|
||||
"hardware_profile": dict(hardware) if hardware is not None else hardware_profile(),
|
||||
"software_profile": dict(software) if software is not None else software_profile(
|
||||
package_versions=full_package_versions),
|
||||
}
|
||||
|
||||
|
||||
def _canonicalize(value: Any) -> Any:
|
||||
if dataclasses.is_dataclass(value) and not isinstance(value, type):
|
||||
return _canonicalize(dataclasses.asdict(value))
|
||||
if isinstance(value, Mapping):
|
||||
return {
|
||||
str(key): _canonicalize(value[key])
|
||||
for key in sorted(value, key=lambda item: str(item))
|
||||
}
|
||||
if isinstance(value, tuple):
|
||||
return [_canonicalize(item) for item in value]
|
||||
if isinstance(value, list):
|
||||
return [_canonicalize(item) for item in value]
|
||||
if isinstance(value, enum.Enum):
|
||||
return value.name
|
||||
if isinstance(value, Path):
|
||||
return value.expanduser().as_posix()
|
||||
if isinstance(value, os.PathLike):
|
||||
return os.fspath(value)
|
||||
if isinstance(value, type):
|
||||
return f"{value.__module__}.{value.__qualname__}"
|
||||
if isinstance(value, torch.dtype):
|
||||
return str(value).removeprefix("torch.")
|
||||
return value
|
||||
|
||||
|
||||
def _none_if_empty(value: Any) -> Any | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, str) and not value.strip():
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _optional_digest(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return sha256_hexdigest(str(value))
|
||||
|
||||
|
||||
def _looks_like_local_path(value: str) -> bool:
|
||||
if value.startswith(("/", "./", "../", "~")):
|
||||
return True
|
||||
looks_like_hf_repo = re.match(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$", value)
|
||||
return os.sep in value and looks_like_hf_repo is None
|
||||
|
||||
|
||||
def _normalize_gpu_device(device: Mapping[str, Any]) -> dict[str, Any]:
|
||||
memory_gb = device.get("memory_gb")
|
||||
if memory_gb is None and device.get("memory_bytes") is not None:
|
||||
memory_gb = round(float(device["memory_bytes"]) / (1024**3))
|
||||
return {
|
||||
"name": _none_if_empty(device.get("name")) or "unknown",
|
||||
"memory_gb": memory_gb,
|
||||
"compute_capability": _none_if_empty(device.get("compute_capability")),
|
||||
}
|
||||
|
||||
|
||||
def _detect_cuda_interconnect(gpu_count: int) -> str:
|
||||
if gpu_count <= 1:
|
||||
return "single_gpu"
|
||||
try:
|
||||
from fastvideo.platforms import current_platform
|
||||
|
||||
if current_platform.is_cuda() and current_platform.is_full_nvlink(list(range(gpu_count))):
|
||||
return "full_nvlink"
|
||||
return "none_or_partial"
|
||||
except Exception:
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _installed_package_versions() -> dict[str, str | None]:
|
||||
return {
|
||||
name: _distribution_version(*distribution_names)
|
||||
for name, distribution_names in _PACKAGE_DISTRIBUTIONS.items()
|
||||
}
|
||||
|
||||
|
||||
def _distribution_version(*names: str) -> str | None:
|
||||
for name in names:
|
||||
try:
|
||||
return importlib.metadata.version(name)
|
||||
except importlib.metadata.PackageNotFoundError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _major_minor(version: str | None) -> str | None:
|
||||
if not version:
|
||||
return None
|
||||
match = re.search(r"(\d+)\.(\d+)", version)
|
||||
if match is None:
|
||||
return version
|
||||
return f"{match.group(1)}.{match.group(2)}"
|
||||
|
||||
|
||||
__all__ = [
|
||||
"RECIPE_SCHEMA_VERSION",
|
||||
"canonical_json",
|
||||
"environment_fingerprint",
|
||||
"environment_metadata",
|
||||
"hardware_profile",
|
||||
"hardware_profile_id",
|
||||
"normalize_model_path",
|
||||
"profile_id",
|
||||
"recipe_fingerprint",
|
||||
"build_recipe_from_benchmark_config",
|
||||
"sha256_hexdigest",
|
||||
"software_profile",
|
||||
"software_profile_id",
|
||||
]
|
||||
@@ -119,6 +119,40 @@ def test_boolean_regression_threshold_values_are_ignored():
|
||||
assert latency.threshold_percent == 0.08
|
||||
assert latency.threshold_absolute == 0.5
|
||||
assert latency.gated is False
|
||||
def test_normalized_record_preserves_identity_metadata(monkeypatch):
|
||||
monkeypatch.setenv("PERF_RUN_SOURCE", "pr")
|
||||
raw = _raw_result()
|
||||
raw.update({
|
||||
"recipe": {
|
||||
"recipe_schema_version": 1,
|
||||
},
|
||||
"recipe_fingerprint": "recipe-1",
|
||||
"hardware_profile": {
|
||||
"gpu_count": 1,
|
||||
},
|
||||
"hardware_profile_id": "hw-1",
|
||||
"software_profile": {
|
||||
"python": "3.12",
|
||||
},
|
||||
"software_profile_id": "sw-1",
|
||||
"environment_metadata": {
|
||||
"env": {
|
||||
"IMAGE_VERSION": "latest",
|
||||
},
|
||||
},
|
||||
"environment_fingerprint": "env-1",
|
||||
})
|
||||
|
||||
record = compare_baseline.normalize_performance_result(raw)
|
||||
|
||||
assert record["recipe"] == {"recipe_schema_version": 1}
|
||||
assert record["recipe_fingerprint"] == "recipe-1"
|
||||
assert record["hardware_profile"] == {"gpu_count": 1}
|
||||
assert record["hardware_profile_id"] == "hw-1"
|
||||
assert record["software_profile"] == {"python": "3.12"}
|
||||
assert record["software_profile_id"] == "sw-1"
|
||||
assert record["environment_metadata"] == {"env": {"IMAGE_VERSION": "latest"}}
|
||||
assert record["environment_fingerprint"] == "env-1"
|
||||
|
||||
|
||||
def test_baseline_eligibility_only_for_successful_scheduled_main():
|
||||
|
||||
@@ -142,6 +142,47 @@ def test_build_latest_summary_separates_informational_threshold_crossing():
|
||||
assert rows[0]["threshold_exceeded_metrics"] == ["latency"]
|
||||
assert rows[0]["failing_metrics"] == []
|
||||
assert rows[0]["computed_regression_status"] == "pass"
|
||||
def test_build_latest_summary_keeps_identity_cohorts_separate():
|
||||
records = [
|
||||
_record(
|
||||
"2026-01-01T00:00:00+00:00",
|
||||
"a" * 40,
|
||||
10.0,
|
||||
10.0,
|
||||
recipe_fingerprint="recipe-a",
|
||||
hardware_profile_id="hw-l40s",
|
||||
software_profile_id="sw-cu130",
|
||||
run_source="scheduled_main",
|
||||
baseline_eligible=True,
|
||||
),
|
||||
_record(
|
||||
"2026-01-02T00:00:00+00:00",
|
||||
"b" * 40,
|
||||
20.0,
|
||||
5.0,
|
||||
recipe_fingerprint="recipe-b",
|
||||
hardware_profile_id="hw-l40s",
|
||||
software_profile_id="sw-cu130",
|
||||
run_source="scheduled_main",
|
||||
baseline_eligible=True,
|
||||
),
|
||||
_record(
|
||||
"2026-01-03T00:00:00+00:00",
|
||||
"c" * 40,
|
||||
22.0,
|
||||
4.5,
|
||||
recipe_fingerprint="recipe-b",
|
||||
hardware_profile_id="hw-l40s",
|
||||
software_profile_id="sw-cu130",
|
||||
),
|
||||
]
|
||||
|
||||
rows = build_latest_summary(records, max_regression=0.05)
|
||||
recipe_b_row = next(row for row in rows if row["recipe_fingerprint"] == "recipe-b")
|
||||
|
||||
assert len(rows) == 2
|
||||
assert recipe_b_row["baseline_n"] == 1
|
||||
assert recipe_b_row["metrics"]["latency"]["baseline"] == 20.0
|
||||
|
||||
|
||||
def test_filter_records_and_trends_preserve_metric_points():
|
||||
@@ -218,3 +259,51 @@ def test_load_records_can_filter_baseline_eligible_records(tmp_path):
|
||||
"2026-01-02T00:00:00+00:00",
|
||||
"2026-01-03T00:00:00+00:00",
|
||||
}
|
||||
|
||||
|
||||
def test_load_records_for_model_filters_identity_cohort(tmp_path):
|
||||
model_dir = tmp_path / "wan"
|
||||
model_dir.mkdir()
|
||||
(model_dir / "matching.json").write_text(
|
||||
"""
|
||||
{
|
||||
"model_id": "wan",
|
||||
"gpu_type": "NVIDIA L40S",
|
||||
"timestamp": "2026-01-01T00:00:00+00:00",
|
||||
"success": true,
|
||||
"baseline_eligible": true,
|
||||
"recipe_fingerprint": "recipe-a",
|
||||
"hardware_profile_id": "hw-l40s",
|
||||
"software_profile_id": "sw-cu130"
|
||||
}
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(model_dir / "other_recipe.json").write_text(
|
||||
"""
|
||||
{
|
||||
"model_id": "wan",
|
||||
"gpu_type": "NVIDIA L40S",
|
||||
"timestamp": "2026-01-02T00:00:00+00:00",
|
||||
"success": true,
|
||||
"baseline_eligible": true,
|
||||
"recipe_fingerprint": "recipe-b",
|
||||
"hardware_profile_id": "hw-l40s",
|
||||
"software_profile_id": "sw-cu130"
|
||||
}
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
records = hf_store.load_records_for_model(
|
||||
str(tmp_path),
|
||||
"wan",
|
||||
"NVIDIA L40S",
|
||||
recipe_fingerprint="recipe-a",
|
||||
hardware_profile_id="hw-l40s",
|
||||
software_profile_id="sw-cu130",
|
||||
baseline_eligible_only=True,
|
||||
)
|
||||
|
||||
assert len(records) == 1
|
||||
assert records[0]["recipe_fingerprint"] == "recipe-a"
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from copy import deepcopy
|
||||
|
||||
from fastvideo.tests.performance.identity import (
|
||||
build_recipe_from_benchmark_config,
|
||||
canonical_json,
|
||||
environment_fingerprint,
|
||||
environment_metadata,
|
||||
hardware_profile,
|
||||
hardware_profile_id,
|
||||
recipe_fingerprint,
|
||||
software_profile,
|
||||
software_profile_id,
|
||||
)
|
||||
|
||||
|
||||
def _benchmark_config():
|
||||
return {
|
||||
"benchmark_id": "wan-t2v-1.3b-2gpu",
|
||||
"model": {
|
||||
"model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
|
||||
"model_short_name": "Wan2.1-T2V-1.3B",
|
||||
},
|
||||
"init_kwargs": {
|
||||
"num_gpus": 2,
|
||||
"sp_size": 2,
|
||||
"tp_size": 1,
|
||||
"vae_sp": True,
|
||||
"vae_tiling": True,
|
||||
"text_encoder_precisions": ["fp32"],
|
||||
},
|
||||
"generation_kwargs": {
|
||||
"height": 480,
|
||||
"width": 832,
|
||||
"num_frames": 45,
|
||||
"num_inference_steps": 4,
|
||||
"guidance_scale": 3,
|
||||
"embedded_cfg_scale": 6,
|
||||
"seed": 1024,
|
||||
"fps": 24,
|
||||
"neg_prompt": "low quality",
|
||||
},
|
||||
"test_prompts": ["A cinematic video."],
|
||||
}
|
||||
|
||||
|
||||
def _fingerprint(cfg, attention_backend="FLASH_ATTN"):
|
||||
recipe = build_recipe_from_benchmark_config(cfg, attention_backend=attention_backend)
|
||||
return recipe_fingerprint(recipe)
|
||||
|
||||
|
||||
def test_canonical_json_is_stable_for_mapping_order():
|
||||
left = {"b": [2, 1], "a": {"z": "last", "m": "middle"}}
|
||||
right = {"a": {"m": "middle", "z": "last"}, "b": [2, 1]}
|
||||
|
||||
assert canonical_json(left) == canonical_json(right)
|
||||
|
||||
|
||||
def test_same_config_produces_same_recipe_fingerprint():
|
||||
cfg = _benchmark_config()
|
||||
|
||||
assert _fingerprint(cfg) == _fingerprint(deepcopy(cfg))
|
||||
|
||||
|
||||
def test_semantically_equivalent_sequence_forms_match():
|
||||
cfg = _benchmark_config()
|
||||
equivalent = deepcopy(cfg)
|
||||
equivalent["init_kwargs"]["text_encoder_precisions"] = tuple(
|
||||
equivalent["init_kwargs"]["text_encoder_precisions"])
|
||||
|
||||
assert _fingerprint(cfg) == _fingerprint(equivalent)
|
||||
|
||||
|
||||
def test_output_path_does_not_change_recipe_fingerprint():
|
||||
cfg = _benchmark_config()
|
||||
with_output_path = deepcopy(cfg)
|
||||
with_output_path["generation_kwargs"]["output_path"] = "/tmp/generated"
|
||||
|
||||
assert _fingerprint(cfg) == _fingerprint(with_output_path)
|
||||
|
||||
|
||||
def test_num_inference_steps_changes_recipe_fingerprint():
|
||||
cfg = _benchmark_config()
|
||||
changed = deepcopy(cfg)
|
||||
changed["generation_kwargs"]["num_inference_steps"] = 8
|
||||
|
||||
assert _fingerprint(cfg) != _fingerprint(changed)
|
||||
|
||||
|
||||
def test_attention_backend_changes_recipe_fingerprint():
|
||||
cfg = _benchmark_config()
|
||||
|
||||
assert _fingerprint(cfg, attention_backend="FLASH_ATTN") != _fingerprint(
|
||||
cfg, attention_backend="TORCH_SDPA")
|
||||
|
||||
|
||||
def test_precision_changes_recipe_fingerprint():
|
||||
cfg = _benchmark_config()
|
||||
changed = deepcopy(cfg)
|
||||
changed["init_kwargs"]["text_encoder_precisions"] = ["bf16"]
|
||||
|
||||
assert _fingerprint(cfg) != _fingerprint(changed)
|
||||
|
||||
|
||||
def test_dimensions_and_seed_change_recipe_fingerprint():
|
||||
cfg = _benchmark_config()
|
||||
different_dimensions = deepcopy(cfg)
|
||||
different_dimensions["generation_kwargs"]["width"] = 1024
|
||||
different_seed = deepcopy(cfg)
|
||||
different_seed["generation_kwargs"]["seed"] = 2048
|
||||
|
||||
assert _fingerprint(cfg) != _fingerprint(different_dimensions)
|
||||
assert _fingerprint(cfg) != _fingerprint(different_seed)
|
||||
|
||||
|
||||
def test_distributed_layout_changes_recipe_fingerprint():
|
||||
cfg = _benchmark_config()
|
||||
changed = deepcopy(cfg)
|
||||
changed["init_kwargs"]["sp_size"] = 1
|
||||
changed["init_kwargs"]["tp_size"] = 2
|
||||
|
||||
assert _fingerprint(cfg) != _fingerprint(changed)
|
||||
|
||||
|
||||
def test_software_profile_uses_major_minor_versions_for_id():
|
||||
profile_a = software_profile(
|
||||
python_version="3.12.4",
|
||||
torch_version="2.12.0+cu130",
|
||||
cuda_version="13.0",
|
||||
package_versions={
|
||||
"triton": "3.4.1",
|
||||
"fastvideo_kernel": "0.3.2",
|
||||
},
|
||||
)
|
||||
profile_b = software_profile(
|
||||
python_version="3.12.5",
|
||||
torch_version="2.12.1+cu130",
|
||||
cuda_version="13.0",
|
||||
package_versions={
|
||||
"triton": "3.4.9",
|
||||
"fastvideo_kernel": "0.3.7",
|
||||
},
|
||||
)
|
||||
|
||||
assert profile_a == {
|
||||
"python": "3.12",
|
||||
"pytorch": "2.12",
|
||||
"cuda": "13.0",
|
||||
"packages": {
|
||||
"fastvideo_kernel": "0.3",
|
||||
"triton": "3.4",
|
||||
},
|
||||
}
|
||||
assert software_profile_id(profile_a) == software_profile_id(profile_b)
|
||||
|
||||
|
||||
def test_hardware_profile_id_uses_normalized_gpu_cohort():
|
||||
profile = hardware_profile(
|
||||
num_gpus=2,
|
||||
gpu_devices=[
|
||||
{
|
||||
"name": "NVIDIA L40S",
|
||||
"memory_bytes": 48 * 1024**3,
|
||||
"compute_capability": "8.9",
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA L40S",
|
||||
"memory_gb": 48,
|
||||
"compute_capability": "8.9",
|
||||
},
|
||||
],
|
||||
interconnect="none_or_partial",
|
||||
)
|
||||
|
||||
assert profile == {
|
||||
"device_type": "cuda",
|
||||
"gpu_count": 2,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA L40S",
|
||||
"memory_gb": 48,
|
||||
"compute_capability": "8.9",
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA L40S",
|
||||
"memory_gb": 48,
|
||||
"compute_capability": "8.9",
|
||||
},
|
||||
],
|
||||
"interconnect": "none_or_partial",
|
||||
}
|
||||
assert hardware_profile_id(profile).startswith("hw-")
|
||||
|
||||
|
||||
def test_environment_metadata_is_separate_audit_fingerprint():
|
||||
cfg = _benchmark_config()
|
||||
recipe_hash = _fingerprint(cfg)
|
||||
audit_a = environment_metadata(
|
||||
env={"IMAGE_VERSION": "py3.12-cuda13.0.0"},
|
||||
package_versions={"triton": "3.4.1"},
|
||||
hardware={"device_type": "cuda", "gpu_count": 2},
|
||||
software={"python": "3.12", "pytorch": "2.12", "cuda": "13.0"},
|
||||
)
|
||||
audit_b = environment_metadata(
|
||||
env={"IMAGE_VERSION": "py3.12-cuda13.0.1"},
|
||||
package_versions={"triton": "3.4.1"},
|
||||
hardware={"device_type": "cuda", "gpu_count": 2},
|
||||
software={"python": "3.12", "pytorch": "2.12", "cuda": "13.0"},
|
||||
)
|
||||
|
||||
assert recipe_hash == _fingerprint(cfg)
|
||||
assert environment_fingerprint(audit_a) != environment_fingerprint(audit_b)
|
||||
@@ -18,6 +18,16 @@ import pytest
|
||||
|
||||
from fastvideo import VideoGenerator
|
||||
from fastvideo.logger import init_logger
|
||||
from fastvideo.tests.performance.identity import (
|
||||
build_recipe_from_benchmark_config,
|
||||
environment_fingerprint,
|
||||
environment_metadata,
|
||||
hardware_profile,
|
||||
hardware_profile_id,
|
||||
recipe_fingerprint,
|
||||
software_profile,
|
||||
software_profile_id,
|
||||
)
|
||||
from fastvideo.worker.multiproc_executor import MultiprocExecutor
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -227,6 +237,27 @@ def _write_results(results):
|
||||
logger.info("Performance results written to %s", filepath)
|
||||
|
||||
|
||||
def _build_identity_fields(cfg, init_kwargs):
|
||||
recipe = build_recipe_from_benchmark_config(cfg)
|
||||
num_gpus = init_kwargs.get("num_gpus", cfg.get("run_config", {}).get("required_gpus", 1))
|
||||
hw_profile = hardware_profile(num_gpus=num_gpus)
|
||||
sw_profile = software_profile()
|
||||
env_metadata = environment_metadata(
|
||||
hardware=hw_profile,
|
||||
software=sw_profile,
|
||||
)
|
||||
return {
|
||||
"recipe": recipe,
|
||||
"recipe_fingerprint": recipe_fingerprint(recipe),
|
||||
"hardware_profile": hw_profile,
|
||||
"hardware_profile_id": hardware_profile_id(hw_profile),
|
||||
"software_profile": sw_profile,
|
||||
"software_profile_id": software_profile_id(sw_profile),
|
||||
"environment_metadata": env_metadata,
|
||||
"environment_fingerprint": environment_fingerprint(env_metadata),
|
||||
}
|
||||
|
||||
|
||||
# -- Test -------------------------------------------------------------------
|
||||
|
||||
def _run_benchmark(cfg):
|
||||
@@ -318,6 +349,7 @@ def _run_benchmark(cfg):
|
||||
"dit_time_s": _avg_component(all_component_times, "dit_time_s"),
|
||||
"vae_decode_time_s": _avg_component(all_component_times,
|
||||
"vae_decode_time_s"),
|
||||
**_build_identity_fields(cfg, init_kwargs),
|
||||
}
|
||||
|
||||
logger.info(
|
||||
|
||||
Reference in New Issue
Block a user