From 663dfb2f37477bbe71fb6c09e6ff192d0672a28d Mon Sep 17 00:00:00 2001 From: Mac Lee Date: Sat, 4 Jul 2026 12:33:12 +0000 Subject: [PATCH] [ci]: add performance fingerprint cohorts --- docs/contributing/performance_benchmarks.md | 37 +- fastvideo/performance/hf_store.py | 20 +- fastvideo/performance_dashboard/service.py | 33 +- .../tests/performance/compare_baseline.py | 49 ++- fastvideo/tests/performance/identity.py | 365 ++++++++++++++++++ .../test_compare_baseline_policy.py | 34 ++ .../performance/test_dashboard_service.py | 89 +++++ fastvideo/tests/performance/test_identity.py | 213 ++++++++++ .../performance/test_inference_performance.py | 32 ++ 9 files changed, 860 insertions(+), 12 deletions(-) create mode 100644 fastvideo/tests/performance/identity.py create mode 100644 fastvideo/tests/performance/test_identity.py diff --git a/docs/contributing/performance_benchmarks.md b/docs/contributing/performance_benchmarks.md index 827d9f767..59a85151f 100644 --- a/docs/contributing/performance_benchmarks.md +++ b/docs/contributing/performance_benchmarks.md @@ -269,7 +269,33 @@ Written by `test_inference_performance.py`. One file per benchmark run. "timestamp": "2026-05-08T22:00:00+00:00", "text_encoder_time_s": 2.141, "dit_time_s": 8.437, - "vae_decode_time_s": 3.208 + "vae_decode_time_s": 3.208, + "recipe": { + "recipe_schema_version": 1, + "benchmark": { "benchmark_id": "wan-t2v-1.3b-2gpu" }, + "model": { "model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" }, + "init_kwargs": { "num_gpus": 2, "sp_size": 2, "tp_size": 1 }, + "generation_kwargs": { "height": 480, "width": 832, "num_frames": 45 }, + "inputs": { "prompt_count": 1, "prompt_sha256": [""] }, + "attention": { "requested_backend": "FLASH_ATTN" } + }, + "recipe_fingerprint": "", + "hardware_profile": { + "device_type": "cuda", + "gpu_count": 2, + "gpus": [{ "name": "NVIDIA L40S", "memory_gb": 48, "compute_capability": "8.9" }], + "interconnect": "none_or_partial" + }, + "hardware_profile_id": "hw-", + "software_profile": { + "python": "3.12", + "pytorch": "2.12", + "cuda": "13.0", + "packages": { "fastvideo_kernel": "0.3", "triton": "3.4" } + }, + "software_profile_id": "sw-", + "environment_metadata": { "env": { "IMAGE_VERSION": "py3.12-cuda13.0.0" } }, + "environment_fingerprint": "env-" } ``` @@ -297,6 +323,10 @@ result, used as the rolling-baseline source of truth. "gated": true } }, + "recipe_fingerprint": "", + "hardware_profile_id": "hw-", + "software_profile_id": "sw-", + "environment_fingerprint": "env-", "success": true } ``` @@ -309,6 +339,11 @@ dashboard lists skipped plots for metric series that have no non-null values. Records missing both `run_source` and `baseline_eligible` are treated as legacy successful main/full-suite uploads and remain eligible for rolling baselines. +New records compare only against the same `model_id`, `gpu_type`, +`recipe_fingerprint`, `hardware_profile_id`, and `software_profile_id` cohort. +`environment_metadata` and `environment_fingerprint` are audit data and are not +part of the comparison key. + ## Environment variable reference | Variable | Default | Used by | Purpose | diff --git a/fastvideo/performance/hf_store.py b/fastvideo/performance/hf_store.py index 09eecdf06..41ef8be69 100644 --- a/fastvideo/performance/hf_store.py +++ b/fastvideo/performance/hf_store.py @@ -268,16 +268,25 @@ def load_records_for_model( model_id: str, gpu_type: str | None = None, *, + recipe_fingerprint: str | None = None, + hardware_profile_id: str | None = None, + software_profile_id: str | None = None, last_n: int | None = None, successful_only: bool = True, baseline_eligible_only: bool = False, ) -> list[dict[str, Any]]: - """Return records for a specific *model_id*, optionally filtered by GPU. + """Return records for a specific *model_id*, optionally filtered by cohort. Args: local_dir: Root directory previously populated by :func:`sync_from_hf`. model_id: Matches the ``model_id`` field inside each JSON record. gpu_type: When set, only records whose ``gpu_type`` matches are returned. + recipe_fingerprint: When set, only records from the same benchmark + recipe are returned. + hardware_profile_id: When set, only records from the same hardware + cohort are returned. + software_profile_id: When set, only records from the same software + cohort are returned. last_n: When set, return only the most recent *n* records (after all other filters). Useful for sliding-window baseline calculations. successful_only: Passed through to :func:`load_records`. @@ -299,6 +308,15 @@ def load_records_for_model( if gpu_type is not None: records = [r for r in records if r.get("gpu_type") == gpu_type] + identity_filters = { + "recipe_fingerprint": recipe_fingerprint, + "hardware_profile_id": hardware_profile_id, + "software_profile_id": software_profile_id, + } + for key, expected in identity_filters.items(): + if expected is not None: + records = [r for r in records if r.get(key) == expected] + if last_n is not None: records = records[-last_n:] diff --git a/fastvideo/performance_dashboard/service.py b/fastvideo/performance_dashboard/service.py index 1d835221e..8b9bcdbc1 100644 --- a/fastvideo/performance_dashboard/service.py +++ b/fastvideo/performance_dashboard/service.py @@ -17,6 +17,11 @@ from fastvideo.performance.hf_store import is_baseline_eligible_record, safe_flo from fastvideo.performance.metric_policy import regression_delta, resolve_metric_policies Record = dict[str, Any] +COMPARISON_COHORT_KEYS = ( + "recipe_fingerprint", + "hardware_profile_id", + "software_profile_id", +) def parse_timestamp(value: Any) -> datetime | None: @@ -77,12 +82,19 @@ def record_metadata(record: Record) -> Record: } -def group_by_model_gpu(records: list[Record]) -> dict[tuple[str, str], list[Record]]: - groups: dict[tuple[str, str], list[Record]] = defaultdict(list) +def record_comparison_metadata(record: Record) -> Record: + return {key: record.get(key) or "" for key in COMPARISON_COHORT_KEYS} + + +def group_by_comparison_cohort(records: list[Record]) -> dict[tuple[str, str, str, str, str], list[Record]]: + groups: dict[tuple[str, str, str, str, str], list[Record]] = defaultdict(list) for record in records: model_id = str(record.get("model_id") or "unknown") gpu_type = str(record.get("gpu_type") or "unknown") - groups[(model_id, gpu_type)].append(record) + recipe_fingerprint = str(record.get("recipe_fingerprint") or "") + hardware_profile = str(record.get("hardware_profile_id") or "") + software_profile = str(record.get("software_profile_id") or "") + groups[(model_id, gpu_type, recipe_fingerprint, hardware_profile, software_profile)].append(record) return {key: sorted(value, key=record_sort_key) for key, value in groups.items()} @@ -99,7 +111,7 @@ def build_latest_summary(records: list[Record], baseline_window: int = 5, run_source: str | None = None) -> list[Record]: rows: list[Record] = [] - for (model_id, gpu_type), group in group_by_model_gpu(records).items(): + for (model_id, gpu_type, _recipe, _hardware, _software), group in group_by_comparison_cohort(records).items(): latest_candidates = group if run_source: latest_candidates = [record for record in group if record_run_source(record) == run_source] @@ -156,6 +168,7 @@ def build_latest_summary(records: list[Record], "timestamp": latest.get("timestamp"), "commit_sha": latest.get("commit_sha"), **record_metadata(latest), + **record_comparison_metadata(latest), "success": success, "baseline_n": len(baseline_records), "worst_regression_pct": worst_regression, @@ -166,12 +179,14 @@ def build_latest_summary(records: list[Record], "metrics": metrics, }) - return sorted(rows, key=lambda row: (row["status"] != "fail", row["model_id"], row["gpu_type"])) + return sorted(rows, + key=lambda row: (row["status"] != "fail", row["model_id"], row["gpu_type"], row["recipe_fingerprint"], + row["hardware_profile_id"], row["software_profile_id"])) def build_trends(records: list[Record]) -> list[Record]: trends: list[Record] = [] - for (model_id, gpu_type), group in group_by_model_gpu(records).items(): + for (model_id, gpu_type, _recipe, _hardware, _software), group in group_by_comparison_cohort(records).items(): points = [] for record in group: metric_policies = resolve_metric_policies(record.get("regression_thresholds")) @@ -179,6 +194,7 @@ def build_trends(records: list[Record]) -> list[Record]: "timestamp": record.get("timestamp"), "commit_sha": record.get("commit_sha"), **record_metadata(record), + **record_comparison_metadata(record), "success": bool(record.get("success", True)), "metrics": { policy.key: safe_float(record.get(policy.key)) @@ -189,6 +205,9 @@ def build_trends(records: list[Record]) -> list[Record]: trends.append({ "model_id": model_id, "gpu_type": gpu_type, + **record_comparison_metadata(group[-1]), "points": points, }) - return sorted(trends, key=lambda trend: (trend["model_id"], trend["gpu_type"])) + return sorted(trends, + key=lambda trend: (trend["model_id"], trend["gpu_type"], trend["recipe_fingerprint"], trend[ + "hardware_profile_id"], trend["software_profile_id"])) diff --git a/fastvideo/tests/performance/compare_baseline.py b/fastvideo/tests/performance/compare_baseline.py index 15408e5d4..070ee6c4c 100644 --- a/fastvideo/tests/performance/compare_baseline.py +++ b/fastvideo/tests/performance/compare_baseline.py @@ -5,7 +5,8 @@ This script: 1) reads current benchmark results from fastvideo/tests/performance/results, 2) syncs the canonical baseline from the configured HF dataset repo, 3) compares each current record against the median of up to 5 prior - baseline-eligible successful records (filtered by gpu_type), + baseline-eligible successful records in the same GPU/recipe/hardware/software + cohort, 4) writes normalized records back to the HF dataset repo according to PERF_UPLOAD_POLICY, 5) exits non-zero if any gated metric exceeds both its percent and absolute @@ -64,6 +65,21 @@ PERF_REPORTS_DIR = os.environ.get("PERF_REPORTS_DIR", "/root/data/perf_reports") UPLOAD_POLICY = os.environ.get("PERF_UPLOAD_POLICY", "never").strip().lower() VALID_UPLOAD_POLICIES = {"never", "pass", "always"} VALID_RUN_SOURCES = {"pr", "local", "scheduled_main", "unknown"} +IDENTITY_KEYS = ( + "recipe", + "recipe_fingerprint", + "hardware_profile", + "hardware_profile_id", + "software_profile", + "software_profile_id", + "environment_metadata", + "environment_fingerprint", +) +COMPARISON_IDENTITY_KEYS = ( + "recipe_fingerprint", + "hardware_profile_id", + "software_profile_id", +) def _should_persist_tracking() -> bool: @@ -136,6 +152,29 @@ def _record_metadata(run_source: str, result: dict[str, Any]) -> dict[str, Any]: } +def _identity_metadata(result: dict[str, Any]) -> dict[str, Any]: + return { + key: result[key] + for key in IDENTITY_KEYS + if key in result and result[key] is not None + } + + +def _comparison_identity_filters(record: dict[str, Any]) -> dict[str, str]: + return { + key: str(record[key]) + for key in COMPARISON_IDENTITY_KEYS + if record.get(key) + } + + +def _format_identity_filters(filters: dict[str, str]) -> str: + if not filters: + return "" + compact = ", ".join(f"{key}={value}" for key, value in filters.items()) + return f" ({compact})" + + def _load_current_results() -> list[dict[str, Any]]: pattern = os.path.join(RESULTS_DIR, "perf_*.json") records: list[dict[str, Any]] = [] @@ -169,7 +208,7 @@ def normalize_performance_result(result: dict[str, Any]) -> dict[str, Any]: vae_decode_time = safe_float(result.get("vae_decode_time_s")) metric_policies = resolve_metric_policies(result.get("regression_thresholds")) - return { + record = { "model_id": model_id, "timestamp": timestamp, "commit_sha": commit_sha, @@ -184,6 +223,8 @@ def normalize_performance_result(result: dict[str, Any]) -> dict[str, Any]: "success": True, **_record_metadata(_detect_run_source(), result), } + record.update(_identity_metadata(result)) + return record def _normalize_record(result: dict[str, Any]) -> dict[str, Any]: @@ -422,6 +463,7 @@ def main() -> int: TRACKING_ROOT, record["model_id"], record["gpu_type"], + **_comparison_identity_filters(record), last_n=5, successful_only=True, baseline_eligible_only=True, @@ -429,7 +471,8 @@ def main() -> int: if not baseline_records: print(f"No baseline for {record['model_id']} on " - f"{record['gpu_type']}. Initializing...") + f"{record['gpu_type']}" + f"{_format_identity_filters(_comparison_identity_filters(record))}. Initializing...") failures: list[str] = [] record["success"] = True else: diff --git a/fastvideo/tests/performance/identity.py b/fastvideo/tests/performance/identity.py new file mode 100644 index 000000000..fb9880ef8 --- /dev/null +++ b/fastvideo/tests/performance/identity.py @@ -0,0 +1,365 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Comparable identity helpers for performance benchmark records.""" + +from __future__ import annotations + +import dataclasses +import enum +import hashlib +import importlib.metadata +import json +import os +import platform +import re +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +import torch + +RECIPE_SCHEMA_VERSION = 1 +PROFILE_ID_LENGTH = 16 +_PATH_EXCLUDED_GENERATION_KEYS = {"output_path", "output_video_name"} +_PROMPT_KEYS = {"prompt", "negative_prompt", "neg_prompt"} +_PROFILE_ENV_VARS = ( + "CUDA_VISIBLE_DEVICES", + "FASTVIDEO_ATTENTION_BACKEND", + "IMAGE_VERSION", + "UV_TORCH_BACKEND", +) +_PACKAGE_DISTRIBUTIONS = { + "fastvideo_kernel": ("fastvideo-kernel", "fastvideo_kernel"), + "flash_attn": ("flash-attn", "flash_attn"), + "sageattention": ("sageattention",), + "sageattn3": ("sageattn3",), + "triton": ("triton",), + "xformers": ("xformers",), +} + + +def canonical_json(payload: Any) -> str: + """Return deterministic, compact JSON for a recipe/profile payload.""" + + return json.dumps( + _canonicalize(payload), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ) + + +def sha256_hexdigest(payload: str | bytes) -> str: + if isinstance(payload, str): + payload = payload.encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def recipe_fingerprint(recipe: Mapping[str, Any]) -> str: + return sha256_hexdigest(canonical_json(recipe)) + + +def profile_id(prefix: str, profile: Mapping[str, Any]) -> str: + return f"{prefix}-{sha256_hexdigest(canonical_json(profile))[:PROFILE_ID_LENGTH]}" + + +def hardware_profile_id(profile: Mapping[str, Any]) -> str: + return profile_id("hw", profile) + + +def software_profile_id(profile: Mapping[str, Any]) -> str: + return profile_id("sw", profile) + + +def environment_fingerprint(metadata: Mapping[str, Any]) -> str: + return profile_id("env", metadata) + + +def build_recipe_from_benchmark_config( + cfg: Mapping[str, Any], + *, + attention_backend: str | None = None, + resolved_attention_backend: str | None = None, + resolved_model_revision: str | None = None, +) -> dict[str, Any]: + """Build a deterministic recipe document from a benchmark config. + + The recipe captures fields that describe what benchmark workload was run. It + intentionally excludes timings, timestamps, commit metadata, and output paths. + """ + + model = dict(cfg.get("model") or {}) + init_kwargs = dict(cfg.get("init_kwargs") or {}) + generation_kwargs = dict(cfg.get("generation_kwargs") or {}) + prompts = list(cfg.get("test_prompts") or ["A cinematic video."]) + if attention_backend is None: + attention_backend = os.environ.get("FASTVIDEO_ATTENTION_BACKEND") + + negative_prompt = generation_kwargs.pop("negative_prompt", generation_kwargs.pop("neg_prompt", None)) + generation_recipe = { + key: value + for key, value in generation_kwargs.items() + if key not in _PATH_EXCLUDED_GENERATION_KEYS and key not in _PROMPT_KEYS + } + + return { + "recipe_schema_version": RECIPE_SCHEMA_VERSION, + "benchmark": { + "benchmark_id": cfg.get("benchmark_id"), + "workload_id": cfg.get("workload_id"), + "variant_id": cfg.get("variant_id"), + "benchmark_version": cfg.get("benchmark_version"), + }, + "model": { + "model_path": normalize_model_path(model.get("model_path")), + "revision": _none_if_empty(model.get("revision") or cfg.get("revision")), + "resolved_revision": _none_if_empty(resolved_model_revision), + }, + "init_kwargs": init_kwargs, + "generation_kwargs": generation_recipe, + "inputs": { + "prompt_count": len(prompts), + "prompt_sha256": [_optional_digest(prompt) for prompt in prompts], + "negative_prompt_sha256": _optional_digest(negative_prompt), + }, + "attention": { + "requested_backend": _none_if_empty(attention_backend) or "auto", + "resolved_backend": _none_if_empty(resolved_attention_backend), + }, + } + + +def normalize_model_path(value: Any) -> str | None: + if value is None: + return None + text = os.fspath(value) if isinstance(value, os.PathLike) else str(value) + text = text.strip() + if not text: + return None + if _looks_like_local_path(text): + return Path(text).expanduser().as_posix() + return re.sub(r"/+", "/", text) + + +def hardware_profile( + *, + num_gpus: int | None = None, + gpu_devices: Sequence[Mapping[str, Any]] | None = None, + device_type: str | None = None, + interconnect: str | None = None, +) -> dict[str, Any]: + """Return normalized hardware cohort metadata. + + ``gpu_devices`` is an injection point for tests and non-CUDA callers. + """ + + if gpu_devices is not None: + devices = [_normalize_gpu_device(device) for device in gpu_devices] + return { + "device_type": device_type or "cuda", + "gpu_count": num_gpus if num_gpus is not None else len(devices), + "gpus": devices, + "interconnect": interconnect or "unknown", + } + + if not torch.cuda.is_available(): + return { + "device_type": device_type or "cpu", + "gpu_count": 0, + "gpus": [], + "interconnect": interconnect or "none", + } + + gpu_count = int(num_gpus if num_gpus is not None else torch.cuda.device_count()) + devices = [] + for device_id in range(gpu_count): + props = torch.cuda.get_device_properties(device_id) + capability = torch.cuda.get_device_capability(device_id) + devices.append( + _normalize_gpu_device({ + "name": props.name, + "memory_bytes": props.total_memory, + "compute_capability": f"{capability[0]}.{capability[1]}", + })) + + return { + "device_type": device_type or "cuda", + "gpu_count": gpu_count, + "gpus": devices, + "interconnect": interconnect or _detect_cuda_interconnect(gpu_count), + } + + +def software_profile( + *, + python_version: str | None = None, + torch_version: str | None = None, + cuda_version: str | None = None, + package_versions: Mapping[str, str | None] | None = None, +) -> dict[str, Any]: + """Return normalized software cohort metadata using major/minor versions.""" + + versions = dict(package_versions) if package_versions is not None else _installed_package_versions() + return { + "python": _major_minor(python_version or platform.python_version()), + "pytorch": _major_minor(torch_version or torch.__version__), + "cuda": _major_minor(cuda_version or torch.version.cuda), + "packages": { + name: _major_minor(version) + for name, version in sorted(versions.items()) + if version + }, + } + + +def environment_metadata( + *, + env: Mapping[str, str] | None = None, + package_versions: Mapping[str, str | None] | None = None, + hardware: Mapping[str, Any] | None = None, + software: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Return audit metadata kept separate from comparable identity keys.""" + + source_env = env if env is not None else os.environ + full_package_versions = dict(package_versions) if package_versions is not None else _installed_package_versions() + return { + "python": { + "version": platform.python_version(), + "implementation": platform.python_implementation(), + }, + "platform": { + "system": platform.system(), + "machine": platform.machine(), + "release": platform.release(), + }, + "torch": { + "version": torch.__version__, + "cuda": torch.version.cuda, + "cudnn": torch.backends.cudnn.version(), + }, + "env": { + key: source_env.get(key) + for key in _PROFILE_ENV_VARS + if source_env.get(key) is not None + }, + "packages": { + key: value + for key, value in sorted(full_package_versions.items()) + if value + }, + "hardware_profile": dict(hardware) if hardware is not None else hardware_profile(), + "software_profile": dict(software) if software is not None else software_profile( + package_versions=full_package_versions), + } + + +def _canonicalize(value: Any) -> Any: + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return _canonicalize(dataclasses.asdict(value)) + if isinstance(value, Mapping): + return { + str(key): _canonicalize(value[key]) + for key in sorted(value, key=lambda item: str(item)) + } + if isinstance(value, tuple): + return [_canonicalize(item) for item in value] + if isinstance(value, list): + return [_canonicalize(item) for item in value] + if isinstance(value, enum.Enum): + return value.name + if isinstance(value, Path): + return value.expanduser().as_posix() + if isinstance(value, os.PathLike): + return os.fspath(value) + if isinstance(value, type): + return f"{value.__module__}.{value.__qualname__}" + if isinstance(value, torch.dtype): + return str(value).removeprefix("torch.") + return value + + +def _none_if_empty(value: Any) -> Any | None: + if value is None: + return None + if isinstance(value, str) and not value.strip(): + return None + return value + + +def _optional_digest(value: Any) -> str | None: + if value is None: + return None + return sha256_hexdigest(str(value)) + + +def _looks_like_local_path(value: str) -> bool: + if value.startswith(("/", "./", "../", "~")): + return True + looks_like_hf_repo = re.match(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$", value) + return os.sep in value and looks_like_hf_repo is None + + +def _normalize_gpu_device(device: Mapping[str, Any]) -> dict[str, Any]: + memory_gb = device.get("memory_gb") + if memory_gb is None and device.get("memory_bytes") is not None: + memory_gb = round(float(device["memory_bytes"]) / (1024**3)) + return { + "name": _none_if_empty(device.get("name")) or "unknown", + "memory_gb": memory_gb, + "compute_capability": _none_if_empty(device.get("compute_capability")), + } + + +def _detect_cuda_interconnect(gpu_count: int) -> str: + if gpu_count <= 1: + return "single_gpu" + try: + from fastvideo.platforms import current_platform + + if current_platform.is_cuda() and current_platform.is_full_nvlink(list(range(gpu_count))): + return "full_nvlink" + return "none_or_partial" + except Exception: + return "unknown" + + +def _installed_package_versions() -> dict[str, str | None]: + return { + name: _distribution_version(*distribution_names) + for name, distribution_names in _PACKAGE_DISTRIBUTIONS.items() + } + + +def _distribution_version(*names: str) -> str | None: + for name in names: + try: + return importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + continue + return None + + +def _major_minor(version: str | None) -> str | None: + if not version: + return None + match = re.search(r"(\d+)\.(\d+)", version) + if match is None: + return version + return f"{match.group(1)}.{match.group(2)}" + + +__all__ = [ + "RECIPE_SCHEMA_VERSION", + "canonical_json", + "environment_fingerprint", + "environment_metadata", + "hardware_profile", + "hardware_profile_id", + "normalize_model_path", + "profile_id", + "recipe_fingerprint", + "build_recipe_from_benchmark_config", + "sha256_hexdigest", + "software_profile", + "software_profile_id", +] diff --git a/fastvideo/tests/performance/test_compare_baseline_policy.py b/fastvideo/tests/performance/test_compare_baseline_policy.py index c5e60b197..65e831b45 100644 --- a/fastvideo/tests/performance/test_compare_baseline_policy.py +++ b/fastvideo/tests/performance/test_compare_baseline_policy.py @@ -119,6 +119,40 @@ def test_boolean_regression_threshold_values_are_ignored(): assert latency.threshold_percent == 0.08 assert latency.threshold_absolute == 0.5 assert latency.gated is False +def test_normalized_record_preserves_identity_metadata(monkeypatch): + monkeypatch.setenv("PERF_RUN_SOURCE", "pr") + raw = _raw_result() + raw.update({ + "recipe": { + "recipe_schema_version": 1, + }, + "recipe_fingerprint": "recipe-1", + "hardware_profile": { + "gpu_count": 1, + }, + "hardware_profile_id": "hw-1", + "software_profile": { + "python": "3.12", + }, + "software_profile_id": "sw-1", + "environment_metadata": { + "env": { + "IMAGE_VERSION": "latest", + }, + }, + "environment_fingerprint": "env-1", + }) + + record = compare_baseline.normalize_performance_result(raw) + + assert record["recipe"] == {"recipe_schema_version": 1} + assert record["recipe_fingerprint"] == "recipe-1" + assert record["hardware_profile"] == {"gpu_count": 1} + assert record["hardware_profile_id"] == "hw-1" + assert record["software_profile"] == {"python": "3.12"} + assert record["software_profile_id"] == "sw-1" + assert record["environment_metadata"] == {"env": {"IMAGE_VERSION": "latest"}} + assert record["environment_fingerprint"] == "env-1" def test_baseline_eligibility_only_for_successful_scheduled_main(): diff --git a/fastvideo/tests/performance/test_dashboard_service.py b/fastvideo/tests/performance/test_dashboard_service.py index 174ef09fb..d6c3d0328 100644 --- a/fastvideo/tests/performance/test_dashboard_service.py +++ b/fastvideo/tests/performance/test_dashboard_service.py @@ -142,6 +142,47 @@ def test_build_latest_summary_separates_informational_threshold_crossing(): assert rows[0]["threshold_exceeded_metrics"] == ["latency"] assert rows[0]["failing_metrics"] == [] assert rows[0]["computed_regression_status"] == "pass" +def test_build_latest_summary_keeps_identity_cohorts_separate(): + records = [ + _record( + "2026-01-01T00:00:00+00:00", + "a" * 40, + 10.0, + 10.0, + recipe_fingerprint="recipe-a", + hardware_profile_id="hw-l40s", + software_profile_id="sw-cu130", + run_source="scheduled_main", + baseline_eligible=True, + ), + _record( + "2026-01-02T00:00:00+00:00", + "b" * 40, + 20.0, + 5.0, + recipe_fingerprint="recipe-b", + hardware_profile_id="hw-l40s", + software_profile_id="sw-cu130", + run_source="scheduled_main", + baseline_eligible=True, + ), + _record( + "2026-01-03T00:00:00+00:00", + "c" * 40, + 22.0, + 4.5, + recipe_fingerprint="recipe-b", + hardware_profile_id="hw-l40s", + software_profile_id="sw-cu130", + ), + ] + + rows = build_latest_summary(records, max_regression=0.05) + recipe_b_row = next(row for row in rows if row["recipe_fingerprint"] == "recipe-b") + + assert len(rows) == 2 + assert recipe_b_row["baseline_n"] == 1 + assert recipe_b_row["metrics"]["latency"]["baseline"] == 20.0 def test_filter_records_and_trends_preserve_metric_points(): @@ -218,3 +259,51 @@ def test_load_records_can_filter_baseline_eligible_records(tmp_path): "2026-01-02T00:00:00+00:00", "2026-01-03T00:00:00+00:00", } + + +def test_load_records_for_model_filters_identity_cohort(tmp_path): + model_dir = tmp_path / "wan" + model_dir.mkdir() + (model_dir / "matching.json").write_text( + """ + { + "model_id": "wan", + "gpu_type": "NVIDIA L40S", + "timestamp": "2026-01-01T00:00:00+00:00", + "success": true, + "baseline_eligible": true, + "recipe_fingerprint": "recipe-a", + "hardware_profile_id": "hw-l40s", + "software_profile_id": "sw-cu130" + } + """, + encoding="utf-8", + ) + (model_dir / "other_recipe.json").write_text( + """ + { + "model_id": "wan", + "gpu_type": "NVIDIA L40S", + "timestamp": "2026-01-02T00:00:00+00:00", + "success": true, + "baseline_eligible": true, + "recipe_fingerprint": "recipe-b", + "hardware_profile_id": "hw-l40s", + "software_profile_id": "sw-cu130" + } + """, + encoding="utf-8", + ) + + records = hf_store.load_records_for_model( + str(tmp_path), + "wan", + "NVIDIA L40S", + recipe_fingerprint="recipe-a", + hardware_profile_id="hw-l40s", + software_profile_id="sw-cu130", + baseline_eligible_only=True, + ) + + assert len(records) == 1 + assert records[0]["recipe_fingerprint"] == "recipe-a" diff --git a/fastvideo/tests/performance/test_identity.py b/fastvideo/tests/performance/test_identity.py new file mode 100644 index 000000000..5ea42e0f4 --- /dev/null +++ b/fastvideo/tests/performance/test_identity.py @@ -0,0 +1,213 @@ +# SPDX-License-Identifier: Apache-2.0 + +from copy import deepcopy + +from fastvideo.tests.performance.identity import ( + build_recipe_from_benchmark_config, + canonical_json, + environment_fingerprint, + environment_metadata, + hardware_profile, + hardware_profile_id, + recipe_fingerprint, + software_profile, + software_profile_id, +) + + +def _benchmark_config(): + return { + "benchmark_id": "wan-t2v-1.3b-2gpu", + "model": { + "model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + "model_short_name": "Wan2.1-T2V-1.3B", + }, + "init_kwargs": { + "num_gpus": 2, + "sp_size": 2, + "tp_size": 1, + "vae_sp": True, + "vae_tiling": True, + "text_encoder_precisions": ["fp32"], + }, + "generation_kwargs": { + "height": 480, + "width": 832, + "num_frames": 45, + "num_inference_steps": 4, + "guidance_scale": 3, + "embedded_cfg_scale": 6, + "seed": 1024, + "fps": 24, + "neg_prompt": "low quality", + }, + "test_prompts": ["A cinematic video."], + } + + +def _fingerprint(cfg, attention_backend="FLASH_ATTN"): + recipe = build_recipe_from_benchmark_config(cfg, attention_backend=attention_backend) + return recipe_fingerprint(recipe) + + +def test_canonical_json_is_stable_for_mapping_order(): + left = {"b": [2, 1], "a": {"z": "last", "m": "middle"}} + right = {"a": {"m": "middle", "z": "last"}, "b": [2, 1]} + + assert canonical_json(left) == canonical_json(right) + + +def test_same_config_produces_same_recipe_fingerprint(): + cfg = _benchmark_config() + + assert _fingerprint(cfg) == _fingerprint(deepcopy(cfg)) + + +def test_semantically_equivalent_sequence_forms_match(): + cfg = _benchmark_config() + equivalent = deepcopy(cfg) + equivalent["init_kwargs"]["text_encoder_precisions"] = tuple( + equivalent["init_kwargs"]["text_encoder_precisions"]) + + assert _fingerprint(cfg) == _fingerprint(equivalent) + + +def test_output_path_does_not_change_recipe_fingerprint(): + cfg = _benchmark_config() + with_output_path = deepcopy(cfg) + with_output_path["generation_kwargs"]["output_path"] = "/tmp/generated" + + assert _fingerprint(cfg) == _fingerprint(with_output_path) + + +def test_num_inference_steps_changes_recipe_fingerprint(): + cfg = _benchmark_config() + changed = deepcopy(cfg) + changed["generation_kwargs"]["num_inference_steps"] = 8 + + assert _fingerprint(cfg) != _fingerprint(changed) + + +def test_attention_backend_changes_recipe_fingerprint(): + cfg = _benchmark_config() + + assert _fingerprint(cfg, attention_backend="FLASH_ATTN") != _fingerprint( + cfg, attention_backend="TORCH_SDPA") + + +def test_precision_changes_recipe_fingerprint(): + cfg = _benchmark_config() + changed = deepcopy(cfg) + changed["init_kwargs"]["text_encoder_precisions"] = ["bf16"] + + assert _fingerprint(cfg) != _fingerprint(changed) + + +def test_dimensions_and_seed_change_recipe_fingerprint(): + cfg = _benchmark_config() + different_dimensions = deepcopy(cfg) + different_dimensions["generation_kwargs"]["width"] = 1024 + different_seed = deepcopy(cfg) + different_seed["generation_kwargs"]["seed"] = 2048 + + assert _fingerprint(cfg) != _fingerprint(different_dimensions) + assert _fingerprint(cfg) != _fingerprint(different_seed) + + +def test_distributed_layout_changes_recipe_fingerprint(): + cfg = _benchmark_config() + changed = deepcopy(cfg) + changed["init_kwargs"]["sp_size"] = 1 + changed["init_kwargs"]["tp_size"] = 2 + + assert _fingerprint(cfg) != _fingerprint(changed) + + +def test_software_profile_uses_major_minor_versions_for_id(): + profile_a = software_profile( + python_version="3.12.4", + torch_version="2.12.0+cu130", + cuda_version="13.0", + package_versions={ + "triton": "3.4.1", + "fastvideo_kernel": "0.3.2", + }, + ) + profile_b = software_profile( + python_version="3.12.5", + torch_version="2.12.1+cu130", + cuda_version="13.0", + package_versions={ + "triton": "3.4.9", + "fastvideo_kernel": "0.3.7", + }, + ) + + assert profile_a == { + "python": "3.12", + "pytorch": "2.12", + "cuda": "13.0", + "packages": { + "fastvideo_kernel": "0.3", + "triton": "3.4", + }, + } + assert software_profile_id(profile_a) == software_profile_id(profile_b) + + +def test_hardware_profile_id_uses_normalized_gpu_cohort(): + profile = hardware_profile( + num_gpus=2, + gpu_devices=[ + { + "name": "NVIDIA L40S", + "memory_bytes": 48 * 1024**3, + "compute_capability": "8.9", + }, + { + "name": "NVIDIA L40S", + "memory_gb": 48, + "compute_capability": "8.9", + }, + ], + interconnect="none_or_partial", + ) + + assert profile == { + "device_type": "cuda", + "gpu_count": 2, + "gpus": [ + { + "name": "NVIDIA L40S", + "memory_gb": 48, + "compute_capability": "8.9", + }, + { + "name": "NVIDIA L40S", + "memory_gb": 48, + "compute_capability": "8.9", + }, + ], + "interconnect": "none_or_partial", + } + assert hardware_profile_id(profile).startswith("hw-") + + +def test_environment_metadata_is_separate_audit_fingerprint(): + cfg = _benchmark_config() + recipe_hash = _fingerprint(cfg) + audit_a = environment_metadata( + env={"IMAGE_VERSION": "py3.12-cuda13.0.0"}, + package_versions={"triton": "3.4.1"}, + hardware={"device_type": "cuda", "gpu_count": 2}, + software={"python": "3.12", "pytorch": "2.12", "cuda": "13.0"}, + ) + audit_b = environment_metadata( + env={"IMAGE_VERSION": "py3.12-cuda13.0.1"}, + package_versions={"triton": "3.4.1"}, + hardware={"device_type": "cuda", "gpu_count": 2}, + software={"python": "3.12", "pytorch": "2.12", "cuda": "13.0"}, + ) + + assert recipe_hash == _fingerprint(cfg) + assert environment_fingerprint(audit_a) != environment_fingerprint(audit_b) diff --git a/fastvideo/tests/performance/test_inference_performance.py b/fastvideo/tests/performance/test_inference_performance.py index 59fdb0383..b025d6d6a 100644 --- a/fastvideo/tests/performance/test_inference_performance.py +++ b/fastvideo/tests/performance/test_inference_performance.py @@ -18,6 +18,16 @@ import pytest from fastvideo import VideoGenerator from fastvideo.logger import init_logger +from fastvideo.tests.performance.identity import ( + build_recipe_from_benchmark_config, + environment_fingerprint, + environment_metadata, + hardware_profile, + hardware_profile_id, + recipe_fingerprint, + software_profile, + software_profile_id, +) from fastvideo.worker.multiproc_executor import MultiprocExecutor logger = init_logger(__name__) @@ -227,6 +237,27 @@ def _write_results(results): logger.info("Performance results written to %s", filepath) +def _build_identity_fields(cfg, init_kwargs): + recipe = build_recipe_from_benchmark_config(cfg) + num_gpus = init_kwargs.get("num_gpus", cfg.get("run_config", {}).get("required_gpus", 1)) + hw_profile = hardware_profile(num_gpus=num_gpus) + sw_profile = software_profile() + env_metadata = environment_metadata( + hardware=hw_profile, + software=sw_profile, + ) + return { + "recipe": recipe, + "recipe_fingerprint": recipe_fingerprint(recipe), + "hardware_profile": hw_profile, + "hardware_profile_id": hardware_profile_id(hw_profile), + "software_profile": sw_profile, + "software_profile_id": software_profile_id(sw_profile), + "environment_metadata": env_metadata, + "environment_fingerprint": environment_fingerprint(env_metadata), + } + + # -- Test ------------------------------------------------------------------- def _run_benchmark(cfg): @@ -318,6 +349,7 @@ def _run_benchmark(cfg): "dit_time_s": _avg_component(all_component_times, "dit_time_s"), "vae_decode_time_s": _avg_component(all_component_times, "vae_decode_time_s"), + **_build_identity_fields(cfg, init_kwargs), } logger.info(