# SPDX-License-Identifier: Apache-2.0 import os import sys from html import escape from datetime import datetime import plotly.express as px import pandas as pd try: from fastvideo.performance.hf_store import load_as_dataframe, sync_from_hf except ImportError: repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")) if repo_root not in sys.path: sys.path.insert(0, repo_root) from fastvideo.performance.hf_store import load_as_dataframe, sync_from_hf TRACKING_ROOT = os.environ.get( "PERFORMANCE_TRACKING_ROOT", "/tmp/perf-tracking", ) PERF_REPORTS_DIR = os.environ.get("PERF_REPORTS_DIR", "/root/data/perf_reports") METRICS = ( "latency", "throughput", "memory", "text_encoder_time_s", "dit_time_s", "vae_decode_time_s", ) COMPARISON_COHORT_KEYS = ( "workload_id", "variant_id", "benchmark_version", "recipe_fingerprint", "hardware_profile_id", "software_profile_id", ) GROUP_KEYS = ("_cohort_schema", "_cohort_model_id", "_cohort_gpu_type", *COMPARISON_COHORT_KEYS) def _cohort_value(value: object) -> str: if value is None: return "" try: if pd.isna(value): return "" except (TypeError, ValueError): pass return str(value) def _display_value(value: object) -> str: return _cohort_value(value) or "legacy" def _short_value(value: object) -> str: text = _display_value(value) if text == "legacy" or len(text) <= 14: return text return text[:12] def _cohort_title(record: object) -> str: workload = _display_value(record.get("workload_id")) variant = _display_value(record.get("variant_id")) version = _display_value(record.get("benchmark_version")) version_label = version if version == "legacy" else f"v{version}" return f"{workload} / {variant} / {version_label}" def _cohort_detail(record: object) -> str: return " | ".join(( f"recipe {_short_value(record.get('recipe_fingerprint'))}", _short_value(record.get("hardware_profile_id")), _short_value(record.get("software_profile_id")), )) def _group_record(record: object) -> dict[str, str]: return {key: _cohort_value(record.get(key)) for key in ("model_id", "gpu_type", *COMPARISON_COHORT_KEYS)} def _has_complete_v2_identity(record: object) -> bool: return all(_cohort_value(record.get(key)) for key in COMPARISON_COHORT_KEYS) def _dashboard_frame(df: pd.DataFrame) -> pd.DataFrame: dashboard_df = df.copy() for key in ("model_id", "gpu_type", *COMPARISON_COHORT_KEYS): if key not in dashboard_df.columns: dashboard_df[key] = "" dashboard_df[key] = dashboard_df[key].map(_cohort_value) uses_v2_identity = dashboard_df.apply(_has_complete_v2_identity, axis=1) dashboard_df["_cohort_schema"] = uses_v2_identity.map({True: "v2", False: "legacy"}) dashboard_df["_cohort_model_id"] = dashboard_df["model_id"].where(~uses_v2_identity, "") dashboard_df["_cohort_gpu_type"] = dashboard_df["gpu_type"].where(~uses_v2_identity, "") return dashboard_df # ----------------------------- # 1. Grouping # ----------------------------- def group_data(df: pd.DataFrame): # Group by the same comparison cohort used by baseline gating. return _dashboard_frame(df).groupby(list(GROUP_KEYS), dropna=False) # ----------------------------- # 2. Plot builder # ----------------------------- def build_plots(df: pd.DataFrame) -> tuple[list, list[dict[str, object]]]: figs = [] skipped_metrics: list[dict[str, object]] = [] for _group_key, g in group_data(df): g = g.sort_values("timestamp") display_record = g.iloc[-1] model_id = display_record["model_id"] gpu_type = display_record["gpu_type"] group_record = _group_record(display_record) cohort_title = _cohort_title(display_record) cohort_detail = _cohort_detail(display_record) # One chart per metric so the y-axes aren't on wildly different scales for metric in METRICS: if metric not in g.columns: skipped_metrics.append({ **group_record, "cohort": cohort_title, "cohort_detail": cohort_detail, "metric": metric, "reason": "column missing from loaded records", "records": len(g), "non_null": 0, }) continue non_null = int(g[metric].notna().sum()) if non_null == 0: skipped_metrics.append({ **group_record, "cohort": cohort_title, "cohort_detail": cohort_detail, "metric": metric, "reason": "no non-null values in loaded records", "records": len(g), "non_null": non_null, }) continue fig = px.line( g, x="timestamp", y=metric, markers=True, hover_data=["model_id", "gpu_type", "config_id", "commit_sha", *COMPARISON_COHORT_KEYS], title=f"{model_id} | {gpu_type} | {cohort_title} | {cohort_detail} | {metric}", labels={ "timestamp": "Time", metric: metric }, ) figs.append(fig) return figs, skipped_metrics def render_skipped_metrics(skipped_metrics: list[dict[str, object]]) -> str: if not skipped_metrics: return "" rows = [ "
| Model | GPU | Cohort | Metric | " "Records | Non-null | Reason |
|---|---|---|---|---|---|---|
| {escape(str(item['model_id']))} | " f"{escape(str(item['gpu_type']))} | " f"{escape(str(item['cohort']))}{escape(str(item['cohort_detail']))} | "
f"{escape(str(item['metric']))} | " f"{item['records']} | " f"{item['non_null']} | " f"{escape(str(item['reason']))} | " "