diff --git a/fastvideo/tests/modal/pr_test.py b/fastvideo/tests/modal/pr_test.py index b25606096..cc482efe8 100644 --- a/fastvideo/tests/modal/pr_test.py +++ b/fastvideo/tests/modal/pr_test.py @@ -15,8 +15,6 @@ image = (modal.Image.from_registry( image_tag, add_python="3.12" ).run_commands("rm -rf /FastVideo").apt_install( "cmake", "pkg-config", "build-essential", "curl", "libssl-dev", "ffmpeg" -).pip_install("huggingface_hub", "pandas", "plotly" -).run_commands("pip show plotly || echo 'plotly NOT installed'" ).run_commands( "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable" ).run_commands("echo 'source ~/.cargo/env' >> ~/.bashrc").env({ diff --git a/fastvideo/tests/performance/dashboard.py b/fastvideo/tests/performance/dashboard.py index ea052f589..8c4b38c87 100644 --- a/fastvideo/tests/performance/dashboard.py +++ b/fastvideo/tests/performance/dashboard.py @@ -1,9 +1,10 @@ import os import json +import glob import pandas as pd from datetime import datetime, timedelta, timezone import plotly.express as px -from huggingface_hub import HfApi, hf_hub_download +from huggingface_hub import snapshot_download HF_REPO_ID = os.environ.get("HF_REPO_ID", "FastVideo/performance-tracking") @@ -13,39 +14,67 @@ HF_TOKEN = os.environ.get("HF_API_KEY") # ----------------------------- # 1. Load HF JSON artifacts # ----------------------------- -def load_hf_dataset(days: int = 30) -> pd.DataFrame: - """ - Loads only recent benchmark JSONs from HF repo. - Avoids full history scan when possible. - """ - api = HfApi(token=HF_TOKEN) +# def load_hf_dataset(days: int = 30) -> pd.DataFrame: +# """ +# Loads only recent benchmark JSONs from HF repo. +# Avoids full history scan when possible. +# """ +# api = HfApi(token=HF_TOKEN) - files = api.list_repo_files( +# files = api.list_repo_files( +# repo_id=HF_REPO_ID, +# repo_type="dataset", +# ) + +# cutoff = datetime.now(timezone.utc) - timedelta(days=days) +# records = [] + +# for f in files: +# if not f.endswith(".json"): +# continue + +# path = hf_hub_download( +# repo_id=HF_REPO_ID, +# repo_type="dataset", +# filename=f, +# token=HF_TOKEN, +# ) + +# try: +# with open(path, "r") as fp: +# data = json.load(fp) + +# # fast skip: timestamp filter BEFORE dataframe creation +# ts = pd.to_datetime(data.get("timestamp"), utc=True) +# if ts < cutoff: +# continue + +# records.append(data) + +# except Exception: +# continue + +# return pd.DataFrame(records) + +def load_hf_dataset(days: int = 30) -> pd.DataFrame: + cutoff = datetime.now(timezone.utc) - timedelta(days=days) + + local_dir = snapshot_download( repo_id=HF_REPO_ID, repo_type="dataset", + token=HF_TOKEN, + allow_patterns="*.json" ) - cutoff = datetime.now(timezone.utc) - timedelta(days=days) records = [] - for f in files: - if not f.endswith(".json"): - continue - - path = hf_hub_download( - repo_id=HF_REPO_ID, - repo_type="dataset", - filename=f, - token=HF_TOKEN, - ) - + for path in glob.glob(f"{local_dir}/**/*.json", recursive=True): try: with open(path, "r") as fp: data = json.load(fp) - # fast skip: timestamp filter BEFORE dataframe creation - ts = pd.to_datetime(data.get("timestamp"), utc=True) - if ts < cutoff: + ts = pd.to_datetime(data.get("timestamp"), utc=True, errors="coerce") + if pd.isna(ts) or ts < cutoff: continue records.append(data) @@ -55,7 +84,6 @@ def load_hf_dataset(days: int = 30) -> pd.DataFrame: return pd.DataFrame(records) - # ----------------------------- # 2. Normalize schema (your CI format) # ----------------------------- @@ -79,7 +107,7 @@ def normalize(df: pd.DataFrame) -> pd.DataFrame: # 3. Grouping (your requested design) # ----------------------------- def group_data(df: pd.DataFrame): - keys = ["benchmark_id", "model_id", "gpu_type", "config_id"] + keys = ["model_id", "gpu_type", "config_id"] return df.groupby(keys, dropna=False) @@ -94,9 +122,9 @@ def build_plots(df: pd.DataFrame): for name, g in grouped: g = g.sort_values("timestamp") - benchmark_id, model_id, gpu_type, config_id = name + model_id, gpu_type, config_id = name - title = f"{benchmark_id} | {model_id} | {gpu_type} | {config_id}" + title = f"{model_id} | {gpu_type} | {config_id}" fig = px.line( g, @@ -146,17 +174,30 @@ def annotate_buildkite(html: str): # 6. Buildkite upload # ----------------------------- -def upload_artifact(path: str): - if not os.environ.get("BUILDKITE"): - return +# def upload_artifact(path: str): +# if not os.environ.get("BUILDKITE"): +# return +# import subprocess + +# subprocess.run( +# ["buildkite-agent", "artifact", "upload", path], +# check=False, +# ) + +def upload_artifact(path: str): + import shutil import subprocess + if shutil.which("buildkite-agent") is None: + print("buildkite-agent not found, skipping artifact upload") + return + subprocess.run( ["buildkite-agent", "artifact", "upload", path], - check=False, + check=True, ) - + # ----------------------------- # 7. Main # ----------------------------- diff --git a/pyproject.toml b/pyproject.toml index 5479aff4c..db01f2292 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -124,6 +124,8 @@ test = [ "av", "pytorch-msssim==1.0.0", "pytest", + "pandas", + "plotly", ] dev = [ "fastvideo[lint]", "fastvideo[test]", ]