[feat] eval: add EvalResult aggregate type; ignore _assets/
Two small additions paired with the in-progress eval orchestration: - types.py: EvalResult dataclass with summary/per_video fields plus from_raw / save / print helpers. Used by scripts and any future callers that aggregate per-sample MetricResults into a corpus-level summary (e.g. scripts/eval/run_vbench_e2e.py). - .gitignore: exclude fastvideo/eval/_assets/ which holds calibration data, downloaded videos, and run dumps used by the synthetic-flow evaluator that we don't want to track. Authored by dataloader.
This commit is contained in:
@@ -85,6 +85,9 @@ docs/distillation/examples/
|
||||
dmd_t2v_output/
|
||||
preprocess_output_text/
|
||||
|
||||
# Synthetic-flow eval assets (calibration data, downloaded videos, run dumps)
|
||||
fastvideo/eval/_assets/
|
||||
|
||||
# Next.js / Node artifacts under ui/: see ui/.gitignore
|
||||
|
||||
.claude/
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
@@ -14,3 +16,36 @@ class MetricResult:
|
||||
name: str
|
||||
score: float | None
|
||||
details: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvalResult:
|
||||
"""Aggregate output of :meth:`Evaluator.evaluate_dataset`.
|
||||
|
||||
``summary`` maps metric name → mean score and includes a synthetic
|
||||
``"overall_average"`` key. ``per_video`` is one row per scored video
|
||||
with at least ``prompt``, ``video``, and ``scores`` fields.
|
||||
"""
|
||||
summary: dict[str, float]
|
||||
per_video: list[dict[str, Any]]
|
||||
|
||||
@classmethod
|
||||
def from_raw(cls, by_metric: dict[str, list[float]],
|
||||
per_video: list[dict[str, Any]]) -> EvalResult:
|
||||
summary = {m: sum(v) / len(v) for m, v in by_metric.items() if v}
|
||||
if summary:
|
||||
summary["overall_average"] = sum(summary.values()) / len(summary)
|
||||
return cls(summary=summary, per_video=per_video)
|
||||
|
||||
def save(self, path: str | Path) -> None:
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(
|
||||
{"summary": self.summary, "per_video": self.per_video},
|
||||
indent=2))
|
||||
|
||||
def print(self) -> None:
|
||||
print()
|
||||
print("== summary ==")
|
||||
for k, v in sorted(self.summary.items()):
|
||||
print(f" {k:42s} {v:.4f}")
|
||||
|
||||
Reference in New Issue
Block a user