[feat] eval: add EvalResult aggregate type; ignore _assets/

Two small additions paired with the in-progress eval orchestration:

- types.py: EvalResult dataclass with summary/per_video fields plus
  from_raw / save / print helpers. Used by scripts and any future
  callers that aggregate per-sample MetricResults into a corpus-level
  summary (e.g. scripts/eval/run_vbench_e2e.py).

- .gitignore: exclude fastvideo/eval/_assets/ which holds calibration
  data, downloaded videos, and run dumps used by the synthetic-flow
  evaluator that we don't want to track.

Authored by dataloader.
This commit is contained in:
shaoxiongduan
2026-05-06 00:48:19 +00:00
parent b07cc3f9d4
commit 33efe7a673
2 changed files with 38 additions and 0 deletions
+3
View File
@@ -85,6 +85,9 @@ docs/distillation/examples/
dmd_t2v_output/
preprocess_output_text/
# Synthetic-flow eval assets (calibration data, downloaded videos, run dumps)
fastvideo/eval/_assets/
# Next.js / Node artifacts under ui/: see ui/.gitignore
.claude/
+35
View File
@@ -1,6 +1,8 @@
from __future__ import annotations
import json
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
@@ -14,3 +16,36 @@ class MetricResult:
name: str
score: float | None
details: dict[str, Any] = field(default_factory=dict)
@dataclass
class EvalResult:
"""Aggregate output of :meth:`Evaluator.evaluate_dataset`.
``summary`` maps metric name → mean score and includes a synthetic
``"overall_average"`` key. ``per_video`` is one row per scored video
with at least ``prompt``, ``video``, and ``scores`` fields.
"""
summary: dict[str, float]
per_video: list[dict[str, Any]]
@classmethod
def from_raw(cls, by_metric: dict[str, list[float]],
per_video: list[dict[str, Any]]) -> EvalResult:
summary = {m: sum(v) / len(v) for m, v in by_metric.items() if v}
if summary:
summary["overall_average"] = sum(summary.values()) / len(summary)
return cls(summary=summary, per_video=per_video)
def save(self, path: str | Path) -> None:
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(
{"summary": self.summary, "per_video": self.per_video},
indent=2))
def print(self) -> None:
print()
print("== summary ==")
for k, v in sorted(self.summary.items()):
print(f" {k:42s} {v:.4f}")