From d07a7691feb5dd2138fc2604f7ebe9071acd90fe Mon Sep 17 00:00:00 2001 From: shaoxiongduan Date: Tue, 5 May 2026 23:45:29 +0000 Subject: [PATCH] [refactor] eval: drop EvalRunner; flat script + helpers in eval.io MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Match FastVideo's existing depth: VideoGenerator is the top-level inference object with no Runner above it; loops live in scripts. Eval should mirror that — Evaluator is the top-level scoring object, EvalWorker × N is the layer below, and end-to-end pipelines (prompts → generate → score) are scripts, not classes. Removed: - fastvideo/eval/runner.py (EvalRunner + classmethod constructors). - EvalRunner export from fastvideo.eval. Added: - fastvideo/eval/io/paths.py with sanitize_prompt, default_filename, glob_videos, build_eval_kwargs as free functions. The reusable bits of the runner survive; the class wrapping them does not. Rewritten: - scripts/eval/run_vbench_e2e.py is now top-to-bottom procedural: parse_args → dataset → optional generate() loop → optional score() loop. Reads in one pass; no classmethod indirection to chase. Parity verified on fs-mbz-gpu-085 against runs/vbench_smoke/videos/ (2 videos × 3 metrics, dimensions=subject_consistency): - 1-GPU run (results_v4.json) is byte-identical to v3 baseline. - 2-GPU run (results_v4_2gpu.json) is byte-identical to v3 baseline (order-preserving fan-out via Evaluator.evaluate(samples=[...])). --- fastvideo/eval/__init__.py | 2 - fastvideo/eval/io/__init__.py | 13 +- fastvideo/eval/io/paths.py | 64 ++++++ fastvideo/eval/runner.py | 383 --------------------------------- scripts/eval/run_vbench_e2e.py | 162 ++++++++++---- 5 files changed, 197 insertions(+), 427 deletions(-) create mode 100644 fastvideo/eval/io/paths.py delete mode 100644 fastvideo/eval/runner.py diff --git a/fastvideo/eval/__init__.py b/fastvideo/eval/__init__.py index 5300b6213..1e9200e05 100644 --- a/fastvideo/eval/__init__.py +++ b/fastvideo/eval/__init__.py @@ -27,7 +27,6 @@ from fastvideo.eval.metrics.base import BaseMetric # noqa: E402 from fastvideo.eval.registry import register, list_metrics, get_metric # noqa: E402 from fastvideo.eval.api import evaluate # noqa: E402 from fastvideo.eval.evaluator import Evaluator, create_evaluator # noqa: E402 -from fastvideo.eval.runner import EvalRunner # noqa: E402 # Trigger metric auto-discovery import fastvideo.eval.metrics # noqa: F401, E402 @@ -36,7 +35,6 @@ __all__ = [ "evaluate", "Evaluator", "create_evaluator", - "EvalRunner", "MetricResult", "EvalResult", "BaseMetric", diff --git a/fastvideo/eval/io/__init__.py b/fastvideo/eval/io/__init__.py index 5b3daff48..9eaedcf18 100644 --- a/fastvideo/eval/io/__init__.py +++ b/fastvideo/eval/io/__init__.py @@ -1,3 +1,12 @@ -from fastvideo.eval.io.video import load_video, extract_frames +from fastvideo.eval.io.paths import (build_eval_kwargs, default_filename, + glob_videos, sanitize_prompt) +from fastvideo.eval.io.video import extract_frames, load_video -__all__ = ["load_video", "extract_frames"] +__all__ = [ + "load_video", + "extract_frames", + "sanitize_prompt", + "default_filename", + "glob_videos", + "build_eval_kwargs", +] diff --git a/fastvideo/eval/io/paths.py b/fastvideo/eval/io/paths.py new file mode 100644 index 000000000..37f9970da --- /dev/null +++ b/fastvideo/eval/io/paths.py @@ -0,0 +1,64 @@ +"""Filesystem helpers shared by eval scripts. + +Provides the prompt-sanitization, default filename convention, and +``(row, video_path) → eval-kwargs`` builder. Free functions, not a +class — :class:`fastvideo.eval.Evaluator` is the only stateful object +in the eval surface; loops live in user scripts. +""" +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + +# Filesystem-unsafe characters mirrored from VideoGenerator's output-path +# sanitizer so on-disk filenames match what the generator writes. +_INVALID_CHARS = re.compile(r'[\\/:*?"<>|]') + + +def sanitize_prompt(prompt: str, max_len: int = 100) -> str: + """Prompt → safe filename stem.""" + s = _INVALID_CHARS.sub("", prompt[:max_len]).strip().strip(".") + return re.sub(r"\s+", " ", s) or "output" + + +def default_filename(row: dict, idx: int, ext: str = ".mp4") -> str: + """``-.mp4`` — VBench-style.""" + return f"{sanitize_prompt(row['prompt'])}-{idx}{ext}" + + +def glob_videos(videos_dir: Path, row: dict, ext: str = ".mp4") -> list[Path]: + """Find every generated video for *row*, sorted by trailing ``-``.""" + pattern = f"{sanitize_prompt(row['prompt'])}-*{ext}" + files = list(videos_dir.glob(pattern)) + + def _idx(p: Path) -> int: + try: + return int(p.stem.rsplit("-", 1)[1]) + except (IndexError, ValueError): + return -1 + + return sorted(files, key=_idx) + + +def build_eval_kwargs(row: dict, video_path: Path, *, + fps: float = 24.0) -> dict[str, Any]: + """Build evaluator kwargs from a sample row + a video on disk. + + Loads the video as ``(T,C,H,W)`` and adds the leading batch dim. + Forwards ``prompt`` (as ``text_prompt=[prompt]``) and + ``auxiliary_info`` (as ``[aux]``) when present on the row. + """ + from fastvideo.eval.io.video import load_video + + video = load_video(str(video_path)) # (T, C, H, W) in [0, 1] + kwargs: dict[str, Any] = { + "video": video.unsqueeze(0), # (1, T, C, H, W) + "fps": fps, + } + if "prompt" in row: + kwargs["text_prompt"] = [row["prompt"]] + aux = row.get("auxiliary_info") + if aux: + kwargs["auxiliary_info"] = [aux] + return kwargs diff --git a/fastvideo/eval/runner.py b/fastvideo/eval/runner.py deleted file mode 100644 index da309ce88..000000000 --- a/fastvideo/eval/runner.py +++ /dev/null @@ -1,383 +0,0 @@ -"""End-to-end eval orchestrator. - -:class:`EvalRunner` is the top-level glue that connects a prompt dataset, -an optional :class:`fastvideo.VideoGenerator`, and an -:class:`fastvideo.eval.Evaluator` into a single generate-then-score -pipeline. It owns: - -* the videos directory and filename convention, -* per-run config (fps, seed, n_samples override, generation kwargs), -* the generation loop and its manifest, -* loading videos back from disk into evaluator-ready kwargs. - -The Evaluator below it stays focused on scoring tensors. Multi-GPU -fan-out is delegated to the Evaluator (which holds N -:class:`fastvideo.eval.worker.EvalWorker` replicas internally) — the -runner just drives the work. - -Three named constructors cover the common shapes:: - - # benchmark: prompts → generate → score - EvalRunner.from_dataset(dataset, videos_dir, generator=gen, - num_gpus=4).run() - - # score-only: prompts → score existing videos - EvalRunner.from_dataset(dataset, videos_dir, num_gpus=4).score() - - # quick multi-GPU scoring of a flat folder - EvalRunner.from_videos(["a.mp4", "b.mp4"], num_gpus=2).score() - - # power user: pre-built eval kwargs - EvalRunner.from_samples(my_kwargs_list, num_gpus=4).score() -""" -from __future__ import annotations - -import json -import logging -import re -from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Iterable - -from fastvideo.eval.evaluator import Evaluator, create_evaluator -from fastvideo.eval.types import EvalResult - -if TYPE_CHECKING: - from fastvideo.eval.datasets.base import PromptDataset - -log = logging.getLogger(__name__) - -# Filesystem-unsafe characters mirrored from VideoGenerator's output-path -# sanitizer so on-disk filenames match what the generator writes. -_INVALID_CHARS = re.compile(r'[\\/:*?"<>|]') - - -def sanitize_prompt(prompt: str, max_len: int = 100) -> str: - """Default prompt → safe filename stem.""" - s = _INVALID_CHARS.sub("", prompt[:max_len]).strip().strip(".") - return re.sub(r"\s+", " ", s) or "output" - - -def _default_filename(row: dict, idx: int) -> str: - """``-.mp4`` — VBench-style.""" - return f"{sanitize_prompt(row['prompt'])}-{idx}.mp4" - - -def _default_eval_kwargs(row: dict, video_path: Path, *, - fps: float) -> dict[str, Any]: - """Build evaluator kwargs from a sample row + a video on disk.""" - from fastvideo.eval.io.video import load_video - - video = load_video(str(video_path)) # (T, C, H, W) in [0, 1] - kwargs: dict[str, Any] = { - "video": video.unsqueeze(0), # (1, T, C, H, W) - "fps": fps, - } - if "prompt" in row: - kwargs["text_prompt"] = [row["prompt"]] - aux = row.get("auxiliary_info") - if aux: - kwargs["auxiliary_info"] = [aux] - return kwargs - - -class EvalRunner: - """Composes ``(dataset?, generator?, evaluator)`` for an eval run. - - Construct via one of :meth:`from_dataset`, :meth:`from_videos`, or - :meth:`from_samples`. Use the run methods (:meth:`generate`, - :meth:`score`, :meth:`run`) to drive work. - - Owned config: - videos_dir, fps, seed, n_samples (override), gen_kwargs, - filename_fn (row, idx → name), eval_kwargs_fn (row, path → kwargs). - - The evaluator is either supplied directly or built lazily from - ``metrics`` + ``num_gpus``. - """ - - def __init__( - self, - *, - videos_dir: str | Path | None, - evaluator: Evaluator, - dataset: "PromptDataset | None" = None, - generator: Any = None, # VideoGenerator - sample_rows: list[dict] | None = None, # pre-built (from_samples) - fps: float = 24.0, - seed: int = 0, - n_samples: int | None = None, - gen_kwargs: dict | None = None, - filename_fn: Callable[[dict, int], str] | None = None, - eval_kwargs_fn: Callable[[dict, Path], dict[str, Any]] | None = None, - ) -> None: - self.videos_dir = Path(videos_dir) if videos_dir is not None else None - self.evaluator = evaluator - self.dataset = dataset - self.generator = generator - self._sample_rows = sample_rows - self.fps = fps - self.seed = seed - self.n_samples = n_samples - self.gen_kwargs = dict(gen_kwargs or {}) - self.filename_fn = filename_fn or _default_filename - self._eval_kwargs_fn = eval_kwargs_fn - - # ----------------------------------------------------------------- ctors - - @classmethod - def from_dataset( - cls, - dataset: "PromptDataset", - videos_dir: str | Path, - *, - evaluator: Evaluator | None = None, - generator: Any = None, - metrics: list[str] | str = "all", - num_gpus: int = 1, - fps: float = 24.0, - seed: int = 0, - n_samples: int | None = None, - gen_kwargs: dict | None = None, - filename_fn: Callable[[dict, int], str] | None = None, - eval_kwargs_fn: Callable[[dict, Path], dict[str, Any]] | None = None, - ) -> "EvalRunner": - """Run against a :class:`PromptDataset`. With *generator*, supports - :meth:`generate` + :meth:`score` (and :meth:`run`); without it, - only :meth:`score` (videos must already be on disk under - *videos_dir* matching *filename_fn*).""" - ev = evaluator or create_evaluator(metrics=metrics, num_gpus=num_gpus) - return cls( - videos_dir=videos_dir, - evaluator=ev, - dataset=dataset, - generator=generator, - fps=fps, - seed=seed, - n_samples=n_samples, - gen_kwargs=gen_kwargs, - filename_fn=filename_fn, - eval_kwargs_fn=eval_kwargs_fn, - ) - - @classmethod - def from_videos( - cls, - video_paths: Iterable[str | Path], - *, - prompts: list[str] | None = None, - evaluator: Evaluator | None = None, - metrics: list[str] | str = "all", - num_gpus: int = 1, - fps: float = 24.0, - eval_kwargs_fn: Callable[[dict, Path], dict[str, Any]] | None = None, - ) -> "EvalRunner": - """Score a flat list of video files. Optional *prompts* (same - length) attach a text prompt to each video for prompt-using - metrics.""" - ev = evaluator or create_evaluator(metrics=metrics, num_gpus=num_gpus) - paths = [Path(p) for p in video_paths] - if prompts is not None and len(prompts) != len(paths): - raise ValueError( - f"prompts length {len(prompts)} != video_paths length " - f"{len(paths)}") - - # Build a synthetic row per video so the score loop is uniform. - rows: list[dict] = [] - for i, p in enumerate(paths): - row: dict = {"_video_path": p} - if prompts is not None: - row["prompt"] = prompts[i] - rows.append(row) - - return cls( - videos_dir=None, - evaluator=ev, - sample_rows=rows, - fps=fps, - eval_kwargs_fn=eval_kwargs_fn, - ) - - @classmethod - def from_samples( - cls, - samples: list[dict[str, Any]], - *, - evaluator: Evaluator | None = None, - metrics: list[str] | str = "all", - num_gpus: int = 1, - ) -> "EvalRunner": - """Score a list of pre-built evaluator-kwargs dicts. Each dict is - passed verbatim to :meth:`Evaluator.evaluate` via the list-form - API. Use this when you already have video tensors in memory.""" - ev = evaluator or create_evaluator(metrics=metrics, num_gpus=num_gpus) - return cls( - videos_dir=None, - evaluator=ev, - sample_rows=[{"_eval_kwargs": s} for s in samples], - ) - - # ------------------------------------------------------------- generate - - def generate(self, *, skip_existing: bool = True) -> dict[str, list[Path]]: - """Drive the generator over the dataset. Returns ``{prompt: [paths]}``. - - Requires ``dataset`` and ``generator`` to have been supplied. - Writes a ``manifest.json`` alongside the videos. - """ - if self.dataset is None or self.generator is None: - raise RuntimeError( - "generate() requires both dataset and generator. Use " - "from_dataset(dataset, videos_dir, generator=...).") - if self.videos_dir is None: - raise RuntimeError("generate() requires videos_dir.") - - self.videos_dir.mkdir(parents=True, exist_ok=True) - total = sum((self.n_samples or row.get("n_samples", 1)) - for row in self.dataset) - log.info("[gen] %d prompts -> %d videos in %s", - len(self.dataset), total, self.videos_dir) - - global_idx = 0 - manifest: dict[str, list[str]] = {} - for row in self.dataset: - n = self.n_samples or row.get("n_samples", 1) - paths: list[str] = [] - for k in range(n): - target = self.videos_dir / self.filename_fn(row, k) - paths.append(str(target)) - seed = self.seed + global_idx - global_idx += 1 - if skip_existing and target.exists(): - log.info("[gen %d/%d] SKIP exists: %s", - global_idx, total, target.name) - continue - log.info("[gen %d/%d] prompt=%r seed=%d -> %s", - global_idx, total, row["prompt"][:60], seed, - target.name) - self.generator.generate_video( - prompt=row["prompt"], - output_path=str(target), - save_video=True, - seed=seed, - **self.gen_kwargs, - ) - if target.exists(): - log.info("[gen %d/%d] OK %.1f MB", - global_idx, total, target.stat().st_size / 1e6) - else: - log.warning("[gen %d/%d] missing after generate: %s", - global_idx, total, target) - manifest[row["prompt"]] = paths - (self.videos_dir / "manifest.json").write_text( - json.dumps(manifest, indent=2)) - return {p: [Path(x) for x in v] for p, v in manifest.items()} - - # ----------------------------------------------------------------- score - - def score(self, *, verbose: bool = True) -> EvalResult: - """Score every (eval_kwargs, meta) job for this runner. - - Iteration order: input order. Multi-GPU fan-out happens inside - :meth:`Evaluator.evaluate` when called with the list form. - """ - jobs = list(self._iter_jobs()) - if not jobs: - log.warning("[score] no jobs found") - return EvalResult.from_raw({}, []) - - kwargs_list = [j[0] for j in jobs] - metas = [j[1] for j in jobs] - - # Single dispatch — Evaluator handles the multi-GPU fan-out. - results = self.evaluator.evaluate(kwargs_list) - - by_metric: dict[str, list[float]] = {} - per_video: list[dict] = [] - for meta, res in zip(metas, results, strict=True): - scores: dict[str, float] = {} - for name, mr in res.items(): - if mr is not None and mr.score is not None: - scores[name] = float(mr.score) - by_metric.setdefault(name, []).append(scores[name]) - per_video.append({**meta, "scores": scores}) - if verbose: - log.info("[score] %s -> %s", - meta.get("video", ""), - {k: round(v, 4) for k, v in scores.items()}) - if verbose: - log.info("[score] done. videos_scored=%d", len(per_video)) - return EvalResult.from_raw(by_metric, per_video) - - def run(self) -> EvalResult: - """Generate then score. Convenience for full benchmark runs.""" - self.generate() - return self.score() - - # ----------------------------------------------------------- iteration - - def _iter_jobs(self): - """Yield ``(eval_kwargs, meta)`` pairs for every video to score.""" - if self._sample_rows is not None: - for row in self._sample_rows: - # from_samples: kwargs are pre-built. - if "_eval_kwargs" in row: - kw = row["_eval_kwargs"] - meta = {"video": ""} - yield kw, meta - continue - # from_videos: load the file, build kwargs. - vp: Path = row["_video_path"] - base_row = {k: v for k, v in row.items() - if not k.startswith("_")} - kw = self._build_eval_kwargs(base_row, vp) - meta = {"video": str(vp), "prompt": base_row.get("prompt")} - yield kw, meta - return - - if self.dataset is None or self.videos_dir is None: - raise RuntimeError( - "score() requires either a dataset+videos_dir or " - "from_videos/from_samples construction.") - - for row in self.dataset: - for vp in self._glob_row_videos(row): - meta = { - "prompt": row.get("prompt"), - "video": str(vp), - "dimensions": list(row.get("dimensions", [])), - } - yield self._build_eval_kwargs(row, vp), meta - - def _glob_row_videos(self, row: dict) -> list[Path]: - """Find every generated video for *row* under videos_dir, sorted.""" - assert self.videos_dir is not None - # Strip trailing -.mp4 from the filename_fn(row, 0) output to - # get a glob pattern. Falls back to the row's own k=0..k=N enumeration - # if the convention isn't ``-.mp4``. - first = self.filename_fn(row, 0) - m = re.match(r"^(?P.*)-0(?P\.[^.]+)$", first) - if m: - pattern = f"{m['stem']}-*{m['ext']}" - files = list(self.videos_dir.glob(pattern)) - - def _idx(p: Path) -> int: - try: - return int(p.stem.rsplit("-", 1)[1]) - except (IndexError, ValueError): - return -1 - - return sorted(files, key=_idx) - - # Non-indexed convention: enumerate explicitly. - n = self.n_samples or row.get("n_samples", 1) - out: list[Path] = [] - for k in range(n): - p = self.videos_dir / self.filename_fn(row, k) - if p.exists(): - out.append(p) - return out - - def _build_eval_kwargs(self, row: dict, vp: Path) -> dict[str, Any]: - if self._eval_kwargs_fn is not None: - return self._eval_kwargs_fn(row, vp) - return _default_eval_kwargs(row, vp, fps=self.fps) diff --git a/scripts/eval/run_vbench_e2e.py b/scripts/eval/run_vbench_e2e.py index 4037c152c..2f3534690 100644 --- a/scripts/eval/run_vbench_e2e.py +++ b/scripts/eval/run_vbench_e2e.py @@ -1,5 +1,9 @@ """End-to-end VBench evaluation: prompt → generate → score. +Plain script. No runner class — just :class:`fastvideo.VideoGenerator`, +:class:`fastvideo.eval.Evaluator`, and a :class:`PromptDataset` glued by +two for-loops. + # Generate + score: python scripts/eval/run_vbench_e2e.py \\ --model \\ @@ -24,86 +28,164 @@ from __future__ import annotations import argparse +import json import logging from pathlib import Path logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +log = logging.getLogger("vbench-e2e") -def main() -> None: +def parse_args() -> argparse.Namespace: ap = argparse.ArgumentParser( __doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--dimensions", default="all", - help="Comma-separated VBench dimensions, or 'all'.") + ap.add_argument("--dimensions", default="all") ap.add_argument("--max-prompts", type=int, default=None) ap.add_argument("--videos-dir", required=True, type=Path) ap.add_argument("--results-json", type=Path, default=None) ap.add_argument("--metrics", default="vbench") - ap.add_argument("--model", default=None, - help="HF id / path for VideoGenerator.from_pretrained.") - ap.add_argument("--n-samples", type=int, default=None) + ap.add_argument("--model", default=None) + ap.add_argument("--n-samples", type=int, default=None, + help="Override per-prompt n_samples (default: per-row).") ap.add_argument("--seed", type=int, default=0) ap.add_argument("--height", type=int, default=None) ap.add_argument("--width", type=int, default=None) ap.add_argument("--num-frames", type=int, default=None) ap.add_argument("--num-inference-steps", type=int, default=None) - ap.add_argument("--fps", type=float, default=25.0) ap.add_argument("--num-gpus", type=int, default=1) ap.add_argument("--skip-generation", action="store_true") ap.add_argument("--skip-scoring", action="store_true") - args = ap.parse_args() + return ap.parse_args() - from fastvideo.eval import EvalRunner + +def parse_metrics(spec: str) -> str | list[str]: + spec = spec.strip() + if spec == "all": + return "all" + if "," in spec: + return [m.strip() for m in spec.split(",") if m.strip()] + return spec + + +def generate(dataset, generator, videos_dir: Path, *, base_seed: int, + n_override: int | None, gen_kwargs: dict) -> None: + from fastvideo.eval.io import default_filename + + videos_dir.mkdir(parents=True, exist_ok=True) + total = sum((n_override or row.get("n_samples", 1)) for row in dataset) + log.info("[gen] %d prompts -> %d videos in %s", + len(dataset), total, videos_dir) + + global_idx = 0 + manifest: dict[str, list[str]] = {} + for row in dataset: + n = n_override or row.get("n_samples", 1) + paths: list[str] = [] + for k in range(n): + target = videos_dir / default_filename(row, k) + paths.append(str(target)) + seed = base_seed + global_idx + global_idx += 1 + if target.exists(): + log.info("[gen %d/%d] SKIP exists: %s", + global_idx, total, target.name) + continue + log.info("[gen %d/%d] prompt=%r seed=%d -> %s", + global_idx, total, row["prompt"][:60], seed, + target.name) + generator.generate_video( + prompt=row["prompt"], + output_path=str(target), + save_video=True, + seed=seed, + **gen_kwargs, + ) + if target.exists(): + log.info("[gen %d/%d] OK %.1f MB", + global_idx, total, target.stat().st_size / 1e6) + else: + log.warning("[gen %d/%d] missing after generate: %s", + global_idx, total, target) + manifest[row["prompt"]] = paths + (videos_dir / "manifest.json").write_text(json.dumps(manifest, indent=2)) + + +def score(dataset, evaluator, videos_dir: Path, *, fps: float): + from fastvideo.eval.io import build_eval_kwargs, glob_videos + from fastvideo.eval.types import EvalResult + + samples: list[dict] = [] + metas: list[dict] = [] + for row in dataset: + for vp in glob_videos(videos_dir, row): + samples.append(build_eval_kwargs(row, vp, fps=fps)) + metas.append({ + "prompt": row.get("prompt"), + "video": str(vp), + "dimensions": list(row.get("dimensions", [])), + }) + if not samples: + log.warning("[score] no videos matched in %s", videos_dir) + return EvalResult.from_raw({}, []) + + results = evaluator.evaluate(samples) + + by_metric: dict[str, list[float]] = {} + per_video: list[dict] = [] + for meta, res in zip(metas, results, strict=True): + scores: dict[str, float] = {} + for name, mr in res.items(): + if mr is not None and mr.score is not None: + scores[name] = float(mr.score) + by_metric.setdefault(name, []).append(scores[name]) + per_video.append({**meta, "scores": scores}) + log.info("[score] %s -> %s", meta["video"], + {k: round(v, 4) for k, v in scores.items()}) + log.info("[score] done. videos_scored=%d", len(per_video)) + return EvalResult.from_raw(by_metric, per_video) + + +def main() -> None: + args = parse_args() from fastvideo.eval.datasets import VBenchPromptDataset dims = ("all" if args.dimensions == "all" else [d.strip() for d in args.dimensions.split(",") if d.strip()]) dataset = VBenchPromptDataset(dimensions=dims) + log.info("dataset: %d prompts across %d dimensions", + len(dataset), len(dataset.dimensions)) if args.max_prompts: dataset._rows = dataset._rows[:args.max_prompts] + log.info("dataset: truncated to first %d prompts", len(dataset)) - if args.metrics.strip() == "all": - metrics: str | list[str] = "all" - elif "," in args.metrics: - metrics = [m.strip() for m in args.metrics.split(",") if m.strip()] - else: - metrics = args.metrics.strip() - - generator = None if not args.skip_generation: if not args.model: - ap.error("--model is required unless --skip-generation is set") + raise SystemExit("--model is required unless --skip-generation is set") from fastvideo.entrypoints.video_generator import VideoGenerator + log.info("[gen] loading %s with num_gpus=%d", + args.model, args.num_gpus) generator = VideoGenerator.from_pretrained(args.model, num_gpus=args.num_gpus) + gen_kwargs = {k: v for k, v in { + "height": args.height, "width": args.width, + "num_frames": args.num_frames, + "num_inference_steps": args.num_inference_steps, + "fps": args.fps, + }.items() if v is not None} + generate(dataset, generator, args.videos_dir, + base_seed=args.seed, + n_override=args.n_samples, + gen_kwargs=gen_kwargs) - gen_kwargs = {k: v for k, v in { - "height": args.height, "width": args.width, - "num_frames": args.num_frames, - "num_inference_steps": args.num_inference_steps, - "fps": args.fps, - }.items() if v is not None} - - runner = EvalRunner.from_dataset( - dataset, - videos_dir=args.videos_dir, - generator=generator, - metrics=metrics if not args.skip_scoring else [], - num_gpus=args.num_gpus, - fps=args.fps, - seed=args.seed, - n_samples=args.n_samples, - gen_kwargs=gen_kwargs, - ) - - if not args.skip_generation: - runner.generate() if not args.skip_scoring: - result = runner.score() + from fastvideo.eval import Evaluator + evaluator = Evaluator(metrics=parse_metrics(args.metrics), + num_gpus=args.num_gpus) + result = score(dataset, evaluator, args.videos_dir, fps=args.fps) result.print() if args.results_json: result.save(args.results_json)