shaoxiongduan
d07a7691fe
Match FastVideo's existing depth: VideoGenerator is the top-level inference object with no Runner above it; loops live in scripts. Eval should mirror that — Evaluator is the top-level scoring object, EvalWorker × N is the layer below, and end-to-end pipelines (prompts → generate → score) are scripts, not classes. Removed: - fastvideo/eval/runner.py (EvalRunner + classmethod constructors). - EvalRunner export from fastvideo.eval. Added: - fastvideo/eval/io/paths.py with sanitize_prompt, default_filename, glob_videos, build_eval_kwargs as free functions. The reusable bits of the runner survive; the class wrapping them does not. Rewritten: - scripts/eval/run_vbench_e2e.py is now top-to-bottom procedural: parse_args → dataset → optional generate() loop → optional score() loop. Reads in one pass; no classmethod indirection to chase. Parity verified on fs-mbz-gpu-085 against runs/vbench_smoke/videos/ (2 videos × 3 metrics, dimensions=subject_consistency): - 1-GPU run (results_v4.json) is byte-identical to v3 baseline. - 2-GPU run (results_v4_2gpu.json) is byte-identical to v3 baseline (order-preserving fan-out via Evaluator.evaluate(samples=[...])).
65 lines
2.2 KiB
Python
65 lines
2.2 KiB
Python
"""Filesystem helpers shared by eval scripts.
|
|
|
|
Provides the prompt-sanitization, default filename convention, and
|
|
``(row, video_path) → eval-kwargs`` builder. Free functions, not a
|
|
class — :class:`fastvideo.eval.Evaluator` is the only stateful object
|
|
in the eval surface; loops live in user scripts.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
# Filesystem-unsafe characters mirrored from VideoGenerator's output-path
|
|
# sanitizer so on-disk filenames match what the generator writes.
|
|
_INVALID_CHARS = re.compile(r'[\\/:*?"<>|]')
|
|
|
|
|
|
def sanitize_prompt(prompt: str, max_len: int = 100) -> str:
|
|
"""Prompt → safe filename stem."""
|
|
s = _INVALID_CHARS.sub("", prompt[:max_len]).strip().strip(".")
|
|
return re.sub(r"\s+", " ", s) or "output"
|
|
|
|
|
|
def default_filename(row: dict, idx: int, ext: str = ".mp4") -> str:
|
|
"""``<sanitized-prompt>-<idx>.mp4`` — VBench-style."""
|
|
return f"{sanitize_prompt(row['prompt'])}-{idx}{ext}"
|
|
|
|
|
|
def glob_videos(videos_dir: Path, row: dict, ext: str = ".mp4") -> list[Path]:
|
|
"""Find every generated video for *row*, sorted by trailing ``-<idx>``."""
|
|
pattern = f"{sanitize_prompt(row['prompt'])}-*{ext}"
|
|
files = list(videos_dir.glob(pattern))
|
|
|
|
def _idx(p: Path) -> int:
|
|
try:
|
|
return int(p.stem.rsplit("-", 1)[1])
|
|
except (IndexError, ValueError):
|
|
return -1
|
|
|
|
return sorted(files, key=_idx)
|
|
|
|
|
|
def build_eval_kwargs(row: dict, video_path: Path, *,
|
|
fps: float = 24.0) -> dict[str, Any]:
|
|
"""Build evaluator kwargs from a sample row + a video on disk.
|
|
|
|
Loads the video as ``(T,C,H,W)`` and adds the leading batch dim.
|
|
Forwards ``prompt`` (as ``text_prompt=[prompt]``) and
|
|
``auxiliary_info`` (as ``[aux]``) when present on the row.
|
|
"""
|
|
from fastvideo.eval.io.video import load_video
|
|
|
|
video = load_video(str(video_path)) # (T, C, H, W) in [0, 1]
|
|
kwargs: dict[str, Any] = {
|
|
"video": video.unsqueeze(0), # (1, T, C, H, W)
|
|
"fps": fps,
|
|
}
|
|
if "prompt" in row:
|
|
kwargs["text_prompt"] = [row["prompt"]]
|
|
aux = row.get("auxiliary_info")
|
|
if aux:
|
|
kwargs["auxiliary_info"] = [aux]
|
|
return kwargs
|