shaoxiongduan
de3cd6aab2
``Evaluator.evaluate`` and the one-shot ``evaluate()`` helper now accept ``video`` / ``reference`` as either a pre-loaded ``(T, C, H, W)`` tensor or a path-like (``str`` / ``Path``). Paths are decoded inside the worker thread that picks the sample up — see ``EvalWorker._resolve_video_input``. Why this matters for batch eval: the dispatcher in ``Evaluator.evaluate(samples=[...])`` submits every sample to the ``ThreadPoolExecutor`` upfront. With pre-loaded tensors that meant every video in the batch was resident in CPU RAM at once (≈ 3 GB per video at 1088×1920×121); a full benchmark of hundreds of clips would OOM before scoring started. With paths, the queued futures hold cheap strings; only ``num_gpus`` videos are decoded concurrently. Peak resident memory becomes ``O(num_gpus)`` instead of ``O(len(samples))``. No dispatcher rewrite needed — the change is ~10 lines at the worker boundary. * ``EvalWorker.evaluate`` now normalizes ``sample["video"]`` and ``sample["reference"]`` through ``_resolve_video_input`` (path → ``load_video`` decode; ``(1, T, C, H, W)`` → squeeze; tensor → passthrough). Metrics keep the existing contract: by the time ``compute()`` runs, ``sample["video"]`` is always a 4-D tensor. * ``score_folder.py`` and ``bench_vbench.py`` simplified to pass paths directly. ``bench_physics_iq.py`` already did. * ``fastvideo.eval.api.evaluate`` signature widened to ``Tensor | str | Path``; the worker handles the decode either way. Tests: new ``test_evaluator_paths.py`` covers the kwargs form, the samples-list form, mixing paths and tensors in the same batch, parity between path-form and tensor-form scores, and the missing-path exception path. End-to-end ``score_folder.py`` smoke run confirmed.
96 lines
3.5 KiB
Python
96 lines
3.5 KiB
Python
"""Score a folder of videos in parallel across multiple GPUs.
|
|
|
|
Uses :meth:`Evaluator.evaluate(samples=[...])`, which round-robins each
|
|
sample dict across the GPU replicas the evaluator was built with.
|
|
|
|
Example::
|
|
|
|
python examples/inference/eval/score_folder.py \\
|
|
--videos generated/ \\
|
|
--metrics vbench.aesthetic_quality,vbench.subject_consistency \\
|
|
--num-gpus 4 \\
|
|
--output scores.json
|
|
|
|
Pair each generated video with a same-name reference video (e.g.
|
|
``ref/<stem>.mp4``) by passing ``--reference-dir``::
|
|
|
|
python examples/inference/eval/score_folder.py \\
|
|
--videos generated/ --reference-dir ref/ \\
|
|
--metrics common.psnr,common.ssim,common.lpips \\
|
|
--num-gpus 4
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from fastvideo.eval import create_evaluator
|
|
|
|
|
|
def _list_videos(directory: Path) -> list[Path]:
|
|
exts = {".mp4", ".avi", ".mov", ".mkv", ".gif"}
|
|
return sorted(p for p in directory.iterdir() if p.suffix.lower() in exts)
|
|
|
|
|
|
def main() -> None:
|
|
p = argparse.ArgumentParser(description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
p.add_argument("--videos", type=Path, required=True,
|
|
help="Directory of generated videos.")
|
|
p.add_argument("--reference-dir", type=Path, default=None,
|
|
help="Directory of reference videos with matching stems.")
|
|
p.add_argument("--metrics", default="vbench.aesthetic_quality")
|
|
p.add_argument("--num-gpus", type=int, default=1)
|
|
p.add_argument("--fps", type=float, default=None,
|
|
help="Frame-rate annotation for fps-aware metrics.")
|
|
p.add_argument("--output", type=Path, default=Path("scores.json"))
|
|
args = p.parse_args()
|
|
|
|
video_paths = _list_videos(args.videos)
|
|
if not video_paths:
|
|
raise SystemExit(f"No videos under {args.videos}")
|
|
print(f"Found {len(video_paths)} videos in {args.videos}")
|
|
|
|
metrics: list[str] | str = (
|
|
args.metrics if args.metrics == "all"
|
|
else [m.strip() for m in args.metrics.split(",") if m.strip()]
|
|
)
|
|
evaluator = create_evaluator(metrics=metrics, num_gpus=args.num_gpus)
|
|
|
|
# Build per-video sample dicts holding *paths*, not pre-loaded
|
|
# tensors. Each path is decoded inside the worker thread that picks
|
|
# up its sample, so peak resident memory is bounded by num_gpus
|
|
# rather than scaling with the size of the folder.
|
|
samples: list[dict] = []
|
|
for vp in video_paths:
|
|
sample: dict = {"video": str(vp)}
|
|
if args.reference_dir is not None:
|
|
ref_path = args.reference_dir / vp.name
|
|
if not ref_path.is_file():
|
|
raise FileNotFoundError(f"Missing reference for {vp.name} at {ref_path}")
|
|
sample["reference"] = str(ref_path)
|
|
if args.fps is not None:
|
|
sample["fps"] = args.fps
|
|
samples.append(sample)
|
|
|
|
print(f"Scoring with {len(evaluator.metric_names)} metric(s) "
|
|
f"on {evaluator.num_gpus} GPU(s)...")
|
|
all_results = evaluator.evaluate(samples=samples)
|
|
evaluator.shutdown()
|
|
|
|
payload = [
|
|
{
|
|
"video": str(vp),
|
|
"scores": {name: r.score for name, r in results.items()},
|
|
}
|
|
for vp, results in zip(video_paths, all_results)
|
|
]
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(payload, indent=2))
|
|
print(f"Wrote {args.output}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|