shaoxiongduan
1dcdacc35a
Add four small example scripts under ``examples/inference/eval/``,
each driving the public eval API in a different real-world shape:
* ``score_video.py`` — score one mp4 on one GPU. Smallest possible
use of ``create_evaluator`` + ``Evaluator.evaluate``. Optional
``--reference``, ``--text-prompt``, ``--fps`` for paired / prompt-
aware / fps-aware metrics.
* ``score_folder.py`` — score every mp4 in a directory across ``--num-gpus``
replicas via ``Evaluator.evaluate(samples=[...])``. Pair each
generated video with a same-name reference under ``--reference-dir``
if you want paired metrics.
* ``bench_vbench.py`` — full VBench end-to-end: ``get_dataset("vbench",
dimensions=...)`` → ``VideoGenerator`` (LTX2 by default) → score
with the matching ``vbench.*`` sub-metrics → print per-metric
averages. ``--skip-generation`` re-uses existing mp4s under
``--videos-dir`` so you can iterate on metric selection without
re-paying generation cost.
* ``bench_physics_iq.py`` — full Physics-IQ end-to-end:
``get_dataset("physics_iq", dataset_root=...)`` → ``VideoGenerator``
→ score with the composite ``physics_iq`` metric → aggregate via the
upstream's ``aggregate_components`` recipe.
The pre-existing ``basic_ltx2_eval.py`` and ``eval_ltx2_vbench.py`` are
left in place — they target the single-prompt generate-and-score case
and complement (rather than overlap with) the new full-dataset runners.
Verified end-to-end on a borrowed GPU node:
* ``score_video.py``: PSNR=100, SSIM=1.0 on a self-paired mp4 (sanity).
* ``score_folder.py``: produces well-formed scores.json.
97 lines
3.5 KiB
Python
97 lines
3.5 KiB
Python
"""Score a folder of videos in parallel across multiple GPUs.
|
|
|
|
Uses :meth:`Evaluator.evaluate(samples=[...])`, which round-robins each
|
|
sample dict across the GPU replicas the evaluator was built with.
|
|
|
|
Example::
|
|
|
|
python examples/inference/eval/score_folder.py \\
|
|
--videos generated/ \\
|
|
--metrics vbench.aesthetic_quality,vbench.subject_consistency \\
|
|
--num-gpus 4 \\
|
|
--output scores.json
|
|
|
|
Pair each generated video with a same-name reference video (e.g.
|
|
``ref/<stem>.mp4``) by passing ``--reference-dir``::
|
|
|
|
python examples/inference/eval/score_folder.py \\
|
|
--videos generated/ --reference-dir ref/ \\
|
|
--metrics common.psnr,common.ssim,common.lpips \\
|
|
--num-gpus 4
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from fastvideo.eval import create_evaluator
|
|
from fastvideo.eval.io import load_video
|
|
|
|
|
|
def _list_videos(directory: Path) -> list[Path]:
|
|
exts = {".mp4", ".avi", ".mov", ".mkv", ".gif"}
|
|
return sorted(p for p in directory.iterdir() if p.suffix.lower() in exts)
|
|
|
|
|
|
def main() -> None:
|
|
p = argparse.ArgumentParser(description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
p.add_argument("--videos", type=Path, required=True,
|
|
help="Directory of generated videos.")
|
|
p.add_argument("--reference-dir", type=Path, default=None,
|
|
help="Directory of reference videos with matching stems.")
|
|
p.add_argument("--metrics", default="vbench.aesthetic_quality")
|
|
p.add_argument("--num-gpus", type=int, default=1)
|
|
p.add_argument("--fps", type=float, default=None,
|
|
help="Frame-rate annotation for fps-aware metrics.")
|
|
p.add_argument("--output", type=Path, default=Path("scores.json"))
|
|
args = p.parse_args()
|
|
|
|
video_paths = _list_videos(args.videos)
|
|
if not video_paths:
|
|
raise SystemExit(f"No videos under {args.videos}")
|
|
print(f"Found {len(video_paths)} videos in {args.videos}")
|
|
|
|
metrics: list[str] | str = (
|
|
args.metrics if args.metrics == "all"
|
|
else [m.strip() for m in args.metrics.split(",") if m.strip()]
|
|
)
|
|
evaluator = create_evaluator(metrics=metrics, num_gpus=args.num_gpus)
|
|
|
|
# Build the list of per-video sample dicts. Loading happens upfront
|
|
# so the evaluator can fan all samples out across GPU replicas at
|
|
# once, rather than serializing on a per-call load.
|
|
samples: list[dict] = []
|
|
for vp in video_paths:
|
|
sample: dict = {"video": load_video(str(vp))}
|
|
if args.reference_dir is not None:
|
|
ref_path = args.reference_dir / vp.name
|
|
if not ref_path.is_file():
|
|
raise FileNotFoundError(
|
|
f"Missing reference for {vp.name} at {ref_path}")
|
|
sample["reference"] = load_video(str(ref_path))
|
|
if args.fps is not None:
|
|
sample["fps"] = args.fps
|
|
samples.append(sample)
|
|
|
|
print(f"Scoring with {len(evaluator.metric_names)} metric(s) "
|
|
f"on {evaluator.num_gpus} GPU(s)...")
|
|
all_results = evaluator.evaluate(samples=samples)
|
|
evaluator.shutdown()
|
|
|
|
payload = [
|
|
{
|
|
"video": str(vp),
|
|
"scores": {name: r.score for name, r in results.items()},
|
|
}
|
|
for vp, results in zip(video_paths, all_results)
|
|
]
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(payload, indent=2))
|
|
print(f"Wrote {args.output}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|