shaoxiongduan
1dcdacc35a
Add four small example scripts under ``examples/inference/eval/``,
each driving the public eval API in a different real-world shape:
* ``score_video.py`` — score one mp4 on one GPU. Smallest possible
use of ``create_evaluator`` + ``Evaluator.evaluate``. Optional
``--reference``, ``--text-prompt``, ``--fps`` for paired / prompt-
aware / fps-aware metrics.
* ``score_folder.py`` — score every mp4 in a directory across ``--num-gpus``
replicas via ``Evaluator.evaluate(samples=[...])``. Pair each
generated video with a same-name reference under ``--reference-dir``
if you want paired metrics.
* ``bench_vbench.py`` — full VBench end-to-end: ``get_dataset("vbench",
dimensions=...)`` → ``VideoGenerator`` (LTX2 by default) → score
with the matching ``vbench.*`` sub-metrics → print per-metric
averages. ``--skip-generation`` re-uses existing mp4s under
``--videos-dir`` so you can iterate on metric selection without
re-paying generation cost.
* ``bench_physics_iq.py`` — full Physics-IQ end-to-end:
``get_dataset("physics_iq", dataset_root=...)`` → ``VideoGenerator``
→ score with the composite ``physics_iq`` metric → aggregate via the
upstream's ``aggregate_components`` recipe.
The pre-existing ``basic_ltx2_eval.py`` and ``eval_ltx2_vbench.py`` are
left in place — they target the single-prompt generate-and-score case
and complement (rather than overlap with) the new full-dataset runners.
Verified end-to-end on a borrowed GPU node:
* ``score_video.py``: PSNR=100, SSIM=1.0 on a self-paired mp4 (sanity).
* ``score_folder.py``: produces well-formed scores.json.
69 lines
2.4 KiB
Python
69 lines
2.4 KiB
Python
"""Score one video on one GPU.
|
|
|
|
Smallest possible use of ``fastvideo.eval``: load an mp4, build an
|
|
:class:`Evaluator` for the requested metric set, run it.
|
|
|
|
Examples::
|
|
|
|
# Reference-free (just the generated video):
|
|
python examples/inference/eval/score_video.py \\
|
|
--video clip.mp4 \\
|
|
--metrics vbench.aesthetic_quality,vbench.imaging_quality
|
|
|
|
# Reference-paired (compare against ground truth):
|
|
python examples/inference/eval/score_video.py \\
|
|
--video gen.mp4 --reference ref.mp4 \\
|
|
--metrics common.psnr,common.ssim,common.lpips
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
|
|
from fastvideo.eval import create_evaluator
|
|
from fastvideo.eval.io import load_video
|
|
|
|
|
|
def main() -> None:
|
|
p = argparse.ArgumentParser(description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
p.add_argument("--video", required=True, help="Path to the generated mp4.")
|
|
p.add_argument("--reference", default=None,
|
|
help="Optional path to a reference mp4 (for paired metrics).")
|
|
p.add_argument("--metrics", default="common.psnr,common.ssim",
|
|
help="Comma-separated metric names, or a group name like 'vbench'.")
|
|
p.add_argument("--device", default="cuda:0")
|
|
p.add_argument("--text-prompt", default=None,
|
|
help="Text prompt for prompt-aware metrics "
|
|
"(vbench.overall_consistency, etc.).")
|
|
p.add_argument("--fps", type=float, default=None,
|
|
help="Frame-rate annotation for fps-aware metrics "
|
|
"(vbench.dynamic_degree, etc.).")
|
|
args = p.parse_args()
|
|
|
|
metrics: list[str] | str = (
|
|
args.metrics if args.metrics in ("all",)
|
|
else [m.strip() for m in args.metrics.split(",") if m.strip()]
|
|
)
|
|
evaluator = create_evaluator(metrics=metrics, device=args.device)
|
|
|
|
sample: dict = {"video": load_video(args.video)}
|
|
if args.reference is not None:
|
|
sample["reference"] = load_video(args.reference)
|
|
if args.text_prompt is not None:
|
|
sample["text_prompt"] = args.text_prompt
|
|
if args.fps is not None:
|
|
sample["fps"] = args.fps
|
|
|
|
results = evaluator.evaluate(**sample)
|
|
evaluator.shutdown()
|
|
|
|
print(json.dumps(
|
|
{name: r.score for name, r in results.items()},
|
|
indent=2,
|
|
))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|