shaoxiongduan
f6275f8005
Adds an in-process evaluation suite covering native (SSIM/PSNR/LPIPS/ optical_flow), audio, physics_iq, vbench (16 sub-metrics), and videoscore2. Public API: create_evaluator/evaluate, BaseMetric + @register, ensure_checkpoint, get_cache_dir. CLI: fastvideo eval list/run. VBench upstream pinned as a git submodule at fastvideo/third_party/eval/vbench (Vchitect/VBench@45e79ec); modern-dep compat is achieved via runtime shims in vbench/__init__.py rather than on-disk patches. CLIP/torch.hub caches are routed through ${FASTVIDEO_EVAL_CACHE}; HF cache stays at the system default. ensure_checkpoint delegates to fastvideo.utils.get_lock + huggingface_hub primitives. Evaluator supports release_cuda_memory(), unload(), reload() for training-time eval that frees GPU between calls. Verified parity against upstream vbench (5/8 metrics bit-exact, others within 1% drift driven by transformers/torch version skew) and against upstream VideoScore2 (regex anchored on the actual model's output format, soft-score formula matches upstream's argmax*max_prob/total). Includes docs/contributing/eval-metrics.md as the porting guide. Out of scope (deferred follow-ups): MIND, VBench-2.0, FVD as a registered metric, training-time EvalCallback.
46 lines
1.6 KiB
Python
46 lines
1.6 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/entrypoints/cli/main.py
|
|
from fastvideo.entrypoints.cli.cli_types import CLISubcommand
|
|
from fastvideo.entrypoints.cli.generate import cmd_init as generate_cmd_init
|
|
from fastvideo.utils import FlexibleArgumentParser
|
|
from fastvideo.entrypoints.cli.serve import cmd_init as serve_cmd_init
|
|
from fastvideo.entrypoints.cli.bench import cmd_init as bench_cmd_init
|
|
from fastvideo.entrypoints.cli.eval import cmd_init as eval_cmd_init
|
|
|
|
|
|
def cmd_init() -> list[CLISubcommand]:
|
|
"""Initialize all commands from separate modules"""
|
|
commands = []
|
|
commands.extend(generate_cmd_init())
|
|
commands.extend(serve_cmd_init())
|
|
commands.extend(bench_cmd_init())
|
|
commands.extend(eval_cmd_init())
|
|
return commands
|
|
|
|
|
|
def main() -> None:
|
|
parser = FlexibleArgumentParser(description="FastVideo CLI")
|
|
parser.add_argument('-v', '--version', action='version', version='0.1.0')
|
|
|
|
subparsers = parser.add_subparsers(required=False, dest="subparser")
|
|
|
|
cmds = {}
|
|
for cmd in cmd_init():
|
|
cmd.subparser_init(subparsers).set_defaults(dispatch_function=cmd.cmd)
|
|
cmds[cmd.name] = cmd
|
|
|
|
args, unknown = parser.parse_known_args()
|
|
if unknown and args.subparser not in {"generate", "serve"}:
|
|
parser.error(f"unrecognized arguments: {' '.join(unknown)}")
|
|
args._unknown = unknown
|
|
if args.subparser in cmds:
|
|
cmds[args.subparser].validate(args)
|
|
args.dispatch_function(args)
|
|
return
|
|
|
|
parser.print_help()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|