From 7fa4fb50bbfcd8bd95ce7efd43b5198502f18fa0 Mon Sep 17 00:00:00 2001 From: shaoxiongduan Date: Thu, 7 May 2026 21:47:05 +0000 Subject: [PATCH] =?UTF-8?q?[cleanup]=20eval:=20PR-readiness=20audit=20?= =?UTF-8?q?=E2=80=94=20empty=20inits,=20fix=20stale=20docs,=20drop=20unuse?= =?UTF-8?q?d=20deps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three loose ends from earlier hygiene passes that the final PR review caught: * Empty out 27 sub-metric ``__init__.py`` files that still re-exported the metric class. The earlier ``54f26bea`` commit only cleaned 2 of them; the rest still carried ``from .metric import FooMetric # noqa: F401`` lines copied from the original wm-eval port. Auto-discovery doesn't need them, and ``get_metric("group.name")`` is the canonical access pattern. Now uniformly empty across all sub-metric inits. * Fix stale layout references in ``fastvideo/eval/README.md`` and ``.agents/workflows/evaluation-development.md`` that still described the abandoned ``/external/upstream/`` per-metric layout. The actual layout is the flat ``fastvideo/third_party/eval//`` we switched to during the port. Also drops the stale ``_third_party`` example from ``fastvideo/eval/metrics/__init__.py``'s docstring (the underscore-skip rule is unchanged; the example was just wrong). * Drop ``open_clip_torch`` and ``torchmetrics`` from the ``[eval]`` extra. Neither is imported anywhere under ``fastvideo/eval/`` (or in the vbench submodule we vendor); they were carried over from the wm-eval port's heavier dep graph. Verified: 32 metrics still register, 33/4 tests pass/skip on 1 GPU. --- .agents/workflows/evaluation-development.md | 5 +++-- fastvideo/eval/README.md | 13 ++++++++----- fastvideo/eval/metrics/__init__.py | 8 ++++---- .../metrics/audio/audiobox_aesthetics/__init__.py | 1 - .../eval/metrics/audio/frechet_distance/__init__.py | 1 - .../eval/metrics/audio/kl_divergence/__init__.py | 1 - fastvideo/eval/metrics/audio/wer/__init__.py | 1 - fastvideo/eval/metrics/common/lpips/__init__.py | 1 - fastvideo/eval/metrics/common/psnr/__init__.py | 1 - fastvideo/eval/metrics/common/ssim/__init__.py | 1 - fastvideo/eval/metrics/physics_iq/mse/__init__.py | 1 - .../eval/metrics/physics_iq/spatial_iou/__init__.py | 1 - .../physics_iq/spatiotemporal_iou/__init__.py | 1 - .../physics_iq/weighted_spatial_iou/__init__.py | 1 - .../metrics/vbench/appearance_style/__init__.py | 1 - .../vbench/background_consistency/__init__.py | 1 - fastvideo/eval/metrics/vbench/color/__init__.py | 1 - .../eval/metrics/vbench/dynamic_degree/__init__.py | 1 - .../eval/metrics/vbench/human_action/__init__.py | 1 - .../eval/metrics/vbench/imaging_quality/__init__.py | 1 - .../metrics/vbench/motion_smoothness/__init__.py | 1 - .../metrics/vbench/multiple_objects/__init__.py | 1 - .../eval/metrics/vbench/object_class/__init__.py | 1 - .../metrics/vbench/overall_consistency/__init__.py | 1 - fastvideo/eval/metrics/vbench/scene/__init__.py | 1 - .../metrics/vbench/spatial_relationship/__init__.py | 1 - .../metrics/vbench/subject_consistency/__init__.py | 1 - .../metrics/vbench/temporal_flickering/__init__.py | 1 - .../eval/metrics/vbench/temporal_style/__init__.py | 1 - fastvideo/eval/metrics/videoscore2/__init__.py | 1 - pyproject.toml | 2 -- 31 files changed, 15 insertions(+), 40 deletions(-) diff --git a/.agents/workflows/evaluation-development.md b/.agents/workflows/evaluation-development.md index 8339e7078..4b04fe1ff 100644 --- a/.agents/workflows/evaluation-development.md +++ b/.agents/workflows/evaluation-development.md @@ -64,8 +64,9 @@ via `@register(".")` and are auto-discovered when (`fastvideo/eval/metrics/common/`, `audio/`, `vlm/`). - **Metrics that wrap upstream research code**: follow the vbench pattern in `fastvideo/eval/metrics/vbench/`: - - upstream is a git submodule under `/external/upstream/`, - pinned to a SHA in repo-root `.gitmodules`; + - upstream is a git submodule under + `fastvideo/third_party/eval//`, pinned to a SHA in repo-root + `.gitmodules`; - the metric package's `__init__.py` does the `sys.path` insert AND installs runtime compat shims (attribute-level monkey-patches) for any modern-dep drift. **Do not** modify upstream files on disk and diff --git a/fastvideo/eval/README.md b/fastvideo/eval/README.md index 2ed9cbddd..53eb9f976 100644 --- a/fastvideo/eval/README.md +++ b/fastvideo/eval/README.md @@ -133,11 +133,14 @@ module). Pattern: see `fastvideo/eval/metrics/vbench/`. The contract is: -1. `/external/upstream/` is a git submodule pinned to a SHA - (registered in repo-root `.gitmodules`). -2. `/__init__.py` inserts that path on `sys.path` and installs - any compat shims (attribute-level monkey-patches) needed for modern - torch/transformers/numpy. **Do not modify upstream files on disk.** +1. The upstream code lives as a git submodule under + `fastvideo/third_party/eval//`, pinned to a SHA (registered + in repo-root `.gitmodules`). +2. The metric package's `__init__.py` + (`fastvideo/eval/metrics//__init__.py`) inserts that submodule + path on `sys.path` and installs any compat shims (attribute-level + monkey-patches) needed for modern torch/transformers/numpy. + **Do not modify upstream files on disk.** 3. Per-sub-metric `metric.py` files use `@register(".")`. There is no per-benchmark setup script. Patches live as Python in diff --git a/fastvideo/eval/metrics/__init__.py b/fastvideo/eval/metrics/__init__.py index 04cc64823..b7c8d0b7f 100644 --- a/fastvideo/eval/metrics/__init__.py +++ b/fastvideo/eval/metrics/__init__.py @@ -1,9 +1,9 @@ """Auto-discover and register all built-in metrics (recursive). -Manually walks the metrics package tree and imports any leaf-package's -`metric` module so @register decorators fire. Skips any path component -starting with `_` (e.g. _vendor_*, _third_party) to avoid pulling in -vendored model code that may have side-effecting imports. +Walks the metrics package tree and imports any leaf-package's +``metric`` module so ``@register`` decorators fire. Path components +starting with ``_`` are skipped (used for shared helpers like +``optical_flow/_shared.py`` and `vbench/_grit_helper.py`). """ import importlib diff --git a/fastvideo/eval/metrics/audio/audiobox_aesthetics/__init__.py b/fastvideo/eval/metrics/audio/audiobox_aesthetics/__init__.py index ec0d38874..e69de29bb 100644 --- a/fastvideo/eval/metrics/audio/audiobox_aesthetics/__init__.py +++ b/fastvideo/eval/metrics/audio/audiobox_aesthetics/__init__.py @@ -1 +0,0 @@ -from .metric import VBAudioBoxAestheticsMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/audio/frechet_distance/__init__.py b/fastvideo/eval/metrics/audio/frechet_distance/__init__.py index 4e4e469e3..e69de29bb 100644 --- a/fastvideo/eval/metrics/audio/frechet_distance/__init__.py +++ b/fastvideo/eval/metrics/audio/frechet_distance/__init__.py @@ -1 +0,0 @@ -from .metric import VBFrechetDistanceAudioMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/audio/kl_divergence/__init__.py b/fastvideo/eval/metrics/audio/kl_divergence/__init__.py index 9452911b9..e69de29bb 100644 --- a/fastvideo/eval/metrics/audio/kl_divergence/__init__.py +++ b/fastvideo/eval/metrics/audio/kl_divergence/__init__.py @@ -1 +0,0 @@ -from .metric import VBKLDivergenceAudioMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/audio/wer/__init__.py b/fastvideo/eval/metrics/audio/wer/__init__.py index 2fe6fbfe9..e69de29bb 100644 --- a/fastvideo/eval/metrics/audio/wer/__init__.py +++ b/fastvideo/eval/metrics/audio/wer/__init__.py @@ -1 +0,0 @@ -from .metric import VBWERMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/common/lpips/__init__.py b/fastvideo/eval/metrics/common/lpips/__init__.py index da4b9afe2..e69de29bb 100644 --- a/fastvideo/eval/metrics/common/lpips/__init__.py +++ b/fastvideo/eval/metrics/common/lpips/__init__.py @@ -1 +0,0 @@ -from .metric import LPIPSMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/common/psnr/__init__.py b/fastvideo/eval/metrics/common/psnr/__init__.py index 4dbf1dd88..e69de29bb 100644 --- a/fastvideo/eval/metrics/common/psnr/__init__.py +++ b/fastvideo/eval/metrics/common/psnr/__init__.py @@ -1 +0,0 @@ -from .metric import PSNRMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/common/ssim/__init__.py b/fastvideo/eval/metrics/common/ssim/__init__.py index 4bd26ec80..e69de29bb 100644 --- a/fastvideo/eval/metrics/common/ssim/__init__.py +++ b/fastvideo/eval/metrics/common/ssim/__init__.py @@ -1 +0,0 @@ -from .metric import SSIMMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/physics_iq/mse/__init__.py b/fastvideo/eval/metrics/physics_iq/mse/__init__.py index b3a5ac6d9..e69de29bb 100644 --- a/fastvideo/eval/metrics/physics_iq/mse/__init__.py +++ b/fastvideo/eval/metrics/physics_iq/mse/__init__.py @@ -1 +0,0 @@ -from fastvideo.eval.metrics.physics_iq.mse.metric import PhysicsIQMSEMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/physics_iq/spatial_iou/__init__.py b/fastvideo/eval/metrics/physics_iq/spatial_iou/__init__.py index cc9200247..e69de29bb 100644 --- a/fastvideo/eval/metrics/physics_iq/spatial_iou/__init__.py +++ b/fastvideo/eval/metrics/physics_iq/spatial_iou/__init__.py @@ -1 +0,0 @@ -from fastvideo.eval.metrics.physics_iq.spatial_iou.metric import SpatialIoUMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/physics_iq/spatiotemporal_iou/__init__.py b/fastvideo/eval/metrics/physics_iq/spatiotemporal_iou/__init__.py index 49e8992db..e69de29bb 100644 --- a/fastvideo/eval/metrics/physics_iq/spatiotemporal_iou/__init__.py +++ b/fastvideo/eval/metrics/physics_iq/spatiotemporal_iou/__init__.py @@ -1 +0,0 @@ -from fastvideo.eval.metrics.physics_iq.spatiotemporal_iou.metric import SpatiotemporalIoUMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/physics_iq/weighted_spatial_iou/__init__.py b/fastvideo/eval/metrics/physics_iq/weighted_spatial_iou/__init__.py index 41d7abd81..e69de29bb 100644 --- a/fastvideo/eval/metrics/physics_iq/weighted_spatial_iou/__init__.py +++ b/fastvideo/eval/metrics/physics_iq/weighted_spatial_iou/__init__.py @@ -1 +0,0 @@ -from fastvideo.eval.metrics.physics_iq.weighted_spatial_iou.metric import WeightedSpatialIoUMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/vbench/appearance_style/__init__.py b/fastvideo/eval/metrics/vbench/appearance_style/__init__.py index 847f3aac9..e69de29bb 100644 --- a/fastvideo/eval/metrics/vbench/appearance_style/__init__.py +++ b/fastvideo/eval/metrics/vbench/appearance_style/__init__.py @@ -1 +0,0 @@ -from .metric import AppearanceStyleMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/vbench/background_consistency/__init__.py b/fastvideo/eval/metrics/vbench/background_consistency/__init__.py index bff4aaf7d..e69de29bb 100644 --- a/fastvideo/eval/metrics/vbench/background_consistency/__init__.py +++ b/fastvideo/eval/metrics/vbench/background_consistency/__init__.py @@ -1 +0,0 @@ -from .metric import BackgroundConsistencyMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/vbench/color/__init__.py b/fastvideo/eval/metrics/vbench/color/__init__.py index b1d1688bf..e69de29bb 100644 --- a/fastvideo/eval/metrics/vbench/color/__init__.py +++ b/fastvideo/eval/metrics/vbench/color/__init__.py @@ -1 +0,0 @@ -from .metric import ColorMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/vbench/dynamic_degree/__init__.py b/fastvideo/eval/metrics/vbench/dynamic_degree/__init__.py index 4a64820ff..e69de29bb 100644 --- a/fastvideo/eval/metrics/vbench/dynamic_degree/__init__.py +++ b/fastvideo/eval/metrics/vbench/dynamic_degree/__init__.py @@ -1 +0,0 @@ -from .metric import DynamicDegreeMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/vbench/human_action/__init__.py b/fastvideo/eval/metrics/vbench/human_action/__init__.py index e292f250e..e69de29bb 100644 --- a/fastvideo/eval/metrics/vbench/human_action/__init__.py +++ b/fastvideo/eval/metrics/vbench/human_action/__init__.py @@ -1 +0,0 @@ -from .metric import HumanActionMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/vbench/imaging_quality/__init__.py b/fastvideo/eval/metrics/vbench/imaging_quality/__init__.py index 73295fd42..e69de29bb 100644 --- a/fastvideo/eval/metrics/vbench/imaging_quality/__init__.py +++ b/fastvideo/eval/metrics/vbench/imaging_quality/__init__.py @@ -1 +0,0 @@ -from .metric import ImagingQualityMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/vbench/motion_smoothness/__init__.py b/fastvideo/eval/metrics/vbench/motion_smoothness/__init__.py index 6b77b99b8..e69de29bb 100644 --- a/fastvideo/eval/metrics/vbench/motion_smoothness/__init__.py +++ b/fastvideo/eval/metrics/vbench/motion_smoothness/__init__.py @@ -1 +0,0 @@ -from .metric import MotionSmoothnessMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/vbench/multiple_objects/__init__.py b/fastvideo/eval/metrics/vbench/multiple_objects/__init__.py index 7c5ae8a25..e69de29bb 100644 --- a/fastvideo/eval/metrics/vbench/multiple_objects/__init__.py +++ b/fastvideo/eval/metrics/vbench/multiple_objects/__init__.py @@ -1 +0,0 @@ -from .metric import MultipleObjectsMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/vbench/object_class/__init__.py b/fastvideo/eval/metrics/vbench/object_class/__init__.py index f6e5859f8..e69de29bb 100644 --- a/fastvideo/eval/metrics/vbench/object_class/__init__.py +++ b/fastvideo/eval/metrics/vbench/object_class/__init__.py @@ -1 +0,0 @@ -from .metric import ObjectClassMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/vbench/overall_consistency/__init__.py b/fastvideo/eval/metrics/vbench/overall_consistency/__init__.py index 8cc217229..e69de29bb 100644 --- a/fastvideo/eval/metrics/vbench/overall_consistency/__init__.py +++ b/fastvideo/eval/metrics/vbench/overall_consistency/__init__.py @@ -1 +0,0 @@ -from .metric import OverallConsistencyMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/vbench/scene/__init__.py b/fastvideo/eval/metrics/vbench/scene/__init__.py index e90a95377..e69de29bb 100644 --- a/fastvideo/eval/metrics/vbench/scene/__init__.py +++ b/fastvideo/eval/metrics/vbench/scene/__init__.py @@ -1 +0,0 @@ -from .metric import SceneMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/vbench/spatial_relationship/__init__.py b/fastvideo/eval/metrics/vbench/spatial_relationship/__init__.py index 284a6a7de..e69de29bb 100644 --- a/fastvideo/eval/metrics/vbench/spatial_relationship/__init__.py +++ b/fastvideo/eval/metrics/vbench/spatial_relationship/__init__.py @@ -1 +0,0 @@ -from .metric import SpatialRelationshipMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/vbench/subject_consistency/__init__.py b/fastvideo/eval/metrics/vbench/subject_consistency/__init__.py index 2379ad5c0..e69de29bb 100644 --- a/fastvideo/eval/metrics/vbench/subject_consistency/__init__.py +++ b/fastvideo/eval/metrics/vbench/subject_consistency/__init__.py @@ -1 +0,0 @@ -from .metric import SubjectConsistencyMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/vbench/temporal_flickering/__init__.py b/fastvideo/eval/metrics/vbench/temporal_flickering/__init__.py index 12d6ca34e..e69de29bb 100644 --- a/fastvideo/eval/metrics/vbench/temporal_flickering/__init__.py +++ b/fastvideo/eval/metrics/vbench/temporal_flickering/__init__.py @@ -1 +0,0 @@ -from .metric import TemporalFlickeringMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/vbench/temporal_style/__init__.py b/fastvideo/eval/metrics/vbench/temporal_style/__init__.py index 321a180d0..e69de29bb 100644 --- a/fastvideo/eval/metrics/vbench/temporal_style/__init__.py +++ b/fastvideo/eval/metrics/vbench/temporal_style/__init__.py @@ -1 +0,0 @@ -from .metric import TemporalStyleMetric # noqa: F401 diff --git a/fastvideo/eval/metrics/videoscore2/__init__.py b/fastvideo/eval/metrics/videoscore2/__init__.py index b15c68a25..e69de29bb 100644 --- a/fastvideo/eval/metrics/videoscore2/__init__.py +++ b/fastvideo/eval/metrics/videoscore2/__init__.py @@ -1 +0,0 @@ -from .metric import VideoScore2Metric # noqa: F401 diff --git a/pyproject.toml b/pyproject.toml index 175881728..0a40befe2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -134,11 +134,9 @@ test = [ eval = [ "decord", "openai-clip", - "open_clip_torch", "pyiqa", "ptlflow", "lpips", - "torchmetrics", "easydict", "qwen-vl-utils", ]