diff --git a/fastvideo/eval/metrics/common/lpips/metric.py b/fastvideo/eval/metrics/common/lpips/metric.py index afea7650e..d0e53808e 100644 --- a/fastvideo/eval/metrics/common/lpips/metric.py +++ b/fastvideo/eval/metrics/common/lpips/metric.py @@ -14,7 +14,6 @@ class LPIPSMetric(BaseMetric): higher_is_better = False needs_gpu = True dependencies = ["lpips"] - batch_unit = "frame" def __init__(self, net: str = "alex") -> None: super().__init__() @@ -34,11 +33,6 @@ class LPIPSMetric(BaseMetric): self._model = lpips_lib.LPIPS(net=self.net).to(self.device) self._model.eval() - def trial_forward(self, batch_size, *, height, width, num_frames): - dummy = torch.randn(batch_size, 3, height, width, device=self.device) * 2 - 1 - with torch.no_grad(): - self._model(dummy, dummy) - def compute(self, sample: dict) -> list[MetricResult]: if self._model is None: self.setup() diff --git a/fastvideo/eval/metrics/common/ssim/metric.py b/fastvideo/eval/metrics/common/ssim/metric.py index fb881722d..529e6eb88 100644 --- a/fastvideo/eval/metrics/common/ssim/metric.py +++ b/fastvideo/eval/metrics/common/ssim/metric.py @@ -52,16 +52,11 @@ class SSIMMetric(BaseMetric): requires_reference = True higher_is_better = True needs_gpu = False - batch_unit = "frame" def __init__(self, window_size: int = 11) -> None: super().__init__() self.window_size = window_size - def trial_forward(self, batch_size, *, height, width, num_frames): - dummy = torch.randn(batch_size, 3, height, width, device=self.device) - _ssim_per_frame(dummy, dummy, self.window_size) - def compute(self, sample: dict) -> list[MetricResult]: gen = sample["video"].float() # (B, T, C, H, W) ref = sample["reference"].float() # (B, T, C, H, W) diff --git a/fastvideo/eval/metrics/physics_iq/metric.py b/fastvideo/eval/metrics/physics_iq/metric.py index 6021ed5e0..2db8e33ce 100644 --- a/fastvideo/eval/metrics/physics_iq/metric.py +++ b/fastvideo/eval/metrics/physics_iq/metric.py @@ -22,7 +22,6 @@ class PhysicsIQMetric(BaseMetric): name = "physics_iq" requires_reference = True higher_is_better = True - batch_unit = "video" def __init__( self, @@ -48,15 +47,6 @@ class PhysicsIQMetric(BaseMetric): self._spatial_iou = SpatialIoUMetric(**self._prep_kwargs) self._weighted_spatial_iou = WeightedSpatialIoUMetric(**self._prep_kwargs) - def trial_forward(self, batch_size: int, *, height: int, width: int, num_frames: int) -> None: - sample = { - "video": torch.rand(batch_size, num_frames, 3, height, width, device=self.device), - "reference": torch.rand(batch_size, num_frames, 3, height, width, device=self.device), - "reference_take2": torch.rand(batch_size, num_frames, 3, height, width, device=self.device), - } - with torch.no_grad(): - self.compute(sample) - @staticmethod def _extract_payload(result: MetricResult | Mapping[str, Any]) -> Mapping[str, Any]: if isinstance(result, MetricResult): diff --git a/fastvideo/eval/metrics/vbench/aesthetic_quality/metric.py b/fastvideo/eval/metrics/vbench/aesthetic_quality/metric.py index bdf99fc95..20d39d00a 100644 --- a/fastvideo/eval/metrics/vbench/aesthetic_quality/metric.py +++ b/fastvideo/eval/metrics/vbench/aesthetic_quality/metric.py @@ -38,7 +38,6 @@ class AestheticQualityMetric(BaseMetric): requires_reference = False higher_is_better = True needs_gpu = True - batch_unit = "frame" dependencies = ["clip"] backbone = "clip_vit_l14" @@ -80,13 +79,6 @@ class AestheticQualityMetric(BaseMetric): self._aesthetic_head.to(self.device) self._aesthetic_head.eval() - def trial_forward(self, batch_size, *, height, width, num_frames): - dummy = torch.randn(batch_size, 3, 224, 224, device=self.device) - with torch.no_grad(): - feats = self._clip_model.encode_image(dummy).float() - feats = F.normalize(feats, dim=-1, p=2) - self._aesthetic_head(feats) - @torch.no_grad() def compute(self, sample: dict) -> list[MetricResult]: video = sample["video"] # (B, T, C, H, W) diff --git a/fastvideo/eval/metrics/vbench/appearance_style/metric.py b/fastvideo/eval/metrics/vbench/appearance_style/metric.py index 919f2993f..e0ebccefb 100644 --- a/fastvideo/eval/metrics/vbench/appearance_style/metric.py +++ b/fastvideo/eval/metrics/vbench/appearance_style/metric.py @@ -34,7 +34,6 @@ class AppearanceStyleMetric(BaseMetric): requires_reference = False higher_is_better = True needs_gpu = True - batch_unit = "frame" dependencies = ["clip"] backbone = "clip_vit_b32" @@ -60,11 +59,6 @@ class AppearanceStyleMetric(BaseMetric): ) self._model.eval() - def trial_forward(self, batch_size, *, height, width, num_frames): - dummy = torch.randn(batch_size, 3, 224, 224, device=self.device) - with torch.no_grad(): - self._model.encode_image(dummy) - @torch.no_grad() def compute(self, sample: dict) -> list[MetricResult]: import clip diff --git a/fastvideo/eval/metrics/vbench/background_consistency/metric.py b/fastvideo/eval/metrics/vbench/background_consistency/metric.py index 8d7463ffd..006b86829 100644 --- a/fastvideo/eval/metrics/vbench/background_consistency/metric.py +++ b/fastvideo/eval/metrics/vbench/background_consistency/metric.py @@ -36,7 +36,6 @@ class BackgroundConsistencyMetric(BaseMetric): requires_reference = False higher_is_better = True needs_gpu = True - batch_unit = "frame" dependencies = ["clip"] backbone = "clip_vit_b32" @@ -63,11 +62,6 @@ class BackgroundConsistencyMetric(BaseMetric): model.eval() self._model = model - def trial_forward(self, batch_size, *, height, width, num_frames): - dummy = torch.randn(batch_size, 3, 224, 224, device=self.device) - with torch.no_grad(): - self._model.encode_image(dummy) - @torch.no_grad() def compute(self, sample: dict) -> list[MetricResult]: video = sample["video"] # (B, T, C, H, W) diff --git a/fastvideo/eval/metrics/vbench/dynamic_degree/metric.py b/fastvideo/eval/metrics/vbench/dynamic_degree/metric.py index e6b6fc371..0ac2b86f2 100644 --- a/fastvideo/eval/metrics/vbench/dynamic_degree/metric.py +++ b/fastvideo/eval/metrics/vbench/dynamic_degree/metric.py @@ -25,7 +25,6 @@ class DynamicDegreeMetric(BaseMetric): higher_is_better = True needs_gpu = True dependencies = ["easydict"] - batch_unit = "frame_pair" def __init__(self) -> None: super().__init__() @@ -59,18 +58,6 @@ class DynamicDegreeMetric(BaseMetric): model.eval() self._model = model - def trial_forward(self, batch_size, *, height, width, num_frames): - from vbench.third_party.RAFT.core.utils_core.utils import InputPadder - # RAFT's CorrBlock creates a (B*H*W, 1, H, W) correlation volume. - # Cap resolution to avoid integer overflow at large batch sizes. - h, w = min(height, 256), min(width, 256) - img1 = torch.randn(batch_size, 3, h, w, device=self.device) - img2 = torch.randn(batch_size, 3, h, w, device=self.device) - padder = InputPadder(img1.shape) - img1, img2 = padder.pad(img1, img2) - with torch.no_grad(): - self._model(img1, img2, iters=20, test_mode=True) - def _get_score(self, flow: torch.Tensor) -> float: """Top-5% mean flow magnitude (matching VBench dynamic_degree.get_score).""" flo = flow.permute(1, 2, 0).cpu().numpy() diff --git a/fastvideo/eval/metrics/vbench/human_action/metric.py b/fastvideo/eval/metrics/vbench/human_action/metric.py index ffcba7095..2879a04f4 100644 --- a/fastvideo/eval/metrics/vbench/human_action/metric.py +++ b/fastvideo/eval/metrics/vbench/human_action/metric.py @@ -48,7 +48,6 @@ class HumanActionMetric(BaseMetric): requires_reference = False higher_is_better = True needs_gpu = True - batch_unit = "video" dependencies = ["timm"] def __init__(self) -> None: @@ -97,11 +96,6 @@ class HumanActionMetric(BaseMetric): self._model.to(self.device) self._model.eval() - def trial_forward(self, batch_size, *, height, width, num_frames): - dummy = torch.randn(batch_size, 3, 16, 224, 224, device=self.device) - with torch.no_grad(): - self._model(dummy) - @torch.no_grad() def compute(self, sample: dict) -> list[MetricResult]: video = sample["video"] # (B, T, C, H, W) float [0, 1] diff --git a/fastvideo/eval/metrics/vbench/imaging_quality/metric.py b/fastvideo/eval/metrics/vbench/imaging_quality/metric.py index 9d3cb4dc5..598bb721e 100644 --- a/fastvideo/eval/metrics/vbench/imaging_quality/metric.py +++ b/fastvideo/eval/metrics/vbench/imaging_quality/metric.py @@ -21,7 +21,6 @@ class ImagingQualityMetric(BaseMetric): requires_reference = False higher_is_better = True needs_gpu = True - batch_unit = "frame" dependencies = ["pyiqa"] def __init__(self) -> None: @@ -41,15 +40,6 @@ class ImagingQualityMetric(BaseMetric): self._model = pyiqa.create_metric("musiq-spaq", device=self.device) self._model.eval() - def trial_forward(self, batch_size, *, height, width, num_frames): - h, w = min(height, 512), min(width, 512) - if max(height, width) > 512: - scale = 512.0 / max(height, width) - h, w = int(height * scale), int(width * scale) - dummy = torch.randn(batch_size, 3, h, w, device=self.device) - with torch.no_grad(): - self._model(dummy) - @torch.no_grad() def compute(self, sample: dict) -> list[MetricResult]: video = sample["video"] # (B, T, C, H, W) diff --git a/fastvideo/eval/metrics/vbench/motion_smoothness/metric.py b/fastvideo/eval/metrics/vbench/motion_smoothness/metric.py index 60eb9e13d..aede6a94a 100644 --- a/fastvideo/eval/metrics/vbench/motion_smoothness/metric.py +++ b/fastvideo/eval/metrics/vbench/motion_smoothness/metric.py @@ -25,7 +25,6 @@ class MotionSmoothnessMetric(BaseMetric): requires_reference = False higher_is_better = True needs_gpu = True - batch_unit = "frame_pair" dependencies = ["omegaconf"] def __init__(self) -> None: @@ -87,14 +86,6 @@ class MotionSmoothnessMetric(BaseMetric): scale = 1 / np.floor(1 / np.sqrt(scale) * 16) * 16 return float(scale) - def trial_forward(self, batch_size, *, height, width, num_frames): - h, w = min(height, 512), min(width, 512) - img0 = torch.randn(batch_size, 3, h, w, device=self.device) - img1 = torch.randn(batch_size, 3, h, w, device=self.device) - embt = self._embt.expand(batch_size, -1, -1, -1) - with torch.no_grad(): - self._model(img0, img1, embt, scale_factor=1.0, eval=True) - @torch.no_grad() def compute(self, sample: dict) -> list[MetricResult]: from vbench.third_party.amt.utils.utils import ( diff --git a/fastvideo/eval/metrics/vbench/overall_consistency/metric.py b/fastvideo/eval/metrics/vbench/overall_consistency/metric.py index 4edd8cddc..0da310f3f 100644 --- a/fastvideo/eval/metrics/vbench/overall_consistency/metric.py +++ b/fastvideo/eval/metrics/vbench/overall_consistency/metric.py @@ -34,7 +34,6 @@ class OverallConsistencyMetric(BaseMetric): requires_reference = False higher_is_better = True needs_gpu = True - batch_unit = "video" dependencies = ["timm", "einops", "clip"] backbone = "viclip" @@ -73,11 +72,6 @@ class OverallConsistencyMetric(BaseMetric): self._model.to(self.device) self._model.eval() - def trial_forward(self, batch_size, *, height, width, num_frames): - dummy = torch.randn(batch_size, 8, 3, 224, 224, device=self.device) - with torch.no_grad(): - self._model.encode_vision(dummy, test=True) - @torch.no_grad() def compute(self, sample: dict) -> list[MetricResult]: video = sample["video"] # (B, T, C, H, W) diff --git a/fastvideo/eval/metrics/vbench/scene/metric.py b/fastvideo/eval/metrics/vbench/scene/metric.py index 187132b19..b31df44a4 100644 --- a/fastvideo/eval/metrics/vbench/scene/metric.py +++ b/fastvideo/eval/metrics/vbench/scene/metric.py @@ -37,7 +37,6 @@ class SceneMetric(BaseMetric): requires_reference = False higher_is_better = True needs_gpu = True - batch_unit = "video" dependencies = ["transformers", "qwen_omni_utils"] backbone = "avocado" diff --git a/fastvideo/eval/metrics/vbench/subject_consistency/metric.py b/fastvideo/eval/metrics/vbench/subject_consistency/metric.py index fcbd84009..bfb194c5e 100644 --- a/fastvideo/eval/metrics/vbench/subject_consistency/metric.py +++ b/fastvideo/eval/metrics/vbench/subject_consistency/metric.py @@ -28,7 +28,6 @@ class SubjectConsistencyMetric(BaseMetric): requires_reference = False higher_is_better = True needs_gpu = True - batch_unit = "frame" backbone = "dino_vitb16" def __init__(self) -> None: @@ -49,11 +48,6 @@ class SubjectConsistencyMetric(BaseMetric): model.eval() self._model = model - def trial_forward(self, batch_size, *, height, width, num_frames): - dummy = torch.randn(batch_size, 3, 224, 224, device=self.device) - with torch.no_grad(): - self._model(dummy) - @torch.no_grad() def compute(self, sample: dict) -> list[MetricResult]: video = sample["video"] # (B, T, C, H, W) diff --git a/fastvideo/eval/metrics/vbench/temporal_flickering/metric.py b/fastvideo/eval/metrics/vbench/temporal_flickering/metric.py index af99ad4f1..2612b23ac 100644 --- a/fastvideo/eval/metrics/vbench/temporal_flickering/metric.py +++ b/fastvideo/eval/metrics/vbench/temporal_flickering/metric.py @@ -21,7 +21,6 @@ class TemporalFlickeringMetric(BaseMetric): requires_reference = False higher_is_better = True needs_gpu = False - batch_unit = "video" @torch.no_grad() def compute(self, sample: dict) -> list[MetricResult]: diff --git a/fastvideo/eval/metrics/videoscore2/metric.py b/fastvideo/eval/metrics/videoscore2/metric.py index 8d8195ca6..f7fb3d088 100644 --- a/fastvideo/eval/metrics/videoscore2/metric.py +++ b/fastvideo/eval/metrics/videoscore2/metric.py @@ -148,7 +148,6 @@ class VideoScore2Metric(BaseMetric): higher_is_better = True needs_gpu = True dependencies = ["transformers", "qwen_vl_utils"] - batch_unit = "video" def __init__( self,