diff --git a/fastvideo/tests/ssim/inference_similarity_utils.py b/fastvideo/tests/ssim/inference_similarity_utils.py index a3e419d35..9ae9b0d46 100644 --- a/fastvideo/tests/ssim/inference_similarity_utils.py +++ b/fastvideo/tests/ssim/inference_similarity_utils.py @@ -9,8 +9,7 @@ from logging import Logger from fastvideo import VideoGenerator from fastvideo.tests.ssim.bootstrap_references import ( - xfail_missing_reference_in_bootstrap_mode, -) + xfail_missing_reference_in_bootstrap_mode, ) from fastvideo.tests.ssim.reference_utils import ( build_generated_output_dir, build_reference_folder_path, @@ -26,6 +25,10 @@ DEVICE_MAPPINGS = ( ("L40S", "L40S"), ("H100", "H100"), ("H200", "H200"), + # GB200 must precede B200: the lookup is a substring scan and + # "B200" matches "NVIDIA GB200", so the coarser pattern would + # otherwise claim every Grace-Blackwell host. + ("GB200", "GB200"), ("B200", "B200"), ) @@ -115,11 +118,9 @@ def _assert_similarity( reference_folder=reference_folder, artifact_kind=artifact_kind, ) - error_msg = ( - f"Reference video folder does not exist: {reference_folder}\n" - f"To download reference videos, run:\n" - f" python fastvideo/tests/ssim/reference_videos_cli.py download" - ) + error_msg = (f"Reference video folder does not exist: {reference_folder}\n" + f"To download reference videos, run:\n" + f" python fastvideo/tests/ssim/reference_videos_cli.py download") raise FileNotFoundError(error_msg) try: @@ -167,15 +168,11 @@ def _assert_similarity( if not success: logger.error("Failed to write SSIM results to file") - assert mean_ssim >= min_acceptable_ssim, ( - f"SSIM value {mean_ssim} is below threshold {min_acceptable_ssim} " - f"for {model_id} with backend {attention_backend_name}" - ) + assert mean_ssim >= min_acceptable_ssim, (f"SSIM value {mean_ssim} is below threshold {min_acceptable_ssim} " + f"for {model_id} with backend {attention_backend_name}") -def build_init_kwargs( - base_params: dict[str, object], -) -> dict[str, object]: +def build_init_kwargs(base_params: dict[str, object], ) -> dict[str, object]: init_kwargs: dict[str, object] = { "num_gpus": base_params["num_gpus"], "sp_size": base_params.get("sp_size", 1), @@ -193,18 +190,14 @@ def build_init_kwargs( init_kwargs["text_encoder_precisions"] = base_params["text-encoder-precision"] if base_params.get("ltx2_vae_tiling"): init_kwargs["ltx2_vae_tiling"] = True - init_kwargs["ltx2_vae_spatial_tile_size_in_pixels"] = base_params.get( - "ltx2_vae_spatial_tile_size_in_pixels", 512 - ) + init_kwargs["ltx2_vae_spatial_tile_size_in_pixels"] = base_params.get("ltx2_vae_spatial_tile_size_in_pixels", + 512) init_kwargs["ltx2_vae_spatial_tile_overlap_in_pixels"] = base_params.get( - "ltx2_vae_spatial_tile_overlap_in_pixels", 64 - ) - init_kwargs["ltx2_vae_temporal_tile_size_in_frames"] = base_params.get( - "ltx2_vae_temporal_tile_size_in_frames", 64 - ) + "ltx2_vae_spatial_tile_overlap_in_pixels", 64) + init_kwargs["ltx2_vae_temporal_tile_size_in_frames"] = base_params.get("ltx2_vae_temporal_tile_size_in_frames", + 64) init_kwargs["ltx2_vae_temporal_tile_overlap_in_frames"] = base_params.get( - "ltx2_vae_temporal_tile_overlap_in_frames", 24 - ) + "ltx2_vae_temporal_tile_overlap_in_frames", 24) return init_kwargs diff --git a/fastvideo/tests/ssim/test_device_reference_folder.py b/fastvideo/tests/ssim/test_device_reference_folder.py new file mode 100644 index 000000000..89e7c99a0 --- /dev/null +++ b/fastvideo/tests/ssim/test_device_reference_folder.py @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Device-name -> reference-folder resolution. + +The lookup is a substring scan over an ordered table, which makes it +order-sensitive in a way that is easy to get wrong: ``"B200"`` is a substring of +``"NVIDIA GB200"``, so a table listing B200 first silently routes every +Grace-Blackwell host to the B200 references. That is not a cosmetic mismatch -- +whichever host seeds a reference defines the baseline every later run on the +*other* host is scored against. +""" +from __future__ import annotations + +import pytest + +from fastvideo.tests.ssim.inference_similarity_utils import DEVICE_MAPPINGS +from fastvideo.tests.ssim.reference_utils import resolve_device_reference_folder + + +@pytest.mark.parametrize( + ("device_name", "expected"), + [ + ("NVIDIA GB200", "GB200_reference_videos"), + ("NVIDIA B200", "B200_reference_videos"), + ("NVIDIA H200", "H200_reference_videos"), + ("NVIDIA H100 80GB HBM3", "H100_reference_videos"), + ("NVIDIA L40S", "L40S_reference_videos"), + ("NVIDIA A40", "A40_reference_videos"), + ], +) +def test_device_resolves_to_its_own_folder(device_name: str, expected: str) -> None: + assert resolve_device_reference_folder(DEVICE_MAPPINGS, device_name=device_name) == expected + + +def test_gb200_is_not_captured_by_the_b200_pattern() -> None: + """The regression this file exists for. + + Asserting the mapping order directly, so reordering the table fails here + rather than silently scoring GB200 runs against B200 references. + """ + patterns = [pattern for pattern, _ in DEVICE_MAPPINGS] + assert "GB200" in patterns, "GB200 missing from DEVICE_MAPPINGS" + assert patterns.index("GB200") < patterns.index("B200"), ( + "GB200 must precede B200: the lookup is a substring scan, so B200 " + "would otherwise match 'NVIDIA GB200' first") + + +def test_unknown_device_returns_none() -> None: + assert resolve_device_reference_folder(DEVICE_MAPPINGS, device_name="NVIDIA RTX 5090") is None diff --git a/fastvideo/tests/ssim/test_glm_image_similarity.py b/fastvideo/tests/ssim/test_glm_image_similarity.py index 87f8ab357..e2e315755 100644 --- a/fastvideo/tests/ssim/test_glm_image_similarity.py +++ b/fastvideo/tests/ssim/test_glm_image_similarity.py @@ -10,8 +10,7 @@ import torch from fastvideo.logger import init_logger from fastvideo.tests.ssim.inference_similarity_utils import ( - run_text_to_video_similarity_test, -) + run_text_to_video_similarity_test, ) from fastvideo.tests.ssim.reference_utils import ( get_cuda_device_name, resolve_device_reference_folder, @@ -22,9 +21,7 @@ logger = init_logger(__name__) REQUIRED_GPUS = 1 REPO_ROOT = Path(__file__).resolve().parents[3] -LOCAL_WEIGHTS_DIR = Path( - os.getenv("GLM_IMAGE_LOCAL_WEIGHTS_DIR", - REPO_ROOT / "official_weights" / "glm_image")) +LOCAL_WEIGHTS_DIR = Path(os.getenv("GLM_IMAGE_LOCAL_WEIGHTS_DIR", REPO_ROOT / "official_weights" / "glm_image")) GLM_IMAGE_MODEL_PATH = os.getenv("GLM_IMAGE_MODEL_DIR", str(LOCAL_WEIGHTS_DIR)) device_reference_folder = resolve_device_reference_folder( @@ -33,6 +30,10 @@ device_reference_folder = resolve_device_reference_folder( ("L40S", "L40S"), ("H100", "H100"), ("H200", "H200"), + # GB200 must precede B200: the lookup is a substring scan and + # "B200" matches "NVIDIA GB200", so the coarser pattern would + # otherwise claim every Grace-Blackwell host. + ("GB200", "GB200"), ("B200", "B200"), ), device_name=get_cuda_device_name(), @@ -87,9 +88,7 @@ GLM_IMAGE_FULL_QUALITY_MODEL_TO_PARAMS = { def _has_weights() -> bool: - required = ["transformer", "vae", "text_encoder", - "vision_language_encoder", "processor", "tokenizer", - "scheduler"] + required = ["transformer", "vae", "text_encoder", "vision_language_encoder", "processor", "tokenizer", "scheduler"] return all((LOCAL_WEIGHTS_DIR / r).exists() for r in required) diff --git a/fastvideo/tests/ssim/test_zimage_similarity.py b/fastvideo/tests/ssim/test_zimage_similarity.py index 2a96d7efe..541b6ff6b 100644 --- a/fastvideo/tests/ssim/test_zimage_similarity.py +++ b/fastvideo/tests/ssim/test_zimage_similarity.py @@ -10,8 +10,7 @@ import torch from fastvideo.logger import init_logger from fastvideo.tests.ssim.inference_similarity_utils import ( - run_text_to_video_similarity_test, -) + run_text_to_video_similarity_test, ) from fastvideo.tests.ssim.reference_utils import ( get_cuda_device_name, resolve_device_reference_folder, @@ -44,6 +43,10 @@ device_reference_folder = resolve_device_reference_folder( ("L40S", "L40S"), ("H100", "H100"), ("H200", "H200"), + # GB200 must precede B200: the lookup is a substring scan and + # "B200" matches "NVIDIA GB200", so the coarser pattern would + # otherwise claim every Grace-Blackwell host. + ("GB200", "GB200"), ("B200", "B200"), ), device_name=get_cuda_device_name(), @@ -67,10 +70,7 @@ ZIMAGE_MODEL_TO_PARAMS: dict[str, dict[str, object]] = { }, } -ZIMAGE_FULL_QUALITY_MODEL_TO_PARAMS = { - model_id: dict(params) - for model_id, params in ZIMAGE_MODEL_TO_PARAMS.items() -} +ZIMAGE_FULL_QUALITY_MODEL_TO_PARAMS = {model_id: dict(params) for model_id, params in ZIMAGE_MODEL_TO_PARAMS.items()} @pytest.mark.skipif( @@ -87,10 +87,8 @@ def test_zimage_similarity( ) -> None: is_hf_repo = "/" in ZIMAGE_MODEL_PATH and not ZIMAGE_MODEL_PATH.startswith("/") if not is_hf_repo and not os.path.isdir(ZIMAGE_MODEL_PATH): - pytest.skip( - f"Z-Image weights not found at {ZIMAGE_MODEL_PATH} " - "(set ZIMAGE_MODEL_DIR to override)" - ) + pytest.skip(f"Z-Image weights not found at {ZIMAGE_MODEL_PATH} " + "(set ZIMAGE_MODEL_DIR to override)") run_text_to_video_similarity_test( logger=logger,