nodes/hao-ai-lab-FastVideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[bugfix]: give GB200 its own SSIM reference folder instead of B200's (#1676)

Browse Source
This commit is contained in:
William Lin
2026-08-03 15:14:25 -07:00
committed by GitHub
parent e8812bef0b
commit c3d07c870b
4 changed files with 80 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fastvideo/tests/ssim/inference_similarity_utils.py
+17 -24
View File
@@ -9,8 +9,7 @@ from logging import Logger
from fastvideo import VideoGenerator
from fastvideo.tests.ssim.bootstrap_references import (
xfail_missing_reference_in_bootstrap_mode,
)
xfail_missing_reference_in_bootstrap_mode, )
from fastvideo.tests.ssim.reference_utils import (
build_generated_output_dir,
build_reference_folder_path,
@@ -26,6 +25,10 @@ DEVICE_MAPPINGS = (
("L40S", "L40S"),
("H100", "H100"),
("H200", "H200"),
# GB200 must precede B200: the lookup is a substring scan and
# "B200" matches "NVIDIA GB200", so the coarser pattern would
# otherwise claim every Grace-Blackwell host.
("GB200", "GB200"),
("B200", "B200"),
)
@@ -115,11 +118,9 @@ def _assert_similarity(
reference_folder=reference_folder,
artifact_kind=artifact_kind,
)
error_msg = (
f"Reference video folder does not exist: {reference_folder}\n"
f"To download reference videos, run:\n"
f" python fastvideo/tests/ssim/reference_videos_cli.py download"
)
error_msg = (f"Reference video folder does not exist: {reference_folder}\n"
f"To download reference videos, run:\n"
f" python fastvideo/tests/ssim/reference_videos_cli.py download")
raise FileNotFoundError(error_msg)
try:
@@ -167,15 +168,11 @@ def _assert_similarity(
if not success:
logger.error("Failed to write SSIM results to file")
assert mean_ssim >= min_acceptable_ssim, (
f"SSIM value {mean_ssim} is below threshold {min_acceptable_ssim} "
f"for {model_id} with backend {attention_backend_name}"
)
assert mean_ssim >= min_acceptable_ssim, (f"SSIM value {mean_ssim} is below threshold {min_acceptable_ssim} "
f"for {model_id} with backend {attention_backend_name}")
def build_init_kwargs(
base_params: dict[str, object],
) -> dict[str, object]:
def build_init_kwargs(base_params: dict[str, object], ) -> dict[str, object]:
init_kwargs: dict[str, object] = {
"num_gpus": base_params["num_gpus"],
"sp_size": base_params.get("sp_size", 1),
@@ -193,18 +190,14 @@ def build_init_kwargs(
init_kwargs["text_encoder_precisions"] = base_params["text-encoder-precision"]
if base_params.get("ltx2_vae_tiling"):
init_kwargs["ltx2_vae_tiling"] = True
init_kwargs["ltx2_vae_spatial_tile_size_in_pixels"] = base_params.get(
"ltx2_vae_spatial_tile_size_in_pixels", 512
)
init_kwargs["ltx2_vae_spatial_tile_size_in_pixels"] = base_params.get("ltx2_vae_spatial_tile_size_in_pixels",
512)
init_kwargs["ltx2_vae_spatial_tile_overlap_in_pixels"] = base_params.get(
"ltx2_vae_spatial_tile_overlap_in_pixels", 64
)
init_kwargs["ltx2_vae_temporal_tile_size_in_frames"] = base_params.get(
"ltx2_vae_temporal_tile_size_in_frames", 64
)
"ltx2_vae_spatial_tile_overlap_in_pixels", 64)
init_kwargs["ltx2_vae_temporal_tile_size_in_frames"] = base_params.get("ltx2_vae_temporal_tile_size_in_frames",
64)
init_kwargs["ltx2_vae_temporal_tile_overlap_in_frames"] = base_params.get(
"ltx2_vae_temporal_tile_overlap_in_frames", 24
)
"ltx2_vae_temporal_tile_overlap_in_frames", 24)
return init_kwargs
fastvideo/tests/ssim/test_device_reference_folder.py
+48
View File
@@ -0,0 +1,48 @@
# SPDX-License-Identifier: Apache-2.0
"""Device-name -> reference-folder resolution.
The lookup is a substring scan over an ordered table, which makes it
order-sensitive in a way that is easy to get wrong: ``"B200"`` is a substring of
``"NVIDIA GB200"``, so a table listing B200 first silently routes every
Grace-Blackwell host to the B200 references. That is not a cosmetic mismatch --
whichever host seeds a reference defines the baseline every later run on the
*other* host is scored against.
"""
from __future__ import annotations
import pytest
from fastvideo.tests.ssim.inference_similarity_utils import DEVICE_MAPPINGS
from fastvideo.tests.ssim.reference_utils import resolve_device_reference_folder
@pytest.mark.parametrize(
("device_name", "expected"),
[
("NVIDIA GB200", "GB200_reference_videos"),
("NVIDIA B200", "B200_reference_videos"),
("NVIDIA H200", "H200_reference_videos"),
("NVIDIA H100 80GB HBM3", "H100_reference_videos"),
("NVIDIA L40S", "L40S_reference_videos"),
("NVIDIA A40", "A40_reference_videos"),
],
)
def test_device_resolves_to_its_own_folder(device_name: str, expected: str) -> None:
assert resolve_device_reference_folder(DEVICE_MAPPINGS, device_name=device_name) == expected
def test_gb200_is_not_captured_by_the_b200_pattern() -> None:
"""The regression this file exists for.
Asserting the mapping order directly, so reordering the table fails here
rather than silently scoring GB200 runs against B200 references.
"""
patterns = [pattern for pattern, _ in DEVICE_MAPPINGS]
assert "GB200" in patterns, "GB200 missing from DEVICE_MAPPINGS"
assert patterns.index("GB200") < patterns.index("B200"), (
"GB200 must precede B200: the lookup is a substring scan, so B200 "
"would otherwise match 'NVIDIA GB200' first")
def test_unknown_device_returns_none() -> None:
assert resolve_device_reference_folder(DEVICE_MAPPINGS, device_name="NVIDIA RTX 5090") is None
fastvideo/tests/ssim/test_glm_image_similarity.py
+7 -8
View File
@@ -10,8 +10,7 @@ import torch
from fastvideo.logger import init_logger
from fastvideo.tests.ssim.inference_similarity_utils import (
run_text_to_video_similarity_test,
)
run_text_to_video_similarity_test, )
from fastvideo.tests.ssim.reference_utils import (
get_cuda_device_name,
resolve_device_reference_folder,
@@ -22,9 +21,7 @@ logger = init_logger(__name__)
REQUIRED_GPUS = 1
REPO_ROOT = Path(__file__).resolve().parents[3]
LOCAL_WEIGHTS_DIR = Path(
os.getenv("GLM_IMAGE_LOCAL_WEIGHTS_DIR",
REPO_ROOT / "official_weights" / "glm_image"))
LOCAL_WEIGHTS_DIR = Path(os.getenv("GLM_IMAGE_LOCAL_WEIGHTS_DIR", REPO_ROOT / "official_weights" / "glm_image"))
GLM_IMAGE_MODEL_PATH = os.getenv("GLM_IMAGE_MODEL_DIR", str(LOCAL_WEIGHTS_DIR))
device_reference_folder = resolve_device_reference_folder(
@@ -33,6 +30,10 @@ device_reference_folder = resolve_device_reference_folder(
("L40S", "L40S"),
("H100", "H100"),
("H200", "H200"),
# GB200 must precede B200: the lookup is a substring scan and
# "B200" matches "NVIDIA GB200", so the coarser pattern would
# otherwise claim every Grace-Blackwell host.
("GB200", "GB200"),
("B200", "B200"),
),
device_name=get_cuda_device_name(),
@@ -87,9 +88,7 @@ GLM_IMAGE_FULL_QUALITY_MODEL_TO_PARAMS = {
def _has_weights() -> bool:
required = ["transformer", "vae", "text_encoder",
"vision_language_encoder", "processor", "tokenizer",
"scheduler"]
required = ["transformer", "vae", "text_encoder", "vision_language_encoder", "processor", "tokenizer", "scheduler"]
return all((LOCAL_WEIGHTS_DIR / r).exists() for r in required)
fastvideo/tests/ssim/test_zimage_similarity.py
+8 -10
View File
@@ -10,8 +10,7 @@ import torch
from fastvideo.logger import init_logger
from fastvideo.tests.ssim.inference_similarity_utils import (
run_text_to_video_similarity_test,
)
run_text_to_video_similarity_test, )
from fastvideo.tests.ssim.reference_utils import (
get_cuda_device_name,
resolve_device_reference_folder,
@@ -44,6 +43,10 @@ device_reference_folder = resolve_device_reference_folder(
("L40S", "L40S"),
("H100", "H100"),
("H200", "H200"),
# GB200 must precede B200: the lookup is a substring scan and
# "B200" matches "NVIDIA GB200", so the coarser pattern would
# otherwise claim every Grace-Blackwell host.
("GB200", "GB200"),
("B200", "B200"),
),
device_name=get_cuda_device_name(),
@@ -67,10 +70,7 @@ ZIMAGE_MODEL_TO_PARAMS: dict[str, dict[str, object]] = {
},
}
ZIMAGE_FULL_QUALITY_MODEL_TO_PARAMS = {
model_id: dict(params)
for model_id, params in ZIMAGE_MODEL_TO_PARAMS.items()
}
ZIMAGE_FULL_QUALITY_MODEL_TO_PARAMS = {model_id: dict(params) for model_id, params in ZIMAGE_MODEL_TO_PARAMS.items()}
@pytest.mark.skipif(
@@ -87,10 +87,8 @@ def test_zimage_similarity(
) -> None:
is_hf_repo = "/" in ZIMAGE_MODEL_PATH and not ZIMAGE_MODEL_PATH.startswith("/")
if not is_hf_repo and not os.path.isdir(ZIMAGE_MODEL_PATH):
pytest.skip(
f"Z-Image weights not found at {ZIMAGE_MODEL_PATH} "
"(set ZIMAGE_MODEL_DIR to override)"
)
pytest.skip(f"Z-Image weights not found at {ZIMAGE_MODEL_PATH} "
"(set ZIMAGE_MODEL_DIR to override)")
run_text_to_video_similarity_test(
logger=logger,