From a0291a57c1d67965d2f75f05cd2d7a052ac1d611 Mon Sep 17 00:00:00 2001 From: Suhaan Khurana <114335244+SuhaanCoding@users.noreply.github.com> Date: Wed, 7 Oct 2026 02:44:24 -0700 Subject: [PATCH] [ci] Add Wan2.2-TI2V-5B SSIM regression test (#1772) Co-authored-by: SuhaanCommits Co-authored-by: Claude Fable 5 Co-authored-by: SolitaryThinker --- .github/scripts/plan_merge_ci.py | 1 + .../tests/contract/test_merge_ci_plan.py | 7 +- fastvideo/tests/ssim/README.md | 5 +- .../tests/ssim/test_wan_ti2v_similarity.py | 155 ++++++++++++++++++ 4 files changed, 164 insertions(+), 4 deletions(-) create mode 100644 fastvideo/tests/ssim/test_wan_ti2v_similarity.py diff --git a/.github/scripts/plan_merge_ci.py b/.github/scripts/plan_merge_ci.py index 867fca88f..eb6dfad58 100644 --- a/.github/scripts/plan_merge_ci.py +++ b/.github/scripts/plan_merge_ci.py @@ -222,6 +222,7 @@ FAMILY_COVERAGE = ( "test_causal_similarity.py", "test_wan_i2v_similarity.py", "test_wan_t2v_similarity.py", + "test_wan_ti2v_similarity.py", ), ), FamilyCoverage( diff --git a/fastvideo/tests/contract/test_merge_ci_plan.py b/fastvideo/tests/contract/test_merge_ci_plan.py index 9fd5713ec..f68c57c4b 100644 --- a/fastvideo/tests/contract/test_merge_ci_plan.py +++ b/fastvideo/tests/contract/test_merge_ci_plan.py @@ -50,7 +50,8 @@ def test_model_family_change_selects_focused_golden_and_ssim(path, goldens): assert plan.encoded_lanes() == ",golden-gate,ssim," assert plan.encoded_golden_tests() == goldens assert plan.encoded_ssim_tests() == ( - "test_causal_similarity.py,test_wan_i2v_similarity.py,test_wan_t2v_similarity.py") + "test_causal_similarity.py,test_wan_i2v_similarity.py,test_wan_t2v_similarity.py," + "test_wan_ti2v_similarity.py") def test_hunyuan15_change_selects_its_i2v_ssim_test(): @@ -101,9 +102,11 @@ def test_wan_family_relocation_keeps_shared_registry_coverage(): assert plan.encoded_lanes() == ",golden-gate,ssim," assert plan.encoded_golden_tests() == "all" + # Every Wan-family SSIM file plus the shared-registry smoke (flux t2i). assert plan.encoded_ssim_tests() == ( "test_causal_similarity.py,test_flux_t2i_similarity.py," - "test_wan_i2v_similarity.py,test_wan_t2v_similarity.py") + "test_wan_i2v_similarity.py,test_wan_t2v_similarity.py," + "test_wan_ti2v_similarity.py") def test_flux2_change_does_not_pull_unrelated_flux1_quality_tests(): diff --git a/fastvideo/tests/ssim/README.md b/fastvideo/tests/ssim/README.md index 25f708195..0b35fbcef 100644 --- a/fastvideo/tests/ssim/README.md +++ b/fastvideo/tests/ssim/README.md @@ -1,6 +1,7 @@ The reference videos are used as part of e2e SSIM regression tests. -Wan inference coverage now lives in `test_wan_t2v_similarity.py` and -`test_wan_i2v_similarity.py` alongside the other model-specific SSIM files. +Wan inference coverage now lives in `test_wan_t2v_similarity.py`, +`test_wan_i2v_similarity.py` and `test_wan_ti2v_similarity.py` alongside the +other model-specific SSIM files. These tests compare newly generated videos against references to detect quality regressions. diff --git a/fastvideo/tests/ssim/test_wan_ti2v_similarity.py b/fastvideo/tests/ssim/test_wan_ti2v_similarity.py new file mode 100644 index 000000000..f7b1f9c4c --- /dev/null +++ b/fastvideo/tests/ssim/test_wan_ti2v_similarity.py @@ -0,0 +1,155 @@ +# SPDX-License-Identifier: Apache-2.0 +"""SSIM regression for Wan2.2-TI2V-5B. + +One checkpoint serves two workloads, so there are two cases: text-to-video, +and image-to-video when an image is passed. For the image case the pipeline +ignores ``height``/``width`` and derives the output size from the image's +aspect ratio inside a hard-coded 480x832 pixel budget (see +``InputValidationStage``), so the sizes below bind only the text case. +""" +import os + +import pytest + +from fastvideo.api.sampling_param import SamplingParam +from fastvideo.logger import init_logger +from fastvideo.tests.ssim.bootstrap_references import bootstrap_mode_enabled +from fastvideo.tests.ssim.inference_similarity_utils import ( + resolve_inference_device_reference_folder, + run_image_to_video_similarity_test, + run_text_to_video_similarity_test, +) +from fastvideo.tests.ssim.reference_utils import build_reference_folder_path, get_output_quality_tier + +logger = init_logger(__name__) + +REQUIRED_GPUS = 1 + +device_reference_folder = resolve_inference_device_reference_folder(logger) + + +def _skip_without_canonical_reference(attention_backend_name: str, model_id: str) -> None: + """Skip (instead of hard-failing) while no canonical reference exists. + + The shared harness raises FileNotFoundError for a missing reference + folder, which would fail this file's own merge gate and every Wan-family + merge gate before references are generated and promoted. Bootstrap mode + keeps its draft-generation flow and does not skip. + """ + if bootstrap_mode_enabled(): + return + reference_folder = build_reference_folder_path( + os.path.dirname(os.path.abspath(__file__)), + device_reference_folder, + model_id, + attention_backend_name, + ) + if not os.path.exists(reference_folder): + pytest.skip( + f"No canonical SSIM reference for {model_id}/{attention_backend_name}: " + f"{reference_folder} is missing. Generate one with " + "FASTVIDEO_SSIM_BOOTSTRAP_MODE=1, review the draft, then promote it via " + f"`python fastvideo/tests/ssim/reference_videos_cli.py promote-draft " + f"--quality-tier {get_output_quality_tier()} --model-id {model_id}`." + ) + +# Few steps and a short clip keep the CI run small; everything else (guidance, +# flow shift, fps, negative prompt) comes from the model's registered preset. +WAN_TI2V_PARAMS = { + "num_gpus": 1, + "model_path": "Wan-AI/Wan2.2-TI2V-5B-Diffusers", + "height": 480, + "width": 832, + "num_frames": 45, + "num_inference_steps": 4, + "seed": 1024, +} +_WAN_TI2V_FULL_QUALITY_DEFAULTS = SamplingParam.from_pretrained(WAN_TI2V_PARAMS["model_path"]) +WAN_TI2V_FULL_QUALITY_PARAMS = { + "num_gpus": WAN_TI2V_PARAMS["num_gpus"], + "model_path": WAN_TI2V_PARAMS["model_path"], + "height": _WAN_TI2V_FULL_QUALITY_DEFAULTS.height, + "width": _WAN_TI2V_FULL_QUALITY_DEFAULTS.width, + "num_frames": WAN_TI2V_PARAMS["num_frames"], # preset default is 121; 45 keeps the run in budget + "num_inference_steps": _WAN_TI2V_FULL_QUALITY_DEFAULTS.num_inference_steps, + "seed": _WAN_TI2V_FULL_QUALITY_DEFAULTS.seed, +} + +WAN_TI2V_MODEL_TO_PARAMS = { + "Wan2.2-TI2V-5B-Diffusers": WAN_TI2V_PARAMS, +} +FULL_QUALITY_WAN_TI2V_MODEL_TO_PARAMS = { + "Wan2.2-TI2V-5B-Diffusers": WAN_TI2V_FULL_QUALITY_PARAMS, +} + +# Prompts and image are the ones in examples/inference/basic/basic_wan2_2_ti2v.py. +WAN_TI2V_T2V_TEST_PROMPTS = [ + ("A majestic lion strides across the golden savanna, its powerful frame glistening under the warm " + "afternoon sun. The tall grass ripples gently in the breeze, enhancing the lion's commanding presence. " + "The tone is vibrant, embodying the raw energy of the wild. Low angle, steady tracking shot, cinematic."), +] + +WAN_TI2V_I2V_TEST_CASES = [ + ( + ("Summer beach vacation style, a white cat wearing sunglasses sits on a surfboard. The fluffy-furred " + "feline gazes directly at the camera with a relaxed expression. Blurred beach scenery forms the " + "background featuring crystal-clear waters, distant green hills, and a blue sky dotted with white " + "clouds. The cat assumes a naturally relaxed posture, as if savoring the sea breeze and warm sunlight. " + "A close-up shot highlights the feline's intricate details and the refreshing atmosphere of the seaside."), + "https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/wan_i2v_input.JPG", + ), +] + + +@pytest.mark.parametrize("prompt", WAN_TI2V_T2V_TEST_PROMPTS) +@pytest.mark.parametrize("attention_backend_name", ["FLASH_ATTN", "TORCH_SDPA"]) +@pytest.mark.parametrize("model_id", list(WAN_TI2V_MODEL_TO_PARAMS.keys())) +def test_wan_ti2v_t2v_inference_similarity( + prompt: str, + attention_backend_name: str, + model_id: str, +) -> None: + _skip_without_canonical_reference(attention_backend_name, model_id) + run_text_to_video_similarity_test( + logger=logger, + script_dir=os.path.dirname(os.path.abspath(__file__)), + device_reference_folder=device_reference_folder, + prompt=prompt, + attention_backend_name=attention_backend_name, + model_id=model_id, + default_params_map=WAN_TI2V_MODEL_TO_PARAMS, + full_quality_params_map=FULL_QUALITY_WAN_TI2V_MODEL_TO_PARAMS, + # Provisional bound copied from test_wan_t2v_similarity.py + # (Wan2.1-T2V-1.3B, 4 steps). Calibrate against the promoted + # Wan2.2-TI2V-5B reference and record before/after MS-SSIM numbers + # per fastvideo/tests/ssim/AGENTS.md "PR Expectations". + min_acceptable_ssim=0.93, + ) + + +@pytest.mark.parametrize(("prompt", "image_path"), WAN_TI2V_I2V_TEST_CASES) +@pytest.mark.parametrize("attention_backend_name", ["FLASH_ATTN"]) +@pytest.mark.parametrize("model_id", list(WAN_TI2V_MODEL_TO_PARAMS.keys())) +def test_wan_ti2v_i2v_inference_similarity( + prompt: str, + image_path: str, + attention_backend_name: str, + model_id: str, +) -> None: + _skip_without_canonical_reference(attention_backend_name, model_id) + run_image_to_video_similarity_test( + logger=logger, + script_dir=os.path.dirname(os.path.abspath(__file__)), + device_reference_folder=device_reference_folder, + prompt=prompt, + image_path=image_path, + attention_backend_name=attention_backend_name, + model_id=model_id, + default_params_map=WAN_TI2V_MODEL_TO_PARAMS, + full_quality_params_map=FULL_QUALITY_WAN_TI2V_MODEL_TO_PARAMS, + # Provisional bound copied from test_wan_i2v_similarity.py + # (Wan2.1-I2V-14B-480P, 2 steps). Calibrate against the promoted + # Wan2.2-TI2V-5B reference and record before/after MS-SSIM numbers + # per fastvideo/tests/ssim/AGENTS.md "PR Expectations". + min_acceptable_ssim=0.97, + )