[ci]: cover transformers 5.x config compat for Qwen2.5-VL (#1576)

transformers 5.x stopped forwarding text-model attributes from the
composite Qwen2_5_VLConfig, turned rope_scaling into a property backed by
rope_parameters, dropped "default" from ROPE_INIT_FUNCTIONS and moved
rope_theta inside rope_parameters. The compat layer absorbing all of this
landed incidentally in 02e1143f (#1471) and has no test coverage, while
#1431 unpinned transformers to >=4.57.3 -- so fresh installs now resolve
to 5.13.0 and every new user is on the v5 path.

Pin both the config normalization and the resulting RoPE numerics.
CPU-only and weight-free: the config is built inline, not downloaded.
Also covers the reason1.py fallback config, which carries no rope_scaling
and takes a different branch on each transformers line.

Verified passing on transformers 4.57.3, 5.12.1 and 5.13.0. Neutering
_flatten_text_config to a no-op fails the suite on 5.x, reproducing the
AttributeErrors reported in #1576.

fastvideo/attention/backends/sdpa.py

@@ -15,8 +15,14 @@ class SDPABackend(AttentionBackend):
     accept_output_buffer: bool = True
 
     @staticmethod
-    def get_supported_head_sizes() -> list[int]:
-        return [32, 64, 96, 128, 160, 192, 224, 256]
+    def get_supported_head_sizes() -> list[int] | None:
+        # torch.nn.functional.scaled_dot_product_attention is head-size
+        # agnostic: the math backend handles any size and the fused kernels
+        # fall back internally when they cannot. The previous list
+        # ([32, 64, ..., 256]) was copied from FlashAttentionBackend in the
+        # v1 refactor (#270) and wrongly excluded sizes such as 80 (e.g.
+        # CLIP vision encoders). None means "no restriction".
+        return None
 
     @staticmethod
     def get_name() -> str:

fastvideo/models/dits/ltx2.py

@@ -72,8 +72,13 @@ def _rms_norm_dispatch(
     """Use QuACK RMSNorm only for LTX-2 refine stage, else Torch RMSNorm."""
     if _is_ltx2_refine_stage():
         return _quack_rmsnorm(x, weight=weight, eps=eps)
+    # Official LTX-2 runs the DiT natively in bf16 with no autocast, so its
+    # rms_norm outputs stay bf16 on every torch version. torch 2.12 added
+    # rms_norm to autocast's float32 cast policy, which under our
+    # autocast(bfloat16) forward returns float32 instead. Restore the input
+    # dtype to match the official numerics (and torch <= 2.11 behavior).
     return torch.nn.functional.rms_norm(
-        x, (x.shape[-1], ), weight=weight, eps=eps)
+        x, (x.shape[-1], ), weight=weight, eps=eps).to(x.dtype)
 
 
 class StageAwareRMSNorm(nn.RMSNorm):

fastvideo/models/encoders/qwen2_5_vl_custom.py

@@ -75,6 +75,25 @@ if not is_flash_attn_2_available():
 
 logger = logging.get_logger(__name__)
 
+
+def _resolve_torch_dtype(dtype, default: torch.dtype = torch.float32) -> torch.dtype:
+    if isinstance(dtype, torch.dtype):
+        return dtype
+    if isinstance(dtype, str):
+        normalized = dtype.strip().removeprefix("torch.").lower()
+        return {
+            "bfloat16": torch.bfloat16,
+            "bf16": torch.bfloat16,
+            "float16": torch.float16,
+            "fp16": torch.float16,
+            "half": torch.float16,
+            "float32": torch.float32,
+            "fp32": torch.float32,
+            "float": torch.float32,
+        }.get(normalized, default)
+    return default
+
+
 class Qwen2_5_VLMLP(nn.Module):
     def __init__(self, config, bias: bool = False):
         super().__init__()
@@ -304,10 +323,13 @@ class Qwen2_5_VisionTransformerPretrainedModel(nn.Module):
     config_class = Qwen2_5_VLVisionConfig
     _no_split_modules = ["Qwen2_5_VLVisionBlock"]
 
-    def __init__(self, config) -> None:
+    def __init__(self, config, parent_torch_dtype=None) -> None:
         super().__init__()
 
-        self.dtype = torch.bfloat16 if config.torch_dtype == "bfloat16" else torch.float32
+        config_torch_dtype = getattr(config, "torch_dtype", None)
+        self.dtype = _resolve_torch_dtype(
+            config_torch_dtype if config_torch_dtype is not None else parent_torch_dtype
+        )
 
         self.spatial_merge_size = config.spatial_merge_size
         self.patch_size = config.patch_size
@@ -1458,7 +1480,10 @@ class Qwen2_5_VLForConditionalGenerationSimple(nn.Module):
         super().__init__()
         config = _flatten_text_config(config)
         self.config = config
-        self.visual = Qwen2_5_VisionTransformerPretrainedModel(config.vision_config)
+        self.visual = Qwen2_5_VisionTransformerPretrainedModel(
+            config.vision_config,
+            parent_torch_dtype=getattr(config, "torch_dtype", None),
+        )
 
         self.model = Qwen2_5_VLModel(config)
         self.vocab_size = config.vocab_size

fastvideo/tests/attention/test_flash_attn_no_pad_resolver.py

@@ -50,9 +50,10 @@ def test_resolver_skips_cute_without_opt_in(monkeypatch) -> None:
     monkeypatch.setattr(builtins, "__import__", spying_import)
 
     mod = _reload_resolver_module()
-    resolved = mod._resolve_flash_attn_varlen_func()
+    resolved, version = mod._resolve_flash_attn_varlen_func()
     assert resolved is not None
     assert resolved.__name__ == "flash_attn_varlen_func"
+    assert version in ("2", "3")
     assert "fastvideo.attention.utils.flash_attn_cute" not in attempted
 
 
@@ -92,9 +93,10 @@ def test_resolver_returns_flash_attn_when_interface_unavailable(monkeypatch) ->
     monkeypatch.setattr(builtins, "__import__", patched_import)
 
     mod = _reload_resolver_module()
-    resolved = mod._resolve_flash_attn_varlen_func()
+    resolved, version = mod._resolve_flash_attn_varlen_func()
     assert resolved is not None
     assert resolved.__module__.startswith("flash_attn")
+    assert version == "2"
 
 
 def test_module_level_impl_is_resolver_output() -> None:

fastvideo/tests/attention/test_sdpa_head_sizes.py

@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: Apache-2.0
+"""CPU tests: SDPA must not restrict attention head sizes.
+
+``torch.nn.functional.scaled_dot_product_attention`` is head-size
+agnostic, but ``SDPABackend.get_supported_head_sizes()`` used to declare
+a list copied from FlashAttentionBackend that omitted e.g. 80 (the head
+size of CLIP-L vision encoders such as matrixgame2's). That made
+capability validation flag SDPA as incompatible with head-80 layers even
+though it handles them fine.
+
+CPU-only; no GPU or flash-attn dependency.
+"""
+
+import torch
+
+from fastvideo.attention.backends.sdpa import SDPABackend, SDPAImpl, SDPAMetadata
+from fastvideo.platforms.cuda import CudaPlatformBase
+from fastvideo.platforms.interface import AttentionBackendEnum
+
+
+def test_sdpa_head_sizes_unrestricted() -> None:
+    """SDPA declares no head-size restriction (None), so 80 is allowed."""
+    assert SDPABackend.get_supported_head_sizes() is None
+    # Once the capability-validation API (#1494) lands, head-80 must pass.
+    validate = getattr(SDPABackend, "validate_compatibility", None)
+    if validate is not None:
+        assert validate(head_size=80, dtype=torch.bfloat16) is None
+
+
+def test_explicit_torch_sdpa_pin_head_80_resolves() -> None:
+    """An explicit TORCH_SDPA pin on a head-80 layer resolves to SDPABackend."""
+    cls_str = CudaPlatformBase.get_attn_backend_cls(AttentionBackendEnum.TORCH_SDPA, head_size=80, dtype=torch.bfloat16)
+    assert cls_str == "fastvideo.attention.backends.sdpa.SDPABackend"
+
+
+def test_sdpa_impl_computes_head_80_attention() -> None:
+    """SDPAImpl produces correct output for head_size=80 (not in the old list)."""
+    bs, seq, heads, head_dim = 2, 8, 2, 80
+    softmax_scale = head_dim**-0.5
+    impl = SDPAImpl(num_heads=heads, head_size=head_dim, causal=False, softmax_scale=softmax_scale)
+    torch.manual_seed(0)
+    q, k, v = (torch.randn(bs, seq, heads, head_dim) for _ in range(3))
+
+    out = impl.forward(q, k, v, SDPAMetadata(current_timestep=0))
+
+    assert out.shape == (bs, seq, heads, head_dim)
+    qt, kt, vt = (t.transpose(1, 2) for t in (q, k, v))
+    ref = torch.softmax(qt @ kt.transpose(-2, -1) * softmax_scale, dim=-1) @ vt
+    torch.testing.assert_close(out, ref.transpose(1, 2), rtol=1e-4, atol=1e-4)

fastvideo/tests/encoders/test_qwen2_5_vl_config_compat.py

@@ -0,0 +1,125 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Config-compat regression tests for the custom Qwen2.5-VL encoder.
+
+transformers 5.x stopped forwarding text-model attributes (``vocab_size``,
+``pad_token_id``, ``hidden_size``, ...) from the composite ``Qwen2_5_VLConfig``
+down from ``text_config``, turned ``rope_scaling`` into a property backed by
+``rope_parameters``, dropped the ``"default"`` entry from ``ROPE_INIT_FUNCTIONS``
+and moved ``rope_theta`` inside ``rope_parameters``. See issue #1576.
+
+``qwen2_5_vl_custom`` absorbs all of that. These tests pin both the config
+normalization and the resulting RoPE numerics, so the compat layer cannot
+silently regress as transformers evolves. They are CPU-only and weight-free:
+the config is built inline rather than downloaded.
+"""
+import torch
+from transformers import Qwen2_5_VLConfig
+
+from fastvideo.models.encoders.qwen2_5_vl_custom import (
+    Qwen2_5_VLRotaryEmbedding,
+    _compute_default_rope_parameters,
+    _flatten_text_config,
+)
+
+# Shape of Qwen/Qwen2.5-VL-7B-Instruct, the checkpoint Cosmos 2.5's Reason1 text encoder is built on.
+HIDDEN_SIZE = 3584
+NUM_ATTENTION_HEADS = 28
+VOCAB_SIZE = 152064
+ROPE_THETA = 1000000.0
+MROPE_SECTION = [16, 24, 24]
+HEAD_DIM = HIDDEN_SIZE // NUM_ATTENTION_HEADS  # 128
+
+
+def _build_config() -> Qwen2_5_VLConfig:
+    # Flat, 4.x-style kwargs: this is what a real checkpoint's config.json carries.
+    return Qwen2_5_VLConfig(
+        hidden_size=HIDDEN_SIZE,
+        intermediate_size=18944,
+        num_attention_heads=NUM_ATTENTION_HEADS,
+        num_hidden_layers=28,
+        num_key_value_heads=4,
+        vocab_size=VOCAB_SIZE,
+        max_position_embeddings=128000,
+        rope_theta=ROPE_THETA,
+        rope_scaling={"type": "mrope", "mrope_section": MROPE_SECTION},
+        tie_word_embeddings=False,
+    )
+
+
+def _expected_inv_freq() -> torch.Tensor:
+    return 1.0 / (ROPE_THETA**(torch.arange(0, HEAD_DIM, 2, dtype=torch.int64).float() / HEAD_DIM))
+
+
+def test_flatten_text_config_exposes_text_attrs_on_composite_config() -> None:
+    """Attributes the module reads off the top-level config must resolve after flattening."""
+    config = _flatten_text_config(_build_config())
+
+    assert config.vocab_size == VOCAB_SIZE
+    assert config.hidden_size == HIDDEN_SIZE
+    assert config.num_attention_heads == NUM_ATTENTION_HEADS
+    assert config.max_position_embeddings == 128000
+    # Present but legitimately None for this checkpoint; the module reads it via getattr.
+    assert hasattr(config, "pad_token_id")
+
+
+def test_flatten_text_config_preserves_mrope_settings() -> None:
+    """``rope_scaling`` must survive the 4.x -> 5.x ``rope_parameters`` rename, mrope section intact."""
+    config = _flatten_text_config(_build_config())
+
+    rope_scaling = config.rope_scaling
+    assert rope_scaling is not None
+    assert rope_scaling["mrope_section"] == MROPE_SECTION
+
+    # The "rope_type" vs legacy "type" lookup performed by Qwen2_5_VLRotaryEmbedding.
+    # Both transformers lines normalize an "mrope" request to "default" before we see it:
+    # 4.x rewrites rope_scaling["type"] in place, 5.x sets rope_parameters["rope_type"].
+    # Asserting "mrope" here would fail on every supported version.
+    rope_type = rope_scaling.get("rope_type", rope_scaling.get("type"))
+    assert rope_type == "default"
+
+
+def test_compute_default_rope_parameters_reads_theta_on_any_transformers() -> None:
+    """``rope_theta`` moved inside ``rope_parameters`` in 5.x; the helper must find it either way."""
+    config = _flatten_text_config(_build_config())
+
+    inv_freq, attention_scaling = _compute_default_rope_parameters(config)
+
+    assert attention_scaling == 1.0
+    assert inv_freq.shape == (HEAD_DIM // 2, )
+    torch.testing.assert_close(inv_freq, _expected_inv_freq())
+
+
+def test_rotary_embedding_handles_config_without_rope_scaling() -> None:
+    """``reason1.py`` falls back to a config carrying no ``rope_scaling``; it must still build.
+
+    This exercises a different branch on each line: on 4.x ``rope_scaling`` is ``None`` and
+    ``Qwen2_5_VLRotaryEmbedding`` takes its ``else``; on 5.x ``rope_parameters`` is populated
+    with a bare ``rope_theta``/``rope_type`` pair. Both must land on the default init fn.
+    """
+    config = _flatten_text_config(
+        Qwen2_5_VLConfig(
+            hidden_size=HIDDEN_SIZE,
+            intermediate_size=18944,
+            max_window_layers=28,
+            num_attention_heads=NUM_ATTENTION_HEADS,
+            num_hidden_layers=28,
+            num_key_value_heads=4,
+            tie_word_embeddings=False,
+            vocab_size=VOCAB_SIZE,
+        ))
+
+    rotary_embedding = Qwen2_5_VLRotaryEmbedding(config)
+
+    assert rotary_embedding.rope_type == "default"
+    assert rotary_embedding.inv_freq.shape == (HEAD_DIM // 2, )
+
+
+def test_rotary_embedding_builds_from_flattened_config() -> None:
+    """End-to-end: the rotary embedding must pick a valid init fn and produce 4.x-identical numerics."""
+    config = _flatten_text_config(_build_config())
+
+    rotary_embedding = Qwen2_5_VLRotaryEmbedding(config)
+
+    assert rotary_embedding.rope_type == "default"
+    assert rotary_embedding.attention_scaling == 1.0
+    torch.testing.assert_close(rotary_embedding.inv_freq, _expected_inv_freq())

fastvideo/tests/encoders/test_qwen2_5_vl_vision_dtype.py

@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from types import SimpleNamespace
+
+import torch
+
+from fastvideo.models.encoders.qwen2_5_vl_custom import (
+    Qwen2_5_VisionTransformerPretrainedModel,
+    Qwen2_5_VLForConditionalGenerationSimple,
+)
+
+
+def _vision_config(torch_dtype=None):
+    return SimpleNamespace(
+        torch_dtype=torch_dtype,
+        spatial_merge_size=2,
+        patch_size=14,
+        fullatt_block_indexes=[],
+        window_size=112,
+        temporal_patch_size=2,
+        in_channels=3,
+        hidden_size=8,
+        num_heads=2,
+        depth=0,
+        _attn_implementation="sdpa",
+        out_hidden_size=8,
+    )
+
+
+def _full_config(torch_dtype):
+    return SimpleNamespace(
+        vision_config=_vision_config(),
+        torch_dtype=torch_dtype,
+        vocab_size=16,
+        hidden_size=8,
+        num_hidden_layers=0,
+        num_attention_heads=2,
+        pad_token_id=0,
+        _attn_implementation="sdpa",
+        rms_norm_eps=1e-6,
+        max_position_embeddings=32,
+        rope_theta=1_000_000.0,
+        rope_scaling=None,
+    )
+
+
+def test_vision_dtype_uses_parent_bfloat16_string_when_vision_dtype_missing():
+    model = Qwen2_5_VisionTransformerPretrainedModel(
+        _vision_config(),
+        parent_torch_dtype="bfloat16",
+    )
+
+    assert model.dtype == torch.bfloat16
+
+
+def test_vision_dtype_accepts_parent_torch_dtype_object():
+    model = Qwen2_5_VisionTransformerPretrainedModel(
+        _vision_config(),
+        parent_torch_dtype=torch.bfloat16,
+    )
+
+    assert model.dtype == torch.bfloat16
+
+
+def test_vision_dtype_strips_parent_dtype_string_whitespace():
+    model = Qwen2_5_VisionTransformerPretrainedModel(
+        _vision_config(),
+        parent_torch_dtype=" torch.bfloat16 ",
+    )
+
+    assert model.dtype == torch.bfloat16
+
+
+def test_vision_dtype_prefers_explicit_vision_dtype_over_parent_dtype():
+    model = Qwen2_5_VisionTransformerPretrainedModel(
+        _vision_config(torch_dtype="float16"),
+        parent_torch_dtype="bfloat16",
+    )
+
+    assert model.dtype == torch.float16
+
+
+def test_vision_dtype_falls_back_to_float32_for_missing_or_unknown_dtype():
+    missing = Qwen2_5_VisionTransformerPretrainedModel(_vision_config())
+    unknown = Qwen2_5_VisionTransformerPretrainedModel(
+        _vision_config(),
+        parent_torch_dtype="not-a-real-dtype",
+    )
+
+    assert missing.dtype == torch.float32
+    assert unknown.dtype == torch.float32
+
+
+def test_conditional_generation_passes_parent_dtype_to_visual_tower():
+    model = Qwen2_5_VLForConditionalGenerationSimple(_full_config("torch.bfloat16"))
+
+    assert model.visual.dtype == torch.bfloat16