[bugfix]: scope dynamo's disable flag with config.patch in the LoRA tests

pytest 9's monkeypatch reads the previous value from the instance __dict__.
torch._dynamo.config serves its flags through __getattr__, so the first
test's teardown deleted the disable entry and every later setattr on it
failed: 28 setup errors in the Unit lane's test_lora_side_branch.py. The
same cause, not torch 2.12, is behind the TF32 teardown errors.

- Use torch._dynamo.config.patch(disable=...) through fixtures in the LoRA
  side-branch and merge-precision tests.
- Correct the tf32_disabled fixture docstring.
- Note that the Qwen-Image-2.1 block references follow the golden lane's
  FA2 runtime.
- Record the pitfall in .agents/lessons.
This commit is contained in:
Aryan Kumar
2026-11-02 16:17:52 -08:00
parent ecb502118b
commit 27180f14e2
5 changed files with 69 additions and 18 deletions
@@ -0,0 +1,37 @@
---
date: 2026-11-02
experiment: FastVideo CI repair on main (Transformer and Unit lanes)
category: infrastructure
severity: important
---
# pytest 9 monkeypatch cannot restore torch flag proxies
## What Happened
After the CI image picked up pytest 9, the Transformer lane reported 22
teardown errors (`'cuBLASModule' object has no attribute 'allow_tf32'`) and
the Unit lane reported 28 setup errors in `test_lora_side_branch.py`
(`torch._dynamo.config ... has no attribute 'disable'`). The test bodies
still passed.
## Root Cause
pytest 9's `monkeypatch.setattr` reads the previous value from the target's
instance `__dict__`. `torch.backends.cuda.matmul` and `torch._dynamo.config`
serve their flags through `__getattr__`, so pytest records the value as
missing. Teardown then calls `delattr`: on cuBLAS that raises and leaves TF32
changed, and on the dynamo config it deletes the `disable` entry, so every
later `setattr` on it fails. pytest 8.4 restores both correctly.
## Fix / Workaround
Use the owner's own scoping instead of monkeypatch:
`torch._dynamo.config.patch(name=value)` for dynamo/inductor config, and the
`tf32_disabled` fixture in `fastvideo/tests/conftest.py` for the TF32 flags.
## Prevention
Do not `monkeypatch.setattr` attributes of `torch.backends.cuda.matmul`,
`torch._dynamo.config`, `torch._inductor.config` or other torch config
proxies. Module functions such as `torch.compiler.is_compiling` are safe.
+3 -2
View File
@@ -51,8 +51,9 @@ def env_overrides():
def tf32_disabled():
"""Disable TF32 for CUDA matmuls and cuDNN during the test, then restore the previous flags.
On torch 2.12 ``torch.backends.cuda.matmul.allow_tf32`` is computed by ``cuBLASModule`` rather
than stored on it, so ``monkeypatch.setattr`` cannot undo it. Restore through the same legacy
``torch.backends.cuda.matmul`` serves ``allow_tf32`` through ``__getattr__``. pytest 9's
``monkeypatch.setattr`` reads the old value from the instance ``__dict__``, finds nothing, and
its teardown ``delattr`` raises instead of restoring the flag. Restore through the same legacy
flags: mixing them with the per-backend ``fp32_precision`` API makes later ``allow_tf32`` reads raise.
"""
matmul, cudnn = torch.backends.cuda.matmul, torch.backends.cudnn
@@ -6,10 +6,12 @@ the target image. Causal text segments use masked SDPA; reference and target
segments use FlashAttention, including their unequal query/key lengths. Complex RoPE and
the two modulation rows match the causal-conditioning block interface.
Requires a BF16-capable CUDA GPU and FlashAttention. Select FA4 with
FASTVIDEO_FA4=1 before starting Python. The shared harness downloads
only checkpoint shards containing block 0 and compares a device/runtime-matched
golden exactly; it never loads the text encoder or VAE.
Requires a BF16-capable CUDA GPU and FlashAttention. Like the other block
goldens, the published references match the golden-gate CI lane: FA2 with
FASTVIDEO_FA4 off. Running with FASTVIDEO_FA4=1 reports an environment
mismatch rather than drift. The shared harness downloads only checkpoint
shards containing block 0 and compares a device/runtime-matched golden
exactly; it never loads the text encoder or VAE.
"""
from __future__ import annotations
@@ -15,8 +15,10 @@ IN, OUT, RANK = 32, 24, 16
@pytest.fixture(autouse=True)
def _cpu_merge(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(torch._dynamo.config, "disable", True)
# config.patch restores the flag; a monkeypatch.setattr teardown would delete it (pytest 9).
monkeypatch.setattr(lora_linear, "get_local_torch_device", lambda: torch.device("cpu"))
with torch._dynamo.config.patch(disable=True):
yield
def _merged(adapter_dtype: torch.dtype) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
+20 -11
View File
@@ -23,9 +23,21 @@ IN, OUT, RANK = 16, 12, 4
@pytest.fixture(autouse=True)
def _eager_forward(monkeypatch: pytest.MonkeyPatch):
"""``BaseLayerWithLoRA.forward`` is ``torch.compile``d; run it eagerly, and merge on CPU even on MPS hosts."""
monkeypatch.setattr(torch._dynamo.config, "disable", True)
"""``BaseLayerWithLoRA.forward`` is ``torch.compile``d; run it eagerly, and merge on CPU even on MPS hosts.
Dynamo flags go through ``config.patch``: its module serves them via ``__getattr__``, so a
``monkeypatch.setattr`` teardown deletes the flag instead of restoring it (pytest 9).
"""
monkeypatch.setattr(lora_linear, "get_local_torch_device", lambda: torch.device("cpu"))
with torch._dynamo.config.patch(disable=True):
yield
@pytest.fixture
def dynamo_enabled():
"""Re-enable dynamo inside the autouse eager scope for tests that compile on purpose."""
with torch._dynamo.config.patch(disable=False):
yield
class _PackedMethod:
@@ -84,14 +96,12 @@ def test_wrapped_layer_at_strength_zero_is_bitwise_the_base_layer(packed: bool)
@pytest.mark.parametrize("backend", ["eager", "inductor"])
@pytest.mark.parametrize("adapter", [False, True])
def test_wrapping_does_not_change_a_compiled_block(monkeypatch: pytest.MonkeyPatch, backend: str,
adapter: bool) -> None:
def test_wrapping_does_not_change_a_compiled_block(dynamo_enabled, backend: str, adapter: bool) -> None:
"""Under a caller's compile the base layer is traced with the block, as it is unwrapped; the wrapper adds no
region of its own, so the compiled block is unchanged with no adapter or at strength 0."""
if backend == "inductor" and not torch.cuda.is_available():
pytest.skip("Inductor execution requires a CUDA GPU for this regression")
device = torch.device("cuda" if backend == "inductor" else "cpu")
monkeypatch.setattr(torch._dynamo.config, "disable", False)
torch._dynamo.reset()
class Block(torch.nn.Module):
@@ -117,9 +127,9 @@ def test_wrapping_does_not_change_a_compiled_block(monkeypatch: pytest.MonkeyPat
torch._dynamo.reset()
def test_wrapper_adds_no_compiled_region_around_the_base_layer(monkeypatch: pytest.MonkeyPatch) -> None:
def test_wrapper_adds_no_compiled_region_around_the_base_layer(monkeypatch: pytest.MonkeyPatch,
dynamo_enabled) -> None:
"""Compiling a packed base layer on its own changes its numerics, so an uncompiled caller runs it eagerly."""
monkeypatch.setattr(torch._dynamo.config, "disable", False)
torch._dynamo.reset()
base = _packed_linear()
layer = get_lora_layer(base)
@@ -179,11 +189,11 @@ def test_side_branch_chunked_update_matches_the_full_product(monkeypatch: pytest
@pytest.mark.parametrize("backend", ["eager", "inductor"])
def test_side_branch_chunked_update_compiles_fullgraph(monkeypatch: pytest.MonkeyPatch, backend: str) -> None:
def test_side_branch_chunked_update_compiles_fullgraph(monkeypatch: pytest.MonkeyPatch, dynamo_enabled,
backend: str) -> None:
if backend == "inductor" and not torch.cuda.is_available():
pytest.skip("Inductor execution requires a CUDA GPU for this regression")
device = torch.device("cuda" if backend == "inductor" else "cpu")
monkeypatch.setattr(torch._dynamo.config, "disable", False)
monkeypatch.setattr(lora_linear, "_SIDE_BRANCH_CHUNK_BYTES", 64)
torch._dynamo.reset()
base = _packed_linear().to(device)
@@ -221,9 +231,8 @@ def test_side_branch_chunked_update_compiles_fullgraph(monkeypatch: pytest.Monke
torch._dynamo.reset()
def test_side_branch_strength_changes_do_not_recompile(monkeypatch: pytest.MonkeyPatch) -> None:
def test_side_branch_strength_changes_do_not_recompile(dynamo_enabled) -> None:
"""A float strength is guarded by value, so fullgraph would fail after ``recompile_limit`` distinct strengths."""
monkeypatch.setattr(torch._dynamo.config, "disable", False)
torch._dynamo.reset()
base = _packed_linear()
layer = get_lora_layer(base)