diff --git a/.agents/lessons/2026-11-02_pytest9-monkeypatch-torch-flag-proxies.md b/.agents/lessons/2026-11-02_pytest9-monkeypatch-torch-flag-proxies.md new file mode 100644 index 000000000..a12ea19fe --- /dev/null +++ b/.agents/lessons/2026-11-02_pytest9-monkeypatch-torch-flag-proxies.md @@ -0,0 +1,37 @@ +--- +date: 2026-11-02 +experiment: FastVideo CI repair on main (Transformer and Unit lanes) +category: infrastructure +severity: important +--- + +# pytest 9 monkeypatch cannot restore torch flag proxies + +## What Happened + +After the CI image picked up pytest 9, the Transformer lane reported 22 +teardown errors (`'cuBLASModule' object has no attribute 'allow_tf32'`) and +the Unit lane reported 28 setup errors in `test_lora_side_branch.py` +(`torch._dynamo.config ... has no attribute 'disable'`). The test bodies +still passed. + +## Root Cause + +pytest 9's `monkeypatch.setattr` reads the previous value from the target's +instance `__dict__`. `torch.backends.cuda.matmul` and `torch._dynamo.config` +serve their flags through `__getattr__`, so pytest records the value as +missing. Teardown then calls `delattr`: on cuBLAS that raises and leaves TF32 +changed, and on the dynamo config it deletes the `disable` entry, so every +later `setattr` on it fails. pytest 8.4 restores both correctly. + +## Fix / Workaround + +Use the owner's own scoping instead of monkeypatch: +`torch._dynamo.config.patch(name=value)` for dynamo/inductor config, and the +`tf32_disabled` fixture in `fastvideo/tests/conftest.py` for the TF32 flags. + +## Prevention + +Do not `monkeypatch.setattr` attributes of `torch.backends.cuda.matmul`, +`torch._dynamo.config`, `torch._inductor.config` or other torch config +proxies. Module functions such as `torch.compiler.is_compiling` are safe. diff --git a/fastvideo/tests/conftest.py b/fastvideo/tests/conftest.py index a92c865e5..42fc6ecce 100644 --- a/fastvideo/tests/conftest.py +++ b/fastvideo/tests/conftest.py @@ -51,8 +51,9 @@ def env_overrides(): def tf32_disabled(): """Disable TF32 for CUDA matmuls and cuDNN during the test, then restore the previous flags. - On torch 2.12 ``torch.backends.cuda.matmul.allow_tf32`` is computed by ``cuBLASModule`` rather - than stored on it, so ``monkeypatch.setattr`` cannot undo it. Restore through the same legacy + ``torch.backends.cuda.matmul`` serves ``allow_tf32`` through ``__getattr__``. pytest 9's + ``monkeypatch.setattr`` reads the old value from the instance ``__dict__``, finds nothing, and + its teardown ``delattr`` raises instead of restoring the flag. Restore through the same legacy flags: mixing them with the per-backend ``fp32_precision`` API makes later ``allow_tf32`` reads raise. """ matmul, cudnn = torch.backends.cuda.matmul, torch.backends.cudnn diff --git a/fastvideo/tests/golden_gate/test_qwen_image21.py b/fastvideo/tests/golden_gate/test_qwen_image21.py index f32ec4a42..9ec559843 100644 --- a/fastvideo/tests/golden_gate/test_qwen_image21.py +++ b/fastvideo/tests/golden_gate/test_qwen_image21.py @@ -6,10 +6,12 @@ the target image. Causal text segments use masked SDPA; reference and target segments use FlashAttention, including their unequal query/key lengths. Complex RoPE and the two modulation rows match the causal-conditioning block interface. -Requires a BF16-capable CUDA GPU and FlashAttention. Select FA4 with -FASTVIDEO_FA4=1 before starting Python. The shared harness downloads -only checkpoint shards containing block 0 and compares a device/runtime-matched -golden exactly; it never loads the text encoder or VAE. +Requires a BF16-capable CUDA GPU and FlashAttention. Like the other block +goldens, the published references match the golden-gate CI lane: FA2 with +FASTVIDEO_FA4 off. Running with FASTVIDEO_FA4=1 reports an environment +mismatch rather than drift. The shared harness downloads only checkpoint +shards containing block 0 and compares a device/runtime-matched golden +exactly; it never loads the text encoder or VAE. """ from __future__ import annotations diff --git a/fastvideo/tests/layers/test_lora_merge_precision.py b/fastvideo/tests/layers/test_lora_merge_precision.py index 6bbf39ca0..51d813ed8 100644 --- a/fastvideo/tests/layers/test_lora_merge_precision.py +++ b/fastvideo/tests/layers/test_lora_merge_precision.py @@ -15,8 +15,10 @@ IN, OUT, RANK = 32, 24, 16 @pytest.fixture(autouse=True) def _cpu_merge(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr(torch._dynamo.config, "disable", True) + # config.patch restores the flag; a monkeypatch.setattr teardown would delete it (pytest 9). monkeypatch.setattr(lora_linear, "get_local_torch_device", lambda: torch.device("cpu")) + with torch._dynamo.config.patch(disable=True): + yield def _merged(adapter_dtype: torch.dtype) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: diff --git a/fastvideo/tests/layers/test_lora_side_branch.py b/fastvideo/tests/layers/test_lora_side_branch.py index 04a804df4..3222fe19e 100644 --- a/fastvideo/tests/layers/test_lora_side_branch.py +++ b/fastvideo/tests/layers/test_lora_side_branch.py @@ -23,9 +23,21 @@ IN, OUT, RANK = 16, 12, 4 @pytest.fixture(autouse=True) def _eager_forward(monkeypatch: pytest.MonkeyPatch): - """``BaseLayerWithLoRA.forward`` is ``torch.compile``d; run it eagerly, and merge on CPU even on MPS hosts.""" - monkeypatch.setattr(torch._dynamo.config, "disable", True) + """``BaseLayerWithLoRA.forward`` is ``torch.compile``d; run it eagerly, and merge on CPU even on MPS hosts. + + Dynamo flags go through ``config.patch``: its module serves them via ``__getattr__``, so a + ``monkeypatch.setattr`` teardown deletes the flag instead of restoring it (pytest 9). + """ monkeypatch.setattr(lora_linear, "get_local_torch_device", lambda: torch.device("cpu")) + with torch._dynamo.config.patch(disable=True): + yield + + +@pytest.fixture +def dynamo_enabled(): + """Re-enable dynamo inside the autouse eager scope for tests that compile on purpose.""" + with torch._dynamo.config.patch(disable=False): + yield class _PackedMethod: @@ -84,14 +96,12 @@ def test_wrapped_layer_at_strength_zero_is_bitwise_the_base_layer(packed: bool) @pytest.mark.parametrize("backend", ["eager", "inductor"]) @pytest.mark.parametrize("adapter", [False, True]) -def test_wrapping_does_not_change_a_compiled_block(monkeypatch: pytest.MonkeyPatch, backend: str, - adapter: bool) -> None: +def test_wrapping_does_not_change_a_compiled_block(dynamo_enabled, backend: str, adapter: bool) -> None: """Under a caller's compile the base layer is traced with the block, as it is unwrapped; the wrapper adds no region of its own, so the compiled block is unchanged with no adapter or at strength 0.""" if backend == "inductor" and not torch.cuda.is_available(): pytest.skip("Inductor execution requires a CUDA GPU for this regression") device = torch.device("cuda" if backend == "inductor" else "cpu") - monkeypatch.setattr(torch._dynamo.config, "disable", False) torch._dynamo.reset() class Block(torch.nn.Module): @@ -117,9 +127,9 @@ def test_wrapping_does_not_change_a_compiled_block(monkeypatch: pytest.MonkeyPat torch._dynamo.reset() -def test_wrapper_adds_no_compiled_region_around_the_base_layer(monkeypatch: pytest.MonkeyPatch) -> None: +def test_wrapper_adds_no_compiled_region_around_the_base_layer(monkeypatch: pytest.MonkeyPatch, + dynamo_enabled) -> None: """Compiling a packed base layer on its own changes its numerics, so an uncompiled caller runs it eagerly.""" - monkeypatch.setattr(torch._dynamo.config, "disable", False) torch._dynamo.reset() base = _packed_linear() layer = get_lora_layer(base) @@ -179,11 +189,11 @@ def test_side_branch_chunked_update_matches_the_full_product(monkeypatch: pytest @pytest.mark.parametrize("backend", ["eager", "inductor"]) -def test_side_branch_chunked_update_compiles_fullgraph(monkeypatch: pytest.MonkeyPatch, backend: str) -> None: +def test_side_branch_chunked_update_compiles_fullgraph(monkeypatch: pytest.MonkeyPatch, dynamo_enabled, + backend: str) -> None: if backend == "inductor" and not torch.cuda.is_available(): pytest.skip("Inductor execution requires a CUDA GPU for this regression") device = torch.device("cuda" if backend == "inductor" else "cpu") - monkeypatch.setattr(torch._dynamo.config, "disable", False) monkeypatch.setattr(lora_linear, "_SIDE_BRANCH_CHUNK_BYTES", 64) torch._dynamo.reset() base = _packed_linear().to(device) @@ -221,9 +231,8 @@ def test_side_branch_chunked_update_compiles_fullgraph(monkeypatch: pytest.Monke torch._dynamo.reset() -def test_side_branch_strength_changes_do_not_recompile(monkeypatch: pytest.MonkeyPatch) -> None: +def test_side_branch_strength_changes_do_not_recompile(dynamo_enabled) -> None: """A float strength is guarded by value, so fullgraph would fail after ``recompile_limit`` distinct strengths.""" - monkeypatch.setattr(torch._dynamo.config, "disable", False) torch._dynamo.reset() base = _packed_linear() layer = get_lora_layer(base)