olivv-cs-ComfyUI-FunPack/modules/sampling/ptd/tests/test_ptd.py

"""PTD on a tiny real MiniMax-H3: all three file layouts, the hop switching, the sampler swap and clean removal.

Structure follows ComfyUI-PTD-Compact's tests (Nynxz, Apache-2.0). They prove the wiring and the maths of
our side on random weights, nothing about picture quality on the real model.
"""

import pytest
import torch
import torch.nn.functional as F

from . import _tiny

HOP_TIMES = (0.999, 0.947, 0.857, 0.667)


@pytest.fixture(autouse=True)
def _needs_comfy(comfyui):
    """Imports comfy."""


@pytest.fixture
def ptd():
    import importlib
    return importlib.import_module("modules.sampling.ptd")


@pytest.fixture
def folder(tmp_path):
    import folder_paths
    folder_paths.add_model_folder_path("ptd", str(tmp_path), is_default=True)
    return tmp_path


def _guarded(patcher):
    from modules._core import patching
    return patching.GuardedPatcher(patcher, "funui.ptd", patching.Dropped())


def _install(ptd, patcher, name, kernels="exact"):
    note = ptd.install(_guarded(patcher), {"enabled": True, "adapter": name, "kernels": kernels}, "funui.ptd")
    return patcher.model_options["funui_ptd_controller"], note


def _clone(base):
    return base.clone()


def _sample(model, sampler=None, sigmas=None):
    import comfy.nested_tensor
    import comfy.samplers
    guider = comfy.samplers.CFGGuider(model)
    context = torch.randn(1, 6, _tiny.DIMS["text_dim"], generator=torch.Generator().manual_seed(3))
    guider.set_conds([[context, {}]], [[context, {}]])
    guider.set_cfg(1.0)
    video, audio = torch.zeros(1, 24, 2, 4, 4), torch.zeros(1, 32, 2, 3)
    noise = comfy.nested_tensor.NestedTensor([torch.randn(video.shape, generator=torch.Generator().manual_seed(4)),
                                              torch.randn(audio.shape, generator=torch.Generator().manual_seed(5))])
    sampler = sampler or comfy.samplers.sampler_object("euler")
    sigmas = sigmas if sigmas is not None else torch.linspace(1.0, 0.0, 21)
    try:
        out = guider.sample(noise, comfy.nested_tensor.NestedTensor([video, audio]), sampler, sigmas,
                            disable_pbar=True, seed=7)
    finally:
        import comfy.model_management
        comfy.model_management.unload_all_models()
    return [t.clone() for t in out.unbind()]


@pytest.mark.parametrize("int8", [False, True], ids=["float", "int8"])
@pytest.mark.parametrize("curve", [False, True], ids=["full", "pruned"])
def test_each_hop_adds_its_own_expert_to_the_layer(ptd, folder, int8, curve):
    base = _tiny.native_model(curve)
    native = base.get_model_object("diffusion_model")
    weights = {n: p.detach().clone() for n, p in native.named_parameters()}
    dense = _tiny.epfl_file(folder / "a.safetensors", native, int8=int8)
    controller, _ = _install(ptd, _clone(base), "a.safetensors")
    for hop, sigma in enumerate(HOP_TIMES):
        for name in ("blocks.0.attn.qkv_proj", "blocks.1.mlp.fc1", "blocks.1.mlp.fc2", "blocks.0.attn.out_proj",
                     "token_refiner.blocks.0.attn.qkv_proj"):
            layer = native.get_submodule(name)
            inp = torch.randn(5, layer.in_features)
            parts = [(p, d) for (n, p, h), d in dense.items() if n == name and h == hop]
            if name.endswith("qkv_proj"):
                order = {"attn.to_q": 0, "attn.to_k": 1, "attn.to_v": 2}
                delta = torch.cat([d for _, d in sorted(parts, key=lambda pd: order[pd[0]])], 0)
            else:
                (part, delta), = parts
                if part == "ff.net.0.proj":
                    up, gate = delta.chunk(2, 0)
                    delta = torch.cat([gate, up], 0)
            want = F.linear(inp, weights[name + ".weight"] + delta, weights.get(name + ".bias"))
            with controller.activate(hop, sigma), controller.forward_hooks(controller.current()):
                got = layer(inp)
            torch.testing.assert_close(got, want, rtol=1e-4, atol=1e-5)
            torch.testing.assert_close(layer(inp), F.linear(inp, weights[name + ".weight"], weights.get(name + ".bias")),
                                       rtol=1e-4, atol=1e-5)          # nothing stays hooked after the hop


def test_a_hop_must_run_at_its_trained_time(ptd, folder):
    base = _tiny.native_model()
    _tiny.epfl_file(folder / "a.safetensors", base.get_model_object("diffusion_model"))
    controller, _ = _install(ptd, _clone(base), "a.safetensors")
    with pytest.raises(ValueError, match="trained time"):
        with controller.activate(1, 0.5):
            pass
    with pytest.raises(RuntimeError, match="no PTD hop is running"):
        controller.current()


def test_the_chosen_sampler_is_replaced_by_the_adapters_own_hops(ptd, folder):
    base = _tiny.native_model()
    native = base.get_model_object("diffusion_model")
    _tiny.epfl_file(folder / "a.safetensors", native)
    _tiny.epfl_file(folder / "zero.safetensors", native, zero_a=True)
    seen = []
    for name in ("a.safetensors", "zero.safetensors"):
        model = _clone(base)
        controller, note = _install(ptd, model, name)
        assert "4 hops" in note
        calls = []
        original = controller.activate

        def spy(hop, sigma, original=original, calls=calls):
            calls.append(hop)
            return original(hop, sigma)

        controller.activate = spy
        # twenty euler steps were asked for; PTD runs its four hops
        out = _sample(model)
        assert calls == [0, 1, 2, 3]
        assert all(torch.isfinite(t).all() for t in out)
        seen.append(out)
    assert not torch.allclose(seen[0][0], seen[1][0])
    # the base model is not left patched
    plain = _sample(_clone(base))
    assert not torch.allclose(plain[0], seen[0][0])


@pytest.mark.parametrize("curve", [False, True], ids=["full", "pruned"])
def test_exact_and_fast_blocks_agree(ptd, folder, curve):
    base = _tiny.native_model(curve)
    _tiny.pdd_file(folder / "r.safetensors", base.get_model_object("diffusion_model"))
    exact = _sample(_installed(ptd, base, "exact"))
    fast = _sample(_installed(ptd, base, "fast"))
    for e, f in zip(exact, fast):
        assert torch.isfinite(e).all()
        assert torch.allclose(e, f, rtol=1e-4, atol=1e-4)


def _installed(ptd, base, kernels):
    model = _clone(base)
    _install(ptd, model, "r.safetensors", kernels)
    return model


def test_int8_ref2va_stays_close_to_float(ptd, folder):
    base = _tiny.native_model()
    native = base.get_model_object("diffusion_model")
    _tiny.pdd_file(folder / "r.safetensors", native)
    _tiny.pdd_file(folder / "q.safetensors", native, int8=True)
    ptd._kept.clear()
    full = _sample(_installed_file(ptd, base, "r.safetensors"))
    small = _sample(_installed_file(ptd, base, "q.safetensors"))
    for f, q in zip(full, small):
        assert torch.allclose(f, q, rtol=1e-2, atol=1e-2)


def _installed_file(ptd, base, name):
    model = _clone(base)
    _install(ptd, model, name)
    return model


def test_ref2va_heads_replace_the_native_heads_for_a_hop_only(ptd, folder):
    base = _tiny.native_model()
    native = base.get_model_object("diffusion_model")
    _tiny.pdd_file(folder / "r.safetensors", native)
    controller, _ = _install(ptd, _clone(base), "r.safetensors")
    head = native.final_layer.video_out
    x = torch.randn(5, head.in_features)
    plain = head(x)
    hop = 1
    with controller.activate(hop, float(controller.grid[hop])), controller.forward_hooks(controller.current()):
        hopped = head(x)
    weight, bias = controller.adapter.heads[hop]["final_layer.video_out"]
    torch.testing.assert_close(hopped, F.linear(x, weight, bias), atol=1e-5, rtol=1e-5)
    assert not torch.allclose(hopped, plain, atol=1e-3)
    assert torch.equal(head(x), plain)


def test_grids_are_the_published_ones(ptd, folder):
    from modules.sampling.ptd import schedule
    assert schedule.RECIPES["h3_4step"][:4] == HOP_TIMES
    for hops in (4, 8):
        grid = schedule.pdd_grid(hops)
        assert len(grid) == hops + 1 and abs(grid[0] - schedule.T_MAX) < 1e-12 and grid[-1] == 0.0
        assert all(a > b for a, b in zip(grid, grid[1:]))
        for video, audio in schedule.pdd_plans(hops):
            for plan in (video, audio):
                assert abs(float(plan.sum()) - 1) < 1e-6 and int((plan > 0).sum()) == 32 // hops


def test_a_file_for_another_model_is_refused_in_plain_words(ptd, folder):
    big = _tiny.native_model()
    _tiny.epfl_file(folder / "a.safetensors", big.get_model_object("diffusion_model"))
    smaller = _tiny.native_model(num_layers=1)
    with pytest.raises(ValueError, match="does not match the loaded model's layers"):
        _install(ptd, _clone(smaller), "a.safetensors")


def test_not_a_ptd_file_is_refused(ptd, folder):
    from safetensors.torch import save_file
    save_file({"x": torch.zeros(2)}, str(folder / "n.safetensors"), metadata={"format": "pt"})
    with pytest.raises(ValueError, match="Not a PTD adapter file"):
        _install(ptd, _clone(_tiny.native_model()), "n.safetensors")


def test_no_file_chosen_is_said_not_quarantined(ptd, folder):
    from modules._core import control
    with pytest.raises(control.Unavailable):
        ptd.install(_guarded(_clone(_tiny.native_model())), {"enabled": True, "adapter": "", "kernels": "exact"},
                    "funui.ptd")


def test_off_installs_nothing(ptd, folder):
    patcher = _clone(_tiny.native_model())
    assert ptd.install(_guarded(patcher), {"enabled": False, "adapter": "", "kernels": "exact"}, "funui.ptd") is None
    assert "funui_ptd_controller" not in patcher.model_options


def test_load_modifiers_strips_ptd_so_a_second_pass_reinstalls_it_once(ptd, folder):
    from modules._core import patching
    base = _tiny.native_model()
    _tiny.epfl_file(folder / "a.safetensors", base.get_model_object("diffusion_model"))
    model = _clone(base)
    _install(ptd, model, "a.safetensors")
    again = model.clone()
    assert patching.strip(again, "funui.") >= 3
    assert "funui_ptd_controller" not in again.model_options
    assert again.model_options.get("model_function_wrapper") is None
    _install(ptd, again, "a.safetensors")                      # no "already carries" error
    assert "funui_ptd_controller" in again.model_options
    assert "funui_ptd_controller" not in base.model_options and not base.object_patches


def test_a_latent_that_does_not_start_empty_is_refused(ptd, folder):
    from modules.sampling.ptd.sampler import _refuse_unsupported

    class Model:
        latent_image = torch.ones(1, 2)

    with pytest.raises(ValueError, match="empty latent"):
        _refuse_unsupported(Model(), {})
    with pytest.raises(ValueError, match="masked"):
        _refuse_unsupported(object(), {"denoise_mask": torch.ones(1)})


def test_it_installs_through_load_modifiers_on_an_h3_model(ptd, folder, monkeypatch):
    from core import registry
    from modules.sampling.modifiers.nodes import FunUILoadModifiers
    base = _tiny.native_model()
    _tiny.pdd_file(folder / "r.safetensors", base.get_model_object("diffusion_model"))
    spec = registry.current().specs["ptd"]
    monkeypatch.setitem(spec.settings["adapter"], "options", [{"value": "r.safetensors", "label": "r"}])
    monkeypatch.setitem(spec.settings["adapter"], "default", "r.safetensors")
    patched, status = FunUILoadModifiers.execute(_clone(base), {"ptd": {"enabled": True}}).result
    assert "ptd: r.safetensors, 4 hops" in status
    assert "failed" not in status
    out = _sample(patched)
    assert all(torch.isfinite(t).all() for t in out)
    # switched off, the same model samples as the plain one does
    plain, status = FunUILoadModifiers.execute(_clone(base), {"ptd": {"enabled": False}}).result
    assert "ptd" not in status.splitlines()[0]


def test_an_ordinary_lora_stacks_with_ptd_in_either_order(ptd, folder):
    import comfy.sd
    base = _tiny.native_model()
    native = base.get_model_object("diffusion_model")
    dense = _tiny.epfl_file(folder / "a.safetensors", native)
    g = torch.Generator().manual_seed(2)
    layer = native.get_submodule("blocks.1.mlp.fc2")
    up, down = torch.randn(layer.out_features, 4, generator=g) * 0.1, torch.randn(4, layer.in_features, generator=g) * 0.1
    lora = {"blocks.1.mlp.fc2.lora_up.weight": up, "blocks.1.mlp.fc2.lora_down.weight": down}
    weight = layer.weight.detach().clone()
    for lora_first in (True, False):
        ptd._kept.clear()
        model = comfy.sd.load_lora_for_models(_clone(base), None, lora, 0.7, 0)[0] if lora_first else _clone(base)
        controller, _ = _install(ptd, model, "a.safetensors")
        if not lora_first:
            model = comfy.sd.load_lora_for_models(model, None, lora, 0.7, 0)[0]
        import comfy.model_management
        comfy.model_management.load_models_gpu([model])
        try:
            inp = torch.randn(5, layer.in_features)
            hop = 2
            want = F.linear(inp, weight + 0.7 * up @ down + dense[("blocks.1.mlp.fc2", "ff.net.2", hop)],
                            layer.bias)
            with controller.activate(hop, HOP_TIMES[hop]), controller.forward_hooks(controller.current()):
                torch.testing.assert_close(layer(inp), want, rtol=1e-4, atol=1e-5)
        finally:
            comfy.model_management.unload_all_models()


def test_step_aware_modules_are_given_the_hop_schedule(ptd, folder):
    base = _tiny.native_model()
    _tiny.epfl_file(folder / "a.safetensors", base.get_model_object("diffusion_model"))
    model = _clone(base)
    seen = []

    def probe(apply_fn, args):
        seen.append(args["c"]["transformer_options"]["sample_sigmas"].tolist())
        return apply_fn(args["input"], args["timestep"], **args["c"])

    model.set_model_unet_function_wrapper(probe)
    _install(ptd, model, "a.safetensors")
    _sample(model)                                  # twenty steps asked for
    assert seen and all(len(s) == 5 for s in seen)


def test_the_file_list_is_read_again_when_settings_open(ptd, folder, monkeypatch):
    pytest.importorskip("aiohttp")
    from core import registry, routes
    spec = registry.current().specs["ptd"]
    monkeypatch.setitem(spec.settings["adapter"], "options", [{"value": "", "label": "No file in models/ptd"}])
    monkeypatch.setitem(spec.settings["adapter"], "default", "")
    base = _tiny.native_model()
    _tiny.epfl_file(folder / "new.safetensors", base.get_model_object("diffusion_model"))
    listed = {m["id"]: m for m in routes.manifest()["modules"]}
    adapter = listed["ptd"]["settings"]["adapter"]
    assert "new.safetensors" in [o["value"] for o in adapter["options"]]
    assert adapter["default"] in [o["value"] for o in adapter["options"]]


def test_the_hop_schedule_sits_where_the_sampler_was_given_its_own(ptd, folder, monkeypatch):
    """On a GPU ComfyUI hands the sampler a CUDA schedule; the one PTD swaps in must be on the same device
    (a CPU copy met the call's CUDA sigma: "Expected all tensors to be on the same device"). The meta device
    stands in for CUDA here."""
    import comfy.samplers
    from comfy.patcher_extension import WrappersMP
    base = _tiny.native_model()
    _tiny.epfl_file(folder / "a.safetensors", base.get_model_object("diffusion_model"))
    model = _clone(base)
    _install(ptd, model, "a.safetensors")
    given = {}
    monkeypatch.setattr(comfy.samplers.KSAMPLER, "sample",
                        lambda self, wrap, sigmas, extra_args, *a, **k: given.update(sigmas=sigmas) or "done")

    class Guider:
        cfg = 1.0
        model_patcher = None

    extra = {"model_options": {"transformer_options": {}}}
    run = model.wrappers[WrappersMP.SAMPLER_SAMPLE]["funui.ptd"][0]
    sigmas = torch.linspace(1.0, 0.0, 21, device="meta")
    assert run(None, Guider(), sigmas, extra, None, None, None, None, True) == "done"
    assert given["sigmas"].device == sigmas.device and given["sigmas"].dtype == sigmas.dtype
    seen = extra["model_options"]["transformer_options"]["sample_sigmas"]
    assert seen.device == sigmas.device and len(seen) == 5