From 75fd0d38858be98e4337bcfba104ce6765ef1347 Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Sun, 8 Nov 2026 19:07:10 +0200 Subject: [PATCH] Leak report: log every parent hand-over and every tracked model's state; test with the node cache keeping the chain Co-Authored-By: Claude Sonnet 5.5 --- core/leak_report.py | 21 ++++++++++++++++----- core/tests/test_leak_report.py | 2 +- modules/loaders/tests/test_veda.py | 8 +++++++- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/core/leak_report.py b/core/leak_report.py index 905b4f7b..b2ff1dae 100644 --- a/core/leak_report.py +++ b/core/leak_report.py @@ -51,6 +51,11 @@ def chains(target, depth=DEPTH, nodes=NODES, paths=PATHS) -> list: def report(mm, when) -> int: dead = [m for m in mm.current_loaded_models if m.is_dead()] + if dead: + log.warning("FunUI Leak", f"{when}: ComfyUI tracks " + ", ".join( + f"{type(l.real_model()).__name__ if callable(l.real_model) and l.real_model() is not None else '?'}" + f"({'DEAD' if l.is_dead() else 'patcher alive' if l.model is not None else 'gone'}" + f"{', has parent link' if getattr(l, '_parent_model', None) is not None else ''})" for l in mm.current_loaded_models)) for m in dead: model = m.real_model() if model is None: @@ -64,11 +69,14 @@ def report(mm, when) -> int: p, n = p.parent, n + 1 return n lineage = ", ".join(f"{type(o).__name__}(parents {depth(o)}, tracked {any(l.model is o for l in mm.current_loaded_models)})" for o in owners) - log.warning("FunUI Leak", f"{when}: {type(model).__name__} outlived its patcher. Patchers holding it: {lineage or 'none'}. What holds it:\n" + "\n".join(chains(model))) + fin = getattr(m, "_patcher_finalizer", None) + state = f"its hand-over {'registered' if fin is not None else 'was never registered'}{' and still pending' if fin is not None and fin.alive else ''}" + log.warning("FunUI Leak", f"{when}: {type(model).__name__} outlived its patcher ({state}). Patchers holding it: {lineage or 'none'}. What holds it:\n" + "\n".join(chains(model))) return len(dead) _in_gc = [False] +_seen = [0] def _gc_flag(phase, info): @@ -88,10 +96,13 @@ def watch_switch(mm) -> bool: try: parent = self._parent_model() real = self.real_model() if callable(self.real_model) else None - if parent is None and real is not None: - live = [type(o).__name__ for o in gc.get_referrers(real) if type(o).__name__.startswith("ModelPatcher")] - log.warning("FunUI Leak", f"a {type(real).__name__} patcher died and its parent was already gone, so ComfyUI loses track of the model. " - f"Python's gc was running: {_in_gc[0]}. Patchers still holding the model: {len(live)}") + _seen[0] += 1 + if _seen[0] <= 40: # every hand-over, so a missing one shows too + depth, p = 0, parent + while p is not None and getattr(p, "parent", None) is not None and depth < 50: + p, depth = p.parent, depth + 1 + log.info("FunUI Leak", f"hand-over {_seen[0]}: a {type(real).__name__ if real is not None else 'dead'} patcher died; its parent is " + f"{'gone' if parent is None else type(parent).__name__ + f' with {depth} more above it'}; Python's gc running: {_in_gc[0]}") except Exception as exc: # noqa: BLE001 -- a diagnostic must never stop a run log.failed("leak report", exc) return original(self) diff --git a/core/tests/test_leak_report.py b/core/tests/test_leak_report.py index 85828280..320edf11 100644 --- a/core/tests/test_leak_report.py +++ b/core/tests/test_leak_report.py @@ -68,7 +68,7 @@ def test_a_hand_over_that_finds_no_parent_is_said(comfyui, capsys): loaded._parent_model = lambda: None # what a gc that cleared the weak link leaves behind loaded._switch_parent() shown = capsys.readouterr().err - assert "parent was already gone" in shown and "Python's gc was running: False" in shown + assert "its parent is gone" in shown and "Python's gc running: False" in shown del keep finally: mm.LoadedModel._switch_parent = original diff --git a/modules/loaders/tests/test_veda.py b/modules/loaders/tests/test_veda.py index d9887a80..bcabc871 100644 --- a/modules/loaders/tests/test_veda.py +++ b/modules/loaders/tests/test_veda.py @@ -639,10 +639,11 @@ def test_the_real_model_class_is_collectable_after_a_sampled_veda_run(veda_on, f mm.unload_all_models() +@pytest.mark.parametrize("cached", [False, True], ids=["all-dropped", "node-cache-keeps-the-chain"]) @pytest.mark.parametrize("interrupted", [False, True], ids=["finished", "interrupted"]) @pytest.mark.parametrize("stage_lora", [None, "First", "Second"], ids=["no-lora", "lora-first", "lora-second"]) @pytest.mark.parametrize("staged", [False, True], ids=["one-stage", "two-stages"]) -def test_every_modifier_on_top_of_veda_leaves_a_collectable_model(veda_on, folder, comfyui, staged, interrupted, stage_lora): +def test_every_modifier_on_top_of_veda_leaves_a_collectable_model(veda_on, folder, comfyui, staged, interrupted, stage_lora, cached): """The class of leak ComfyUI names 'memory leak with model MiniMaxH3': a hook or wrapper that holds its own patcher is a cycle only a full gc frees, and that gc clears ComfyUI's weak link to the parent copy first. Veda in the loader, every modifier on top, a real sampling run, then everything dropped: the model must die by reference counting alone.""" @@ -697,6 +698,11 @@ def test_every_modifier_on_top_of_veda_leaves_a_collectable_model(veda_on, folde gc.collect() gc.disable() try: + if cached: # ComfyUI's node cache still holds the loader and modifier outputs; only the run's own clones are gone + del guider, noise, latent, cond, config + dead = [m for m in mm.current_loaded_models if m.is_dead()] + assert not dead, "ComfyUI lost track of a model the node cache still holds" + return del guider, patched, veda, loaded, base, noise, latent, cond, config dead = [m for m in mm.current_loaded_models if m.is_dead()] assert not dead and ref() is None, [type(r).__name__ for r in gc.get_referrers(ref())] if ref() is not None else "dead model tracked"