Leak report: log every parent hand-over and every tracked model's state; test with the node cache keeping the chain

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
digital-garbage
2026-11-08 19:07:10 +02:00
co-authored by Claude Sonnet 5.5
parent 4f65a23f82
commit 75fd0d3885
3 changed files with 24 additions and 7 deletions
+16 -5
View File
@@ -51,6 +51,11 @@ def chains(target, depth=DEPTH, nodes=NODES, paths=PATHS) -> list:
def report(mm, when) -> int:
dead = [m for m in mm.current_loaded_models if m.is_dead()]
if dead:
log.warning("FunUI Leak", f"{when}: ComfyUI tracks " + ", ".join(
f"{type(l.real_model()).__name__ if callable(l.real_model) and l.real_model() is not None else '?'}"
f"({'DEAD' if l.is_dead() else 'patcher alive' if l.model is not None else 'gone'}"
f"{', has parent link' if getattr(l, '_parent_model', None) is not None else ''})" for l in mm.current_loaded_models))
for m in dead:
model = m.real_model()
if model is None:
@@ -64,11 +69,14 @@ def report(mm, when) -> int:
p, n = p.parent, n + 1
return n
lineage = ", ".join(f"{type(o).__name__}(parents {depth(o)}, tracked {any(l.model is o for l in mm.current_loaded_models)})" for o in owners)
log.warning("FunUI Leak", f"{when}: {type(model).__name__} outlived its patcher. Patchers holding it: {lineage or 'none'}. What holds it:\n" + "\n".join(chains(model)))
fin = getattr(m, "_patcher_finalizer", None)
state = f"its hand-over {'registered' if fin is not None else 'was never registered'}{' and still pending' if fin is not None and fin.alive else ''}"
log.warning("FunUI Leak", f"{when}: {type(model).__name__} outlived its patcher ({state}). Patchers holding it: {lineage or 'none'}. What holds it:\n" + "\n".join(chains(model)))
return len(dead)
_in_gc = [False]
_seen = [0]
def _gc_flag(phase, info):
@@ -88,10 +96,13 @@ def watch_switch(mm) -> bool:
try:
parent = self._parent_model()
real = self.real_model() if callable(self.real_model) else None
if parent is None and real is not None:
live = [type(o).__name__ for o in gc.get_referrers(real) if type(o).__name__.startswith("ModelPatcher")]
log.warning("FunUI Leak", f"a {type(real).__name__} patcher died and its parent was already gone, so ComfyUI loses track of the model. "
f"Python's gc was running: {_in_gc[0]}. Patchers still holding the model: {len(live)}")
_seen[0] += 1
if _seen[0] <= 40: # every hand-over, so a missing one shows too
depth, p = 0, parent
while p is not None and getattr(p, "parent", None) is not None and depth < 50:
p, depth = p.parent, depth + 1
log.info("FunUI Leak", f"hand-over {_seen[0]}: a {type(real).__name__ if real is not None else 'dead'} patcher died; its parent is "
f"{'gone' if parent is None else type(parent).__name__ + f' with {depth} more above it'}; Python's gc running: {_in_gc[0]}")
except Exception as exc: # noqa: BLE001 -- a diagnostic must never stop a run
log.failed("leak report", exc)
return original(self)
+1 -1
View File
@@ -68,7 +68,7 @@ def test_a_hand_over_that_finds_no_parent_is_said(comfyui, capsys):
loaded._parent_model = lambda: None # what a gc that cleared the weak link leaves behind
loaded._switch_parent()
shown = capsys.readouterr().err
assert "parent was already gone" in shown and "Python's gc was running: False" in shown
assert "its parent is gone" in shown and "Python's gc running: False" in shown
del keep
finally:
mm.LoadedModel._switch_parent = original
+7 -1
View File
@@ -639,10 +639,11 @@ def test_the_real_model_class_is_collectable_after_a_sampled_veda_run(veda_on, f
mm.unload_all_models()
@pytest.mark.parametrize("cached", [False, True], ids=["all-dropped", "node-cache-keeps-the-chain"])
@pytest.mark.parametrize("interrupted", [False, True], ids=["finished", "interrupted"])
@pytest.mark.parametrize("stage_lora", [None, "First", "Second"], ids=["no-lora", "lora-first", "lora-second"])
@pytest.mark.parametrize("staged", [False, True], ids=["one-stage", "two-stages"])
def test_every_modifier_on_top_of_veda_leaves_a_collectable_model(veda_on, folder, comfyui, staged, interrupted, stage_lora):
def test_every_modifier_on_top_of_veda_leaves_a_collectable_model(veda_on, folder, comfyui, staged, interrupted, stage_lora, cached):
"""The class of leak ComfyUI names 'memory leak with model MiniMaxH3': a hook or wrapper that holds its own patcher is a cycle only a
full gc frees, and that gc clears ComfyUI's weak link to the parent copy first. Veda in the loader, every modifier on top, a real
sampling run, then everything dropped: the model must die by reference counting alone."""
@@ -697,6 +698,11 @@ def test_every_modifier_on_top_of_veda_leaves_a_collectable_model(veda_on, folde
gc.collect()
gc.disable()
try:
if cached: # ComfyUI's node cache still holds the loader and modifier outputs; only the run's own clones are gone
del guider, noise, latent, cond, config
dead = [m for m in mm.current_loaded_models if m.is_dead()]
assert not dead, "ComfyUI lost track of a model the node cache still holds"
return
del guider, patched, veda, loaded, base, noise, latent, cond, config
dead = [m for m in mm.current_loaded_models if m.is_dead()]
assert not dead and ref() is None, [type(r).__name__ for r in gc.get_referrers(ref())] if ref() is not None else "dead model tracked"