Leak report: log every parent hand-over and every tracked model's state; test with the node cache keeping the chain
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5.5
parent
4f65a23f82
commit
75fd0d3885
+16
-5
@@ -51,6 +51,11 @@ def chains(target, depth=DEPTH, nodes=NODES, paths=PATHS) -> list:
|
||||
|
||||
def report(mm, when) -> int:
|
||||
dead = [m for m in mm.current_loaded_models if m.is_dead()]
|
||||
if dead:
|
||||
log.warning("FunUI Leak", f"{when}: ComfyUI tracks " + ", ".join(
|
||||
f"{type(l.real_model()).__name__ if callable(l.real_model) and l.real_model() is not None else '?'}"
|
||||
f"({'DEAD' if l.is_dead() else 'patcher alive' if l.model is not None else 'gone'}"
|
||||
f"{', has parent link' if getattr(l, '_parent_model', None) is not None else ''})" for l in mm.current_loaded_models))
|
||||
for m in dead:
|
||||
model = m.real_model()
|
||||
if model is None:
|
||||
@@ -64,11 +69,14 @@ def report(mm, when) -> int:
|
||||
p, n = p.parent, n + 1
|
||||
return n
|
||||
lineage = ", ".join(f"{type(o).__name__}(parents {depth(o)}, tracked {any(l.model is o for l in mm.current_loaded_models)})" for o in owners)
|
||||
log.warning("FunUI Leak", f"{when}: {type(model).__name__} outlived its patcher. Patchers holding it: {lineage or 'none'}. What holds it:\n" + "\n".join(chains(model)))
|
||||
fin = getattr(m, "_patcher_finalizer", None)
|
||||
state = f"its hand-over {'registered' if fin is not None else 'was never registered'}{' and still pending' if fin is not None and fin.alive else ''}"
|
||||
log.warning("FunUI Leak", f"{when}: {type(model).__name__} outlived its patcher ({state}). Patchers holding it: {lineage or 'none'}. What holds it:\n" + "\n".join(chains(model)))
|
||||
return len(dead)
|
||||
|
||||
|
||||
_in_gc = [False]
|
||||
_seen = [0]
|
||||
|
||||
|
||||
def _gc_flag(phase, info):
|
||||
@@ -88,10 +96,13 @@ def watch_switch(mm) -> bool:
|
||||
try:
|
||||
parent = self._parent_model()
|
||||
real = self.real_model() if callable(self.real_model) else None
|
||||
if parent is None and real is not None:
|
||||
live = [type(o).__name__ for o in gc.get_referrers(real) if type(o).__name__.startswith("ModelPatcher")]
|
||||
log.warning("FunUI Leak", f"a {type(real).__name__} patcher died and its parent was already gone, so ComfyUI loses track of the model. "
|
||||
f"Python's gc was running: {_in_gc[0]}. Patchers still holding the model: {len(live)}")
|
||||
_seen[0] += 1
|
||||
if _seen[0] <= 40: # every hand-over, so a missing one shows too
|
||||
depth, p = 0, parent
|
||||
while p is not None and getattr(p, "parent", None) is not None and depth < 50:
|
||||
p, depth = p.parent, depth + 1
|
||||
log.info("FunUI Leak", f"hand-over {_seen[0]}: a {type(real).__name__ if real is not None else 'dead'} patcher died; its parent is "
|
||||
f"{'gone' if parent is None else type(parent).__name__ + f' with {depth} more above it'}; Python's gc running: {_in_gc[0]}")
|
||||
except Exception as exc: # noqa: BLE001 -- a diagnostic must never stop a run
|
||||
log.failed("leak report", exc)
|
||||
return original(self)
|
||||
|
||||
@@ -68,7 +68,7 @@ def test_a_hand_over_that_finds_no_parent_is_said(comfyui, capsys):
|
||||
loaded._parent_model = lambda: None # what a gc that cleared the weak link leaves behind
|
||||
loaded._switch_parent()
|
||||
shown = capsys.readouterr().err
|
||||
assert "parent was already gone" in shown and "Python's gc was running: False" in shown
|
||||
assert "its parent is gone" in shown and "Python's gc running: False" in shown
|
||||
del keep
|
||||
finally:
|
||||
mm.LoadedModel._switch_parent = original
|
||||
|
||||
@@ -639,10 +639,11 @@ def test_the_real_model_class_is_collectable_after_a_sampled_veda_run(veda_on, f
|
||||
mm.unload_all_models()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cached", [False, True], ids=["all-dropped", "node-cache-keeps-the-chain"])
|
||||
@pytest.mark.parametrize("interrupted", [False, True], ids=["finished", "interrupted"])
|
||||
@pytest.mark.parametrize("stage_lora", [None, "First", "Second"], ids=["no-lora", "lora-first", "lora-second"])
|
||||
@pytest.mark.parametrize("staged", [False, True], ids=["one-stage", "two-stages"])
|
||||
def test_every_modifier_on_top_of_veda_leaves_a_collectable_model(veda_on, folder, comfyui, staged, interrupted, stage_lora):
|
||||
def test_every_modifier_on_top_of_veda_leaves_a_collectable_model(veda_on, folder, comfyui, staged, interrupted, stage_lora, cached):
|
||||
"""The class of leak ComfyUI names 'memory leak with model MiniMaxH3': a hook or wrapper that holds its own patcher is a cycle only a
|
||||
full gc frees, and that gc clears ComfyUI's weak link to the parent copy first. Veda in the loader, every modifier on top, a real
|
||||
sampling run, then everything dropped: the model must die by reference counting alone."""
|
||||
@@ -697,6 +698,11 @@ def test_every_modifier_on_top_of_veda_leaves_a_collectable_model(veda_on, folde
|
||||
gc.collect()
|
||||
gc.disable()
|
||||
try:
|
||||
if cached: # ComfyUI's node cache still holds the loader and modifier outputs; only the run's own clones are gone
|
||||
del guider, noise, latent, cond, config
|
||||
dead = [m for m in mm.current_loaded_models if m.is_dead()]
|
||||
assert not dead, "ComfyUI lost track of a model the node cache still holds"
|
||||
return
|
||||
del guider, patched, veda, loaded, base, noise, latent, cond, config
|
||||
dead = [m for m in mm.current_loaded_models if m.is_dead()]
|
||||
assert not dead and ref() is None, [type(r).__name__ for r in gc.get_referrers(ref())] if ref() is not None else "dead model tracked"
|
||||
|
||||
Reference in New Issue
Block a user