nodes/ComfyAssets-ComfyUI_PromptManager
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(ingestion): address the #344 review (history re-check, keys, repair)

Browse Source 
- Images without an embedded API graph: the scan now asks recorded
  history before skipping a file with no prompt metadata, and the monitor
  gives a keyless unlinked image one history-only re-check when its run's
  outputs are recorded (a seal notification from the outputs write; at
  most 1000 waiting images, each for an hour).
- Run keys: an image is unkeyable when any positive node has no key, so a
  two-pass graph is never linked through the other pass; links follow
  ComfyUI's is_link exactly.
- A failed save marks the node's run key like an empty editor, so an
  older entry of that key can't stand in for this run.
- A placeholder is the latest entry: the image stays unlinked without
  falling through to history, the same rule as a popped placeholder.
- An index on execution_run_outputs(filename, subfolder, root_key,
  path_style) for the history lookup, added by migration and accepted by
  the output schema check (any other index is still refused).
- Repair tool: unlink removes only the link record and only while the
  image is still linked to the prompt it was judged on; every run prints
  what --apply does.
- Real-host case proves the re-run's PromptManager node was cached.
This commit is contained in:
Vito Sansevero
2026-10-25 14:23:17 -07:00
parent 4ab0119580
commit 417a05a2db
16 changed files with 742 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
utils/image_monitor.py
+128 -5
View File
@@ -28,7 +28,8 @@ import os
import queue
import time
import threading
from typing import Callable, Optional, Dict, Any, Tuple
from collections import OrderedDict
from typing import Callable, Optional, Dict, Any, List, Tuple
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
@@ -42,16 +43,31 @@ from .usage_tracking import survive_switch
try:
from ..database.pair_ingestion import ingest_resolved_pair
from ..database.run_attribution import host_output_directory, stored_run_prompt
from ..database.run_attribution import (
host_output_directory,
stored_run_prompt,
subscribe_outputs_sealed,
)
except ImportError:
from database.pair_ingestion import ingest_resolved_pair
from database.run_attribution import host_output_directory, stored_run_prompt
from database.run_attribution import (
host_output_directory,
stored_run_prompt,
subscribe_outputs_sealed,
)
# Upper bound on images waiting to be processed (a burst beyond this is skipped)
MAX_PENDING_IMAGES = 1000
# A file counts as fully written once its size is unchanged across two reads
SETTLE_INTERVAL_SECONDS = 0.1
SETTLE_ATTEMPTS = 20
# Keyless images (no embedded API graph) left unlinked wait for their run's
# recorded outputs for one history-only re-check (#342): at most this many,
# each for at most this long.
MAX_AWAITING_HISTORY = 1000
AWAIT_HISTORY_SECONDS = 3600
# ``schedule`` snapshot meaning "history-only re-check" (not a new image)
HISTORY_ONLY = object()
class ImageGenerationHandler(FileSystemEventHandler):
@@ -113,6 +129,11 @@ class ImageGenerationHandler(FileSystemEventHandler):
self._work: "queue.Queue[Optional[str]]" = queue.Queue()
self._stop = threading.Event()
self._worker: Optional[threading.Thread] = None
# Keyless unlinked images awaiting their run's recorded outputs:
# normalized path -> (path, monotonic deadline), oldest first.
self._awaiting: "OrderedDict[str, Tuple[str, float]]" = OrderedDict()
self._awaiting_lock = threading.Lock()
self._sealed_subscribed = False
def on_created(self, event):
"""Handle filesystem creation events.
@@ -180,6 +201,8 @@ class ImageGenerationHandler(FileSystemEventHandler):
and exits; the next ``schedule`` starts a fresh worker with its own
queue. Safe to call repeatedly and before anything was scheduled.
"""
with self._awaiting_lock:
self._awaiting.clear()
with self._pending_lock:
self._pending.clear()
worker, self._worker = self._worker, None
@@ -307,6 +330,9 @@ class ImageGenerationHandler(FileSystemEventHandler):
appeared (``PromptTracker.keyed_snapshot``); None reads it now
"""
try:
if prompt_snapshot is HISTORY_ONLY:
self._recheck_history(image_path)
return
self.logger.info(f"Processing image: {image_path}")
if not os.path.exists(image_path):
@@ -399,6 +425,16 @@ class ImageGenerationHandler(FileSystemEventHandler):
"""
keys = image_run_keys(metadata)
name = os.path.basename(image_path)
if not keys:
# No tracked run can be proven: history now, and once more after
# the run's outputs are recorded (``_on_outputs_sealed``)
self._await_history(image_path)
prompt_id = self._history_prompt(image_path)
if prompt_id is None:
return None
self._stop_awaiting(image_path)
self.logger.info(f"History match: prompt {prompt_id} for {name}")
return {"id": prompt_id, "from_history": True}
if len(keys) == 1:
(key,) = keys.values()
if prompt_snapshot is None:
@@ -414,10 +450,14 @@ class ImageGenerationHandler(FileSystemEventHandler):
return None
self.logger.info(f"Queue match: prompt {queued['id']} for {name}")
return queued
if not context.get("placeholder") and run_id and count == 1:
if context.get("placeholder"):
# Same rule as a popped placeholder: that run had no prompt
self.logger.info(f"Latest run for {name} had no prompt text")
return None
if run_id and count == 1:
self.logger.info(f"Run match: prompt {context['id']} for {name}")
return context
elif len(keys) > 1:
else:
self.logger.debug(f"{len(keys)} positive prompt nodes for {name}")
prompt_id = self._history_prompt(image_path)
if prompt_id is None:
@@ -425,6 +465,89 @@ class ImageGenerationHandler(FileSystemEventHandler):
self.logger.info(f"History match: prompt {prompt_id} for {name}")
return {"id": prompt_id, "from_history": True}
@staticmethod
def _awaiting_key(image_path: str) -> str:
return os.path.normcase(os.path.abspath(image_path))
def _prune_awaiting(self) -> None:
"""Drop expired and surplus entries (``_awaiting_lock`` held)."""
now = time.monotonic()
for key in [k for k, (_p, due) in self._awaiting.items() if due <= now]:
del self._awaiting[key]
while len(self._awaiting) > MAX_AWAITING_HISTORY:
self._awaiting.popitem(last=False)
def _await_history(self, image_path: str) -> None:
"""Re-check ``image_path`` once, history only, when its run is sealed."""
if not self._sealed_subscribed:
subscribe_outputs_sealed(self._on_outputs_sealed)
self._sealed_subscribed = True
key = self._awaiting_key(image_path)
with self._awaiting_lock:
self._awaiting.pop(key, None)
self._awaiting[key] = (
image_path,
time.monotonic() + AWAIT_HISTORY_SECONDS,
)
self._prune_awaiting()
def _stop_awaiting(self, image_path: str) -> None:
with self._awaiting_lock:
self._awaiting.pop(self._awaiting_key(image_path), None)
def awaiting_history(self) -> List[str]:
"""Normalized paths still waiting for their run's recorded outputs."""
with self._awaiting_lock:
self._prune_awaiting()
return list(self._awaiting)
def _on_outputs_sealed(self, run_id: str, references: List[dict]) -> None:
"""A run's outputs were recorded: re-check its waiting images once.
Called on the history writer's thread, so it only schedules the work.
"""
root = host_output_directory()
if not root:
return
due = []
with self._awaiting_lock:
self._prune_awaiting()
for reference in references:
if reference.get("folder_type") != "output":
continue
key = self._awaiting_key(
os.path.join(
root, reference.get("subfolder", ""), reference["filename"]
)
)
waiting = self._awaiting.pop(key, None)
if waiting is not None:
due.append(waiting[0])
for image_path in due:
self.schedule(image_path, HISTORY_ONLY)
def _recheck_history(self, image_path: str) -> None:
"""The one history-only re-check of a keyless image (#342)."""
name = os.path.basename(image_path)
if not os.path.exists(image_path):
return
prompt_id = self._history_prompt(image_path)
if prompt_id is None:
self.logger.info(
f"No certain prompt for {name} after its run ended; left unlinked"
)
return
self.logger.info(f"History match after its run ended: {prompt_id} for {name}")
try:
metadata = self.metadata_extractor.extract_metadata(image_path)
except Exception:
metadata = None
self.link_image_to_prompt(
image_path,
{"id": prompt_id, "from_history": True},
metadata or {"file_info": self.get_basic_file_info(image_path)},
)
def _history_prompt(self, image_path: str) -> Optional[int]:
"""The stored prompt recorded history attributes to this file, if one."""
try:
utils/run_keys.py
+32 -18
View File
@@ -4,9 +4,14 @@ A PromptManager node registers the text it executed under a *run key*: a digest
of the node and everything upstream of it in the queued API graph. The image
embeds the same graph, so the image monitor can compute the same key for the
image's positive prompt node. Equal keys mean the node ran with exactly the
inputs (and upstream nodes) that produced this image. That is also the identity
ComfyUI's own output cache uses: a node whose upstream is unchanged is served
from the cache, so a re-run of an unchanged upstream reproduces the same text.
inputs (and upstream nodes) that produced this image.
Equal keys do not always mean equal text. ComfyUI's cache signature also folds
in each node's ``IS_CHANGED``, which the key cannot see: an upstream node that
forces a re-run (a random line, an "always run" LLM) can give two runs one key
and two texts. The key is therefore never used alone: the monitor also requires
the entry to come from the run the snapshot taken when the image appeared names
(``PromptTracker.pop_for_key``, ``ImageGenerationHandler._find_prompt_from_run``).
Pure functions; the only host dependency (the executing run id) is optional.
"""
@@ -23,20 +28,27 @@ except ImportError:
RUN_KEY_VERSION = 1
def is_link(value: Any) -> bool:
"""ComfyUI's own test (``comfy_execution.graph_utils.is_link``), exactly.
A two-item list of a ``str`` node id and an ``int`` or ``float`` slot (a
``bool`` slot passes too, as it does in ComfyUI). Tuples are values.
"""
if not isinstance(value, list):
return False
if len(value) != 2:
return False
if not isinstance(value[0], str):
return False
if not isinstance(value[1], int) and not isinstance(value[1], float):
return False
return True
def _links(inputs: Any) -> List[str]:
if not isinstance(inputs, dict):
return []
found = []
for value in inputs.values():
if (
isinstance(value, (list, tuple))
and len(value) == 2
and isinstance(value[0], (str, int))
and isinstance(value[1], int)
and not isinstance(value[1], bool)
):
found.append(str(value[0]))
return found
return [value[0] for value in inputs.values() if is_link(value)]
def prompt_node_key(graph: Any, node_id: Any) -> Optional[str]:
@@ -79,8 +91,9 @@ def prompt_node_key(graph: Any, node_id: Any) -> Optional[str]:
def image_run_keys(metadata: Any) -> Dict[str, str]:
"""``{node_id: run key}`` for each positive PromptManager node of an image.
Empty without an embedded API graph: an image without one can never be
proven to belong to a tracked run.
Empty without an embedded API graph, and empty when any positive node has
no key: one unkeyable node must not leave another looking like the only
positive prompt (a two-pass workflow would link the wrong pass).
"""
graph = metadata.get("prompt") if isinstance(metadata, dict) else None
if isinstance(graph, str):
@@ -93,8 +106,9 @@ def image_run_keys(metadata: Any) -> Dict[str, str]:
keys = {}
for node_id in run_prompt_nodes(graph):
key = prompt_node_key(graph, node_id)
if key is not None:
keys[node_id] = key
if key is None:
return {}
keys[node_id] = key
return keys