fix(ingestion): address the #344 review (history re-check, keys, repair)

- Images without an embedded API graph: the scan now asks recorded
  history before skipping a file with no prompt metadata, and the monitor
  gives a keyless unlinked image one history-only re-check when its run's
  outputs are recorded (a seal notification from the outputs write; at
  most 1000 waiting images, each for an hour).
- Run keys: an image is unkeyable when any positive node has no key, so a
  two-pass graph is never linked through the other pass; links follow
  ComfyUI's is_link exactly.
- A failed save marks the node's run key like an empty editor, so an
  older entry of that key can't stand in for this run.
- A placeholder is the latest entry: the image stays unlinked without
  falling through to history, the same rule as a popped placeholder.
- An index on execution_run_outputs(filename, subfolder, root_key,
  path_style) for the history lookup, added by migration and accepted by
  the output schema check (any other index is still refused).
- Repair tool: unlink removes only the link record and only while the
  image is still linked to the prompt it was judged on; every run prints
  what --apply does.
- Real-host case proves the re-run's PromptManager node was cached.
This commit is contained in:
Vito Sansevero
2026-10-25 14:23:17 -07:00
parent 4ab0119580
commit 417a05a2db
16 changed files with 742 additions and 63 deletions
+4
View File
@@ -314,12 +314,16 @@ def migrate_execution_history(conn: sqlite3.Connection) -> None:
if version == 1:
install_context_schema(conn)
install_output_schema(conn)
from .execution_output_schema import OUTPUT_READ_INDEX_COLUMNS
# Read projections need reverse content lookup and the complete run order.
for table, columns in (
("execution_prompt_snapshots", "raw_text_id,snapshot_id"),
("execution_contexts", "positive_snapshot_id,context_id"),
("execution_run_contexts", "context_id,run_id,anchor_id"),
("execution_runs", "observed_dequeue_at,session_id,queue_item_id,run_id"),
# Attribution of an output file to its run (#342)
("execution_run_outputs", ",".join(OUTPUT_READ_INDEX_COLUMNS)),
):
conn.execute(
f"CREATE INDEX IF NOT EXISTS idx_history_read_{table} "
+21
View File
@@ -119,6 +119,25 @@ def install_output_schema(conn: sqlite3.Connection) -> None:
conn.execute(_table_sql(table))
# The one optional, non-unique read index (#342): output file -> run. Older
# libraries may lack it; migrate_execution_history adds it.
OUTPUT_READ_INDEX = "idx_history_read_execution_run_outputs"
OUTPUT_READ_INDEX_COLUMNS = ("filename", "subfolder", "root_key", "path_style")
def _is_read_index(table: str, index: tuple, keys: tuple) -> bool:
"""Exactly the attribution read index, as the migration creates it."""
return (
table == "execution_run_outputs"
and index[1] == OUTPUT_READ_INDEX
and not index[2]
and index[3] == "c"
and not index[4]
and tuple(key[2] for key in keys) == OUTPUT_READ_INDEX_COLUMNS
and not any(key[3] or key[4] != "BINARY" for key in keys)
)
def output_schema_error(conn: sqlite3.Connection) -> str | None:
"""Reject altered schemas and invalid rows before returning bounded DTOs."""
if conn.execute(
@@ -157,6 +176,8 @@ def output_schema_error(conn: sqlite3.Connection) -> str | None:
keys = tuple(
row for row in conn.execute(f'PRAGMA index_xinfo("{name}")') if row[5]
)
if _is_read_index(table, index, keys):
continue
if (
not index[2]
or index[3] != "pk"
+6
View File
@@ -4,6 +4,8 @@ import sqlite3
from contextlib import ExitStack
from typing import Callable, ContextManager
from .run_attribution import notify_outputs_sealed
try:
from ..utils.execution_history_events import RunFinish, encode_command
from ..utils.execution_outputs import OutputBundle, validate_output_bundle
@@ -113,6 +115,10 @@ class ExecutionOutputOperations:
"VALUES(?,?,?,?,?,?,?,?)",
[(run_id, *reference) for reference in prepared],
)
# Committed: images waiting for this run may now be attributed (#342)
notify_outputs_sealed(
run_id, [dict(zip(_REFERENCE_FIELDS, row)) for row in prepared]
)
return True
def get_execution_outputs(self, run_id: str) -> dict:
+37 -1
View File
@@ -16,9 +16,12 @@ only when all of this holds:
Anything else returns None: the caller leaves the file unlinked. Read-only.
"""
import logging
import os
import sqlite3
from typing import Any, Optional
import threading
import weakref
from typing import Any, Callable, Iterable, List, Optional
try:
from ..utils.execution_outputs import snapshot_output_roots
@@ -120,3 +123,36 @@ def stored_run_prompt(
return None
(prompt_id,) = named
return prompt_id if isinstance(prompt_id, int) else None
_LOGGER = logging.getLogger("prompt_manager.run_attribution")
_SEALED_LISTENERS: List[weakref.WeakMethod] = []
_SEALED_LOCK = threading.Lock()
def subscribe_outputs_sealed(method: Callable[[str, List[dict]], None]) -> None:
"""Call bound ``method(run_id, references)`` after a run's outputs are recorded.
Held weakly: a listener that is garbage collected is dropped.
"""
with _SEALED_LOCK:
_SEALED_LISTENERS.append(weakref.WeakMethod(method))
def notify_outputs_sealed(run_id: str, references: Iterable[dict]) -> None:
"""Tell listeners a run's output references were just committed.
A listener failure is logged and never reaches the history writer.
"""
references = list(references)
with _SEALED_LOCK:
alive = [ref for ref in _SEALED_LISTENERS if ref() is not None]
_SEALED_LISTENERS[:] = alive
listeners = [ref() for ref in alive]
for listener in listeners:
if listener is None:
continue
try:
listener(run_id, references)
except Exception:
_LOGGER.warning("Output seal listener failed", exc_info=True)