fix(ingestion): address the #344 review (history re-check, keys, repair)
- Images without an embedded API graph: the scan now asks recorded history before skipping a file with no prompt metadata, and the monitor gives a keyless unlinked image one history-only re-check when its run's outputs are recorded (a seal notification from the outputs write; at most 1000 waiting images, each for an hour). - Run keys: an image is unkeyable when any positive node has no key, so a two-pass graph is never linked through the other pass; links follow ComfyUI's is_link exactly. - A failed save marks the node's run key like an empty editor, so an older entry of that key can't stand in for this run. - A placeholder is the latest entry: the image stays unlinked without falling through to history, the same rule as a popped placeholder. - An index on execution_run_outputs(filename, subfolder, root_key, path_style) for the history lookup, added by migration and accepted by the output schema check (any other index is still refused). - Repair tool: unlink removes only the link record and only while the image is still linked to the prompt it was judged on; every run prints what --apply does. - Real-host case proves the re-run's PromptManager node was cached.
This commit is contained in:
@@ -314,12 +314,16 @@ def migrate_execution_history(conn: sqlite3.Connection) -> None:
|
||||
if version == 1:
|
||||
install_context_schema(conn)
|
||||
install_output_schema(conn)
|
||||
from .execution_output_schema import OUTPUT_READ_INDEX_COLUMNS
|
||||
|
||||
# Read projections need reverse content lookup and the complete run order.
|
||||
for table, columns in (
|
||||
("execution_prompt_snapshots", "raw_text_id,snapshot_id"),
|
||||
("execution_contexts", "positive_snapshot_id,context_id"),
|
||||
("execution_run_contexts", "context_id,run_id,anchor_id"),
|
||||
("execution_runs", "observed_dequeue_at,session_id,queue_item_id,run_id"),
|
||||
# Attribution of an output file to its run (#342)
|
||||
("execution_run_outputs", ",".join(OUTPUT_READ_INDEX_COLUMNS)),
|
||||
):
|
||||
conn.execute(
|
||||
f"CREATE INDEX IF NOT EXISTS idx_history_read_{table} "
|
||||
|
||||
@@ -119,6 +119,25 @@ def install_output_schema(conn: sqlite3.Connection) -> None:
|
||||
conn.execute(_table_sql(table))
|
||||
|
||||
|
||||
# The one optional, non-unique read index (#342): output file -> run. Older
|
||||
# libraries may lack it; migrate_execution_history adds it.
|
||||
OUTPUT_READ_INDEX = "idx_history_read_execution_run_outputs"
|
||||
OUTPUT_READ_INDEX_COLUMNS = ("filename", "subfolder", "root_key", "path_style")
|
||||
|
||||
|
||||
def _is_read_index(table: str, index: tuple, keys: tuple) -> bool:
|
||||
"""Exactly the attribution read index, as the migration creates it."""
|
||||
return (
|
||||
table == "execution_run_outputs"
|
||||
and index[1] == OUTPUT_READ_INDEX
|
||||
and not index[2]
|
||||
and index[3] == "c"
|
||||
and not index[4]
|
||||
and tuple(key[2] for key in keys) == OUTPUT_READ_INDEX_COLUMNS
|
||||
and not any(key[3] or key[4] != "BINARY" for key in keys)
|
||||
)
|
||||
|
||||
|
||||
def output_schema_error(conn: sqlite3.Connection) -> str | None:
|
||||
"""Reject altered schemas and invalid rows before returning bounded DTOs."""
|
||||
if conn.execute(
|
||||
@@ -157,6 +176,8 @@ def output_schema_error(conn: sqlite3.Connection) -> str | None:
|
||||
keys = tuple(
|
||||
row for row in conn.execute(f'PRAGMA index_xinfo("{name}")') if row[5]
|
||||
)
|
||||
if _is_read_index(table, index, keys):
|
||||
continue
|
||||
if (
|
||||
not index[2]
|
||||
or index[3] != "pk"
|
||||
|
||||
@@ -4,6 +4,8 @@ import sqlite3
|
||||
from contextlib import ExitStack
|
||||
from typing import Callable, ContextManager
|
||||
|
||||
from .run_attribution import notify_outputs_sealed
|
||||
|
||||
try:
|
||||
from ..utils.execution_history_events import RunFinish, encode_command
|
||||
from ..utils.execution_outputs import OutputBundle, validate_output_bundle
|
||||
@@ -113,6 +115,10 @@ class ExecutionOutputOperations:
|
||||
"VALUES(?,?,?,?,?,?,?,?)",
|
||||
[(run_id, *reference) for reference in prepared],
|
||||
)
|
||||
# Committed: images waiting for this run may now be attributed (#342)
|
||||
notify_outputs_sealed(
|
||||
run_id, [dict(zip(_REFERENCE_FIELDS, row)) for row in prepared]
|
||||
)
|
||||
return True
|
||||
|
||||
def get_execution_outputs(self, run_id: str) -> dict:
|
||||
|
||||
@@ -16,9 +16,12 @@ only when all of this holds:
|
||||
Anything else returns None: the caller leaves the file unlinked. Read-only.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
from typing import Any, Optional
|
||||
import threading
|
||||
import weakref
|
||||
from typing import Any, Callable, Iterable, List, Optional
|
||||
|
||||
try:
|
||||
from ..utils.execution_outputs import snapshot_output_roots
|
||||
@@ -120,3 +123,36 @@ def stored_run_prompt(
|
||||
return None
|
||||
(prompt_id,) = named
|
||||
return prompt_id if isinstance(prompt_id, int) else None
|
||||
|
||||
|
||||
_LOGGER = logging.getLogger("prompt_manager.run_attribution")
|
||||
_SEALED_LISTENERS: List[weakref.WeakMethod] = []
|
||||
_SEALED_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def subscribe_outputs_sealed(method: Callable[[str, List[dict]], None]) -> None:
|
||||
"""Call bound ``method(run_id, references)`` after a run's outputs are recorded.
|
||||
|
||||
Held weakly: a listener that is garbage collected is dropped.
|
||||
"""
|
||||
with _SEALED_LOCK:
|
||||
_SEALED_LISTENERS.append(weakref.WeakMethod(method))
|
||||
|
||||
|
||||
def notify_outputs_sealed(run_id: str, references: Iterable[dict]) -> None:
|
||||
"""Tell listeners a run's output references were just committed.
|
||||
|
||||
A listener failure is logged and never reaches the history writer.
|
||||
"""
|
||||
references = list(references)
|
||||
with _SEALED_LOCK:
|
||||
alive = [ref for ref in _SEALED_LISTENERS if ref() is not None]
|
||||
_SEALED_LISTENERS[:] = alive
|
||||
listeners = [ref() for ref in alive]
|
||||
for listener in listeners:
|
||||
if listener is None:
|
||||
continue
|
||||
try:
|
||||
listener(run_id, references)
|
||||
except Exception:
|
||||
_LOGGER.warning("Output seal listener failed", exc_info=True)
|
||||
|
||||
Reference in New Issue
Block a user