Merge remote-tracking branch 'origin/main' into feat/331-collections-pr1

# Conflicts:
#	database/duplicate_copies.py
#	database/models.py
#	py/api/duplicate_files.py
#	tests/test_duplicate_image_categories.py
This commit is contained in:
Vito Sansevero
2026-10-28 08:52:10 -07:00
40 changed files with 2596 additions and 2686 deletions
+150 -26
View File
@@ -15,6 +15,10 @@ only when all of this holds:
Anything else returns None: the caller leaves the file unlinked. Read-only.
``recorded_output`` applies the first two rules and returns, for the
inspector (#386), the one output node (save node) that run reported the file
from and the texts PromptManager recorded at execution per sampler.
Lookups are batched (``stored_run_prompts``): one parameterized query per
chunk of at most ``LOOKUP_CHUNK`` distinct file names, matched in memory. The
history schema is left exactly as it is (no extra index), so older builds can
@@ -26,6 +30,7 @@ import os
import sqlite3
import threading
import weakref
from dataclasses import dataclass
from typing import Any, Callable, Dict, Iterable, List, Optional
try:
@@ -45,7 +50,8 @@ LOOKUP_CHUNK = 500
_RUNS_FOR_NAMES = (
"SELECT DISTINCT o.filename, o.subfolder, o.root_key, o.path_style, o.run_id, "
"r.observed_dequeue_at, COALESCE(r.terminal_event_at, r.observed_callback_at) "
"r.observed_dequeue_at, COALESCE(r.terminal_event_at, r.observed_callback_at), "
"o.output_node_id "
"FROM execution_run_outputs o JOIN execution_runs r ON r.run_id = o.run_id "
"WHERE o.folder_type = 'output' AND o.filename IN ({marks})"
)
@@ -138,9 +144,10 @@ def stored_run_prompt(
def _runs_by_location(db: Any, names: List[str]) -> Optional[Dict[tuple, dict]]:
"""``{(filename, subfolder, root_key, style): {run_id: (start, end)}}``.
"""``{(filename, subfolder, root_key, style): {run_id: (start, end, nodes)}}``.
One query per ``LOOKUP_CHUNK`` names; None without history tables.
``nodes`` is the set of output node ids that reported the location in that
run. One query per ``LOOKUP_CHUNK`` names; None without history tables.
"""
found: Dict[tuple, dict] = {}
conn = db.model.get_connection()
@@ -151,12 +158,49 @@ def _runs_by_location(db: Any, names: List[str]) -> Optional[Dict[tuple, dict]]:
rows = conn.execute(query, chunk).fetchall() # nosec B608
except sqlite3.Error:
return None # no history tables (a library without history)
for name, subfolder, root_key, style, run_id, began, ended in rows:
for name, subfolder, root_key, style, run_id, began, ended, node in rows:
runs = found.setdefault((name, subfolder, root_key, style), {})
runs[run_id] = (began, ended)
runs.setdefault(run_id, (began, ended, set()))[2].add(node)
return found
def _recorded_runs(
db: Any,
image_paths: Iterable[str],
output_root: Optional[str],
path_style: Optional[str],
) -> Dict[str, tuple]:
"""``{path: (run_id, output node ids)}`` for each file history attributes.
A file is attributed when exactly one recorded run reported its location
and it was last written while that run executed.
"""
paths = list(dict.fromkeys(path for path in image_paths if path))
if not paths or not output_root:
return {}
style = path_style or ("windows" if os.name == "nt" else "posix")
located = {}
for path in paths:
location = _location(path, output_root, style)
if location is not None:
located[path] = location
if not located:
return {}
names = sorted({filename for _key, _sub, filename in located.values()})
runs = _runs_by_location(db, names)
if not runs:
return {}
attributed = {}
for path, (root_key, subfolder, filename) in located.items():
reported = runs.get((filename, subfolder, root_key, style), {})
if len(reported) != 1:
continue
((run_id, (began, ended, nodes)),) = reported.items()
if _written_during(path, began, ended):
attributed[path] = (run_id, nodes)
return attributed
def _named_prompt(db: Any, run_id: str) -> Optional[int]:
contexts = db.get_execution_contexts(run_id).get("contexts") or []
named = {
@@ -183,35 +227,115 @@ def stored_run_prompts(
"""
paths = list(dict.fromkeys(path for path in image_paths if path))
result: Dict[str, Optional[int]] = dict.fromkeys(paths)
if not paths or not output_root:
return result
style = path_style or ("windows" if os.name == "nt" else "posix")
located = {}
for path in paths:
location = _location(path, output_root, style)
if location is not None:
located[path] = location
if not located:
return result
names = sorted({filename for _key, _sub, filename in located.values()})
runs = _runs_by_location(db, names)
if not runs:
return result
prompts: Dict[str, Optional[int]] = {}
for path, (root_key, subfolder, filename) in located.items():
reported = runs.get((filename, subfolder, root_key, style), {})
if len(reported) != 1:
continue
((run_id, (began, ended)),) = reported.items()
if not _written_during(path, began, ended):
continue
for path, (run_id, _nodes) in _recorded_runs(
db, paths, output_root, path_style
).items():
if run_id not in prompts:
prompts[run_id] = _named_prompt(db, run_id)
result[path] = prompts[run_id]
return result
@dataclass(frozen=True)
class RecordedOutput:
"""What recorded history knows about the run that wrote one file (#386).
``saver_id`` is the one output node the run reported the file from (None
when several did). ``texts`` maps each sampler anchor of the run to the
effective texts PromptManager recorded at execution for its roles
(``{"positive": text or None, "negative": text or None}``); a role whose
text was only inferred from the graph is None.
"""
saver_id: Optional[str]
texts: Dict[str, Dict[str, Optional[str]]]
def _recorded_texts(db: Any, run_id: str) -> Dict[str, Dict[str, Optional[str]]]:
texts = {}
for context in db.get_execution_contexts(run_id).get("contexts") or []:
evidence = context.get("evidence") or {}
roles = {}
for role in ("positive", "negative"):
values = context.get(role) or {}
proof = (evidence.get(role) or {}).get("effective") or {}
value = values.get("effective")
recorded = proof.get("provenance") == "recorded"
roles[role] = value if recorded and isinstance(value, str) else None
texts[context["anchor_id"]] = roles
return texts
def recorded_output(
db: Any,
image_path: str,
output_root: Optional[str],
path_style: Optional[str] = None,
) -> Optional[RecordedOutput]:
"""The recorded run facts for ``image_path``, else None (#386).
The file must be attributed to one run (``_recorded_runs``: one run
reported its location and it was written during that run). Read-only; it
never links anything.
"""
if not image_path:
return None
conn = db.model.get_connection()
own_transaction = not conn.in_transaction
if own_transaction: # outputs and texts from one state of history
conn.execute("BEGIN")
try:
found = _recorded_runs(db, [image_path], output_root, path_style)
if image_path not in found:
return None
run_id, nodes = found[image_path]
saver = next(iter(nodes)) if len(nodes) == 1 else None
return RecordedOutput(saver, _recorded_texts(db, run_id))
finally:
if own_transaction:
conn.rollback()
_LOGGER = logging.getLogger("prompt_manager.run_attribution")
def recorded_output_lookup(
db: Any, *, close: bool = False, reference: Optional[dict] = None
) -> Callable[[Any], Any]:
"""``lookup(path) -> RecordedOutput or None`` for the inspect routes (#386).
Uses ComfyUI's output folder. Any failure (no history, a locked library)
is logged and names no node, so the file is judged from its own metadata.
``close`` closes the calling thread's connection afterwards, for routes
that close theirs after reading. ``reference`` (a history occurrence's
output reference) spells the file as history recorded it: the route's
target is case-normalized on Windows, and history matches names exactly.
"""
def lookup(path: Any) -> Optional[RecordedOutput]:
root = host_output_directory()
if reference is not None:
if reference.get("folder_type") != "output" or not root:
return None
path = os.path.join(
root, reference.get("subfolder") or "", reference.get("filename") or ""
)
try:
return recorded_output(db, str(path), root)
except (sqlite3.Error, OSError, ValueError):
_LOGGER.warning("Recorded run lookup failed", exc_info=True)
return None
except Exception:
_LOGGER.exception("Recorded run lookup failed unexpectedly")
return None
finally:
if close:
db.model.close()
return lookup
_SEALED_LISTENERS: List[weakref.WeakMethod] = []
_SEALED_LOCK = threading.Lock()