nodes/ComfyAssets-ComfyUI_PromptManager
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into fix/376-exit-cancel

Browse Source 
# Conflicts:
#	utils/parallel.py
This commit is contained in:
Vito Sansevero
2026-10-28 02:24:48 -07:00
parent a8ab71eb1d 65b15bbe75
commit 8f889f92bd
61 changed files with 3230 additions and 857 deletions
Download Patch File Download Diff File Expand all files Collapse all files
utils/image_monitor.py
+13 -3
View File
@@ -35,6 +35,7 @@ from watchdog.events import FileSystemEventHandler
from .hashing import generate_prompt_hash
from .metadata_extractor import ComfyUIMetadataExtractor
from .parallel import note_refusal, refused_at_shutdown
from .logging_config import get_logger
from .prompt_graph import resolve_text, run_prompt_nodes
from .prompt_pair_resolver import resolve_prompt_pair
@@ -194,8 +195,9 @@ class ImageGenerationHandler(FileSystemEventHandler):
and once its size has stopped changing.
Returns:
False when the path is already pending, the pending set is full or
the handler is closed
False when the path is already pending, the pending set is full,
the handler is closed or the interpreter is exiting (its worker
cannot start)
"""
with self._pending_lock:
if self._closed:
@@ -211,7 +213,15 @@ class ImageGenerationHandler(FileSystemEventHandler):
return False
due = time.monotonic() + self.processing_delay
self._pending[image_path] = (prompt_snapshot, due)
self._ensure_worker()
try:
self._ensure_worker()
except RuntimeError as error:
# No worker will ever take it: never leave it pending (#375).
self._pending.pop(image_path, None)
if not refused_at_shutdown(error):
raise
note_refusal(self.logger, "The image worker")
return False
self._work.put(image_path)
return True
utils/logging_config.py
+239 -53
View File
@@ -39,6 +39,35 @@ class RotatedChanged(Exception):
self.name = name
# Clear all logs (#389): why it changed nothing, as the API and the pages say it.
CLEAR_CHANGED = (
"The log files changed since you confirmed; nothing was cleared."
" Read the list again and clear again."
)
CLEAR_FOREIGN_WRITER = (
"Another program is writing to prompt_manager.log, such as a second ComfyUI"
" sharing this log folder; nothing was cleared. Stop it and clear again."
)
CLEAR_HARD_LINK = (
"prompt_manager.log has another name (a hard link); emptying it would empty"
" that file too, so nothing was cleared. Remove the other name, then clear"
" again."
)
# A confirmed row's identity: "<size>:<mtime_ns>:<ino>", a string because
# mtime_ns and many inodes are larger than a browser's exact integers.
# mtime_ns is negative for a time before 1970 (a restored archive, say).
_IDENTITY = re.compile(r"([0-9]{1,20}):(-?[0-9]{1,20}):([0-9]{1,40})")
MAX_CONFIRMED_ROWS = 10000
class ClearRefused(Exception):
"""Clear all logs changed nothing; ``code`` names why, the message says it."""
def __init__(self, code: str, message: str):
super().__init__(message)
self.code = code
def _name_key(name: str) -> str:
"""How a name compares: case-folded where the filesystem folds it (Windows)."""
return os.path.normcase(name)
@@ -87,35 +116,37 @@ def scan_log_folder(log_dir) -> List[Dict[str, Any]]:
be read is left out. Never logs: the cleanup calls it while it holds the
file handler's lock.
"""
rows = []
try:
names = [entry.name for entry in os.scandir(log_dir)]
except OSError:
return rows
for name in names:
path = os.path.join(log_dir, name)
try:
st = os.lstat(path)
kind = _kind(name, path, st)
except OSError:
continue
try:
modified = datetime.fromtimestamp(st.st_mtime).isoformat()
except (OSError, OverflowError, ValueError): # a time before 1970 on Windows
modified = ""
rows.append(
{
"name": name,
"kind": kind,
"size": st.st_size,
"mtime_ns": st.st_mtime_ns,
"ino": st.st_ino,
"modified": modified,
"_dir": stat.S_ISDIR(st.st_mode) and kind != "link",
"_regular": stat.S_ISREG(st.st_mode) and kind != "link",
}
)
return rows
return []
rows = (scan_log_entry(log_dir, name) for name in names)
return [row for row in rows if row is not None]
def scan_log_entry(log_dir, name) -> Optional[Dict[str, Any]]:
"""One entry's scan row (``lstat``), or None when it cannot be read."""
path = os.path.join(log_dir, name)
try:
st = os.lstat(path)
kind = _kind(name, path, st)
except OSError:
return None
try:
modified = datetime.fromtimestamp(st.st_mtime).isoformat()
except (OSError, OverflowError, ValueError): # a time before 1970 on Windows
modified = ""
return {
"name": name,
"kind": kind,
"size": st.st_size,
"mtime_ns": st.st_mtime_ns,
"ino": st.st_ino,
"modified": modified,
"_dir": stat.S_ISDIR(st.st_mode) and kind != "link",
"_regular": stat.S_ISREG(st.st_mode) and kind != "link",
"_nlink": st.st_nlink,
}
def public_row(row: Dict[str, Any]) -> Dict[str, Any]:
@@ -127,6 +158,37 @@ def rotated_sort_key(row: Dict[str, Any]) -> int:
return _rotated_number(row["name"]) or 0
def file_identity(row: Dict[str, Any]) -> str:
"""A listed file's identity as the client sends it back: size, mtime_ns, ino."""
return f"{row['size']}:{row['mtime_ns']}:{row['ino']}"
def parse_confirmed(expected) -> List[Dict[str, Any]]:
"""The rows a Clear all logs confirmation showed, as review rows.
``expected`` is a list of ``{name, identity}`` objects from
``/logs/files``; anything else raises ValueError.
"""
if not isinstance(expected, list) or len(expected) > MAX_CONFIRMED_ROWS:
raise ValueError("expected must be a list of listed log files")
rows, seen = [], set()
for item in expected:
if not isinstance(item, dict):
raise ValueError("each expected file must be an object")
name, identity = item.get("name"), item.get("identity")
if not isinstance(name, str) or not isinstance(identity, str):
raise ValueError("each expected file needs a name and an identity")
match = _IDENTITY.fullmatch(identity)
if match is None:
raise ValueError("an expected file's identity is not size:mtime_ns:ino")
size, mtime_ns, ino = (int(part) for part in match.groups())
if _name_key(name) in seen:
raise ValueError("an expected file is listed twice")
seen.add(_name_key(name))
rows.append({"name": name, "size": size, "mtime_ns": mtime_ns, "ino": ino})
return rows
def log_file_entry(log_dir, name) -> Path:
"""The path of a downloadable log file named ``name`` (blocking).
@@ -157,6 +219,11 @@ def _stream_is(stream, ino: int) -> bool:
return not (ino and open_ino and open_ino != ino)
def _stream_ino_is(opened: os.stat_result, ino: int) -> bool:
"""Whether a stream's ``fstat`` is the file with inode ``ino`` (0: unknown)."""
return not (ino and opened.st_ino and opened.st_ino != ino)
def _error_text(error: BaseException) -> str:
"""An OSError's reason and file name, without the folder it is in."""
if isinstance(error, OSError):
@@ -432,6 +499,7 @@ class PromptManagerLogger:
"modified": row["modified"],
"is_main": row["kind"] == "active",
"kind": row["kind"],
"identity": file_identity(row),
}
)
@@ -530,8 +598,14 @@ class PromptManagerLogger:
finally:
handler.release()
if not handled:
flags = os.O_WRONLY | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0)
os.close(os.open(str(log_file), flags))
flags = os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0)
fd = os.open(str(log_file), flags)
try:
if os.fstat(fd).st_nlink > 1: # emptying it would empty the other
raise ClearRefused("logs-hard-link", CLEAR_HARD_LINK)
os.ftruncate(fd, 0)
finally:
os.close(fd)
def delete_rotated(self, expected: List[Dict[str, Any]]) -> Dict[str, Any]:
"""Delete exactly the reviewed rotated copies, or none of them (blocking).
@@ -582,47 +656,126 @@ class PromptManagerLogger:
)
return {"removed": removed, "failed": failed}
def truncate_logs(self) -> Dict[str, Any]:
"""Empty the active log and delete its rotated copies (Rule R).
def _active_streams(self, active: Optional[Dict[str, Any]]):
"""``fstat`` and offset of every handler stream writing the active name."""
target = os.path.normcase(
os.path.realpath(os.path.join(self.log_dir, ACTIVE_LOG_NAME))
)
for handler in self._file_handlers():
stream = getattr(handler, "stream", None)
base = getattr(handler, "baseFilename", None)
if stream is None or base is None:
continue
if os.path.normcase(os.path.realpath(base)) != target:
continue
try:
stream.flush()
fd = stream.fileno()
yield os.fstat(fd), os.lseek(fd, 0, os.SEEK_CUR)
except (OSError, ValueError, AttributeError):
continue
Every other file in the log folder, and every link, is kept. Also
clears the memory buffer. Runs under ``_files_lock`` and the file
handlers' locks, so no rollover recreates a just-deleted copy, and
logs only once they are released.
def _check_clear_is_safe(self, rows, active) -> None:
"""Refuse a clear that would empty or delete what nobody confirmed.
Called under the files lock and every handler lock, so nothing of this
process writes or rolls over meanwhile; what changes is another
program's doing. Our handler's stream is flushed: a file larger than
the stream's own offset was appended to by someone else, and a stream
whose file is now a rotated copy was rolled over by someone else.
Either is a second writer (another ComfyUI sharing the folder). This
is best effort, not a lock: writes before our own last write, or any
write while this process has file logging off, go unseen. An
active log with more than one name (a hard link) is never emptied: the
other name would be emptied too.
"""
rotated = {row["ino"] for row in rows if row["kind"] == "rotated"}
rotated.discard(0)
nlink = active["_nlink"] if active is not None else 1
for opened, offset in self._active_streams(active):
if (opened.st_ino and opened.st_ino in rotated) or opened.st_size > offset:
raise ClearRefused("logs-foreign-writer", CLEAR_FOREIGN_WRITER)
if active is not None and _stream_ino_is(opened, active["ino"]):
nlink = max(nlink, opened.st_nlink)
if nlink > 1:
raise ClearRefused("logs-hard-link", CLEAR_HARD_LINK)
@staticmethod
def _confirmed_targets(confirmed, current, active) -> List[Dict[str, Any]]:
"""The rotated copies a confirmation listed, each as it was listed.
Refuses (``logs-changed``) when any listed copy is missing or differs
in kind, size, ``mtime_ns`` or inode, or when the listed active log is
now another file (it was rolled over). A copy nobody listed is kept.
"""
targets = []
for reviewed in confirmed:
key = _name_key(reviewed["name"])
row = current.get(key)
if key == _name_key(ACTIVE_LOG_NAME):
ino = reviewed["ino"]
if active is None or (ino and active["ino"] and active["ino"] != ino):
raise ClearRefused("logs-changed", CLEAR_CHANGED)
continue
if _rotated_number(reviewed["name"]) is None:
continue # never deleted, listed or not
if not _matches_review(row, reviewed):
raise ClearRefused("logs-changed", CLEAR_CHANGED)
targets.append(row)
return targets
def truncate_logs(self, expected=None) -> Dict[str, Any]:
"""Empty the active log and delete its rotated copies (Rule R, #389).
``expected`` is what the confirmation showed: ``{name, identity}``
rows of ``/logs/files`` (``parse_confirmed``; ValueError when
malformed). Then only the listed rotated copies are deleted, and only
when each still has the listed size, ``mtime_ns`` and inode and the
active log is still the listed file; otherwise ClearRefused
(``logs-changed``) with nothing changed. Without it (the legacy admin
page, whose confirmation lists nothing) every rotated copy present now
is deleted.
Either way, a second writer of the active log (``logs-foreign-writer``)
or an active log with a hard link (``logs-hard-link``) refuses the
whole clear first. Each copy's identity is checked again just before
its unlink; one that changed (another process rolling over meanwhile)
stops the clear there, keeping it, the rest and the active log, which
is emptied last. Every other file in the log folder, and every link,
is kept. Also clears the memory buffer. Runs under ``_files_lock`` and
the file handlers' locks and logs only once they are released.
Returns:
Dictionary with:
- truncated: List of successfully truncated files
- errors: List of error messages for failed operations
"""
confirmed = None if expected is None else parse_confirmed(expected)
results = {"truncated": [], "errors": []}
refused = None
with self._files_lock:
held = self._acquire_file_handlers()
try:
rows = scan_log_folder(self.log_dir)
active = [row for row in rows if row["kind"] == "active"]
rotated = sorted(
(row for row in rows if row["kind"] == "rotated"),
key=rotated_sort_key,
)
for row in active + rotated:
path = os.path.join(self.log_dir, row["name"])
try:
if row["kind"] == "active":
self._truncate_main_log(Path(path), row["ino"])
else:
os.unlink(path)
results["truncated"].append(row["name"])
except OSError as error:
results["errors"].append(
f"Failed to truncate {row['name']}: {_error_text(error)}"
)
with self._buffer_lock:
self._log_buffer.clear()
current = {_name_key(row["name"]): row for row in rows}
active = next((row for row in rows if row["kind"] == "active"), None)
try:
if confirmed is None:
targets = [row for row in rows if row["kind"] == "rotated"]
else:
targets = self._confirmed_targets(confirmed, current, active)
self._check_clear_is_safe(rows, active)
except ClearRefused as error:
refused = error
else:
self._clear(sorted(targets, key=rotated_sort_key), active, results)
finally:
self._release_handlers(held)
if refused is not None:
self.logger.warning("Clear all logs refused (%s)", refused.code)
raise refused
if results["truncated"]:
self.logger.info("Truncated log files: %s", ", ".join(results["truncated"]))
for error_msg in results["errors"]:
@@ -630,6 +783,39 @@ class PromptManagerLogger:
return results
def _clear(self, targets, active, results) -> None:
"""Delete ``targets`` (each re-checked) then empty ``active`` (locks held)."""
for reviewed in targets:
name = reviewed["name"]
if not _matches_review(scan_log_entry(self.log_dir, name), reviewed):
results["errors"].append(
f"{name} changed while clearing (another program may be writing"
" to this log folder); it, the copies after it and the active"
" log were kept"
)
return
try:
os.unlink(os.path.join(self.log_dir, name))
results["truncated"].append(name)
except OSError as error:
results["errors"].append(
f"Failed to truncate {name}: {_error_text(error)}"
)
if active is not None:
try:
self._truncate_main_log(
Path(self.log_dir, active["name"]), active["ino"]
)
results["truncated"].append(active["name"])
except ClearRefused as error:
results["errors"].append(str(error))
except OSError as error:
results["errors"].append(
f"Failed to truncate {active['name']}: {_error_text(error)}"
)
with self._buffer_lock:
self._log_buffer.clear()
def update_config(self, new_config: Dict[str, Any]):
"""Update logging configuration.
utils/parallel.py
+69 -3
View File
@@ -5,20 +5,55 @@ a CUDA-initialised, multi-threaded process is unsafe and the spawn start
method would re-import ComfyUI's main module in every child. Pillow releases
the GIL while decoding, resizing and encoding, so threads still scale close
to linearly for both jobs.
Interpreter exit (#375): ``threading._shutdown`` shuts every thread pool
(``submit`` then raises "cannot schedule new futures after interpreter
shutdown") before it joins the non-daemon threads and long before the package
teardown runs, while our daemon threads keep working. Measured on 3.12 and
3.14, ``Thread.start`` still succeeds in that window; it is refused ("can't
create new thread at interpreter shutdown") only once the interpreter is
marked finalizing, after the ``atexit`` callbacks. ``refused_at_shutdown``
recognises both; ``note_refusal`` logs one DEBUG line per place, with no
traceback.
"""
import logging
import threading
from concurrent.futures import ThreadPoolExecutor
from .exit_signal import checked
SHUTDOWN_REFUSAL = "interpreter shutdown"
_logger = logging.getLogger("prompt_manager.parallel")
_noted = set()
_noted_lock = threading.Lock()
def refused_at_shutdown(error):
"""Whether ``error`` is a thread or pool start refused at interpreter exit."""
return isinstance(error, RuntimeError) and SHUTDOWN_REFUSAL in str(error)
def note_refusal(logger, what):
"""Log once per ``what``, at DEBUG and without a traceback, that ``what``
was refused because the interpreter is exiting."""
key = (logger.name, what)
with _noted_lock:
if key in _noted:
return
_noted.add(key)
logger.debug(f"{what}: refused, the interpreter is exiting")
def map_parallel(func, items, workers):
"""Apply ``func`` to every item on up to ``workers`` threads, keeping order.
``workers`` of 1 (or fewer) runs on the calling thread with no pool. An
exception in ``func`` propagates to the caller, so callers that must
survive bad input wrap ``func`` themselves. An item that has not started
when the interpreter starts exiting raises ExitRequested instead (#376).
survive bad input wrap ``func`` themselves. A pool refused at interpreter
exit runs the items it did not start on the calling thread (#375). An item
that has not started when the interpreter starts exiting raises
ExitRequested instead (#376).
"""
func = checked(func)
items = list(items)
@@ -27,4 +62,35 @@ def map_parallel(func, items, workers):
return [func(item) for item in items]
max_workers = min(workers, len(items))
with ThreadPoolExecutor(max_workers, thread_name_prefix="pm-worker") as pool:
return list(pool.map(func, items))
futures = []
try:
for item in items:
futures.append(pool.submit(func, item))
except RuntimeError as error:
if not refused_at_shutdown(error):
raise
note_refusal(_logger, "A worker pool")
# Drop what was queued but not started; it runs here instead.
# (A refused Thread.start leaves its item queued without a future
# here: it is cancelled too, unless a busy pool thread took it in
# between, the one case where an item can run twice.)
pool.shutdown(wait=True, cancel_futures=True)
else:
return [future.result() for future in futures]
return _run_rest(func, items, futures)
def _run_rest(func, items, futures):
"""The pool's finished results, and every item it did not run, run here.
After ``shutdown(cancel_futures=True)`` each submitted future either ran
(its result or exception is kept) or was cancelled before it started.
"""
results = []
for index, item in enumerate(items):
future = futures[index] if index < len(futures) else None
if future is not None and not future.cancelled():
results.append(future.result())
else:
results.append(func(item))
return results
utils/review_capture.py
+176 -31
View File
@@ -40,6 +40,19 @@ while it is pending and undecided (never beside a decision, see
profile the item recorded (``captured_review(..., item_profile(evidence))``).
The inspection profile only ever produces review choices; ingestion's own
links stay on the default profile, which never loads the table (D1).
The saver anchor (#390): the inspection profile resolves a file with its
name, as the workspace inspector does since #383, so the one saver whose
``filename_prefix`` names the file anchors its branch. When that anchor alone
resolves a file the default profile left ambiguous and nothing links
(``saver_anchored``), the scan queues it with the anchor's pair as its one
candidate (evidence ``anchor: "saver"``), never linking it: a file never
falls through both the link and the queue (maintainer ruling). No match,
several matches or an unfinished walk anchor nothing and the file is queued
as before. Apply re-resolves with the file's name, so only the anchored
branch can be applied. ``REVIEW_RULE_VERSION`` is folded into inspection
item versions, so inspection items recorded before the anchor are recorded
again by the next scan.
"""
from __future__ import annotations
@@ -104,10 +117,16 @@ MAX_CANDIDATE_DEPTH = 8
# The resolver profile an item's evidence was built with (``profile``).
DEFAULT = "default"
INSPECTION = "inspection"
# The ``anchor`` an item recorded from the saver anchor names (#390).
SAVER_ANCHOR = "saver"
ANCHOR_REASON = "The saver that wrote this file names its branch"
# Inspection item versions: above every plausible ``RESOLVER_VERSION`` and,
# with 40 bits of the folded digest, below 2**53 (exact as a JSON number).
INSPECTION_VERSION_BASE = 1 << 40
_INSPECTION_VERSION_HEX = 10
# How the scan and Apply use the inspection profile, folded into inspection
# item versions. 1: they resolve with the file's name (the saver anchor, #390).
REVIEW_RULE_VERSION = 1
# Above this many distinct node classes the whole table is the fingerprint.
@@ -154,38 +173,92 @@ def inspection_review_version(metadata: Any = None) -> int | None:
"""The ``resolver_version`` of an item recorded from the inspection profile.
Folds the profile with its budgets, ``RESOLVER_VERSION``,
``INSPECTION_RESOLVER_VERSION`` and ``semantics_version(metadata)``, so
a change of either resolver, of the profile's bounds or of the table
about the graph's classes moves it. None when the table cannot be loaded.
``INSPECTION_RESOLVER_VERSION``, ``REVIEW_RULE_VERSION`` and
``semantics_version(metadata)``, so a change of either resolver, of how
review uses the profile, of the profile's bounds or of the table about
the graph's classes moves it. None when the table cannot be loaded.
"""
table = semantics_version(metadata)
if table is None:
return None
profile = repr(astuple(INSPECTION_PROFILE))
seed = f"{profile}:{RESOLVER_VERSION}:{INSPECTION_RESOLVER_VERSION}:{table}"
if REVIEW_RULE_VERSION: # 0 is the seed items had before the rule existed
seed += f":review-rule-{REVIEW_RULE_VERSION}"
folded = hashlib.sha256(seed.encode("utf-8")).hexdigest()
return INSPECTION_VERSION_BASE + int(folded[:_INSPECTION_VERSION_HEX], 16)
def resolve_inspection(metadata: dict) -> dict:
"""``metadata`` resolved with the inspection profile (bounded work)."""
return resolve_prompt_pair(metadata, profile=INSPECTION_PROFILE)
def resolve_inspection(metadata: dict, file_name: str | None = None) -> dict:
"""``metadata`` resolved with the inspection profile (bounded work).
``file_name`` (the file's last path part) anchors the file on the one
saver that wrote it, as in the workspace inspector (#390).
"""
return resolve_prompt_pair(
metadata, profile=INSPECTION_PROFILE, file_name=file_name
)
def inspection_choice(metadata: dict) -> dict | None:
"""The inspection profile's ambiguous result when it offers a choice.
def inspection_choice(metadata: dict, file_name: str | None = None) -> dict | None:
"""The inspection profile's result when the file is a review case.
The scan asks only for a file the default profile found ambiguous
without a choice that nothing links.
without a choice that nothing links. The result is ambiguous with a
choice, or ``resolved`` when ``file_name`` anchors the file on its saver
and only that anchor resolves it (``saver_anchored``): the item then
offers the anchor's pair. None otherwise.
"""
if inspection_review_version(metadata) is None:
return None
resolution = resolve_inspection(metadata)
resolution = resolve_inspection(metadata, file_name)
if resolution.get("state") == "resolved":
return _anchor_of(metadata, file_name, resolution)
if resolution.get("state") != "ambiguous" or not reviewable(resolution):
return None
return resolution
def _anchor_of(metadata: Any, file_name: str | None, named: dict) -> dict | None:
"""``named`` tagged ``anchor: "saver"`` when only the saver anchor
resolves the file, to a pair with a positive text to choose; else None.
``named`` is ``metadata`` resolved with ``file_name``; the profile runs
once more without the name only when it is resolved and choosable.
"""
if not file_name or named.get("state") != "resolved":
return None
try:
offered = choosable(_projection(named))
except (KeyError, TypeError):
return None
if not offered or resolve_inspection(metadata).get("state") == "resolved":
return None
return {**named, "anchor": SAVER_ANCHOR}
def saver_anchored(metadata: Any, file_name: str | None) -> bool:
"""True when the saver that wrote ``file_name`` resolves its branch (#390).
The inspection profile resolves ``metadata`` with the file's name, and
without it the file does not resolve: the anchor alone decides it. A file
the default profile left ambiguous is then queued with the anchor's pair
as its one candidate. No name, an unusable table, no matching saver,
several, or an unfinished walk anchor nothing (False). The profile
without the name runs only when the named run resolved.
"""
return _anchored(metadata, file_name) is not None
def _anchored(metadata: Any, file_name: str | None) -> dict | None:
"""The anchored result ``saver_anchored`` describes, else None."""
if not file_name or not isinstance(metadata, dict):
return None
if inspection_review_version(metadata) is None:
return None
return _anchor_of(metadata, file_name, resolve_inspection(metadata, file_name))
def item_profile(evidence: Any) -> str:
"""The profile an item's stored evidence was built with."""
if isinstance(evidence, dict) and evidence.get("profile") == INSPECTION:
@@ -246,7 +319,9 @@ def record_ambiguous_file(
"""
versions = item_versions(scanned[0] if scanned is not None else None)
spelled_path, queued = _queued_spelling(db, root, relative, versions)
if queued is not None:
if queued is not None and not (
inspection and _default_item_to_anchor(db, queued, scanned, relative)
):
return _already_queued(db, *queued, versions)
epoch = db.get_execution_epoch()
try:
@@ -272,24 +347,24 @@ def record_ambiguous_file(
if resolution is None or resolution.get("state") != "ambiguous":
return "skipped"
profile, version = DEFAULT, RESOLVER_VERSION
if not reviewable(resolution):
version = inspection_review_version(captured[0]) if inspection else None
if version is None:
named = _captured_inspection(captured[0], relative, scanned) if inspection else None
anchored = named is not None and named[0].get("anchor") == SAVER_ANCHOR
if anchored:
# Only the saver anchor resolves it (#390): offer that one pair.
profile, version = INSPECTION, named[1]
resolution = {
"state": "ambiguous",
"reason": ANCHOR_REASON,
"candidates": [named[0]],
}
elif not reviewable(resolution):
if named is None:
return "untraced"
profile = INSPECTION
if (
scanned is not None
and scanned[1] is not None
and _same_records(captured[0], scanned[0])
):
resolution = scanned[1] # the same records: no second run
else:
# Heavier, so only for a file ambiguous with nothing to choose.
resolution = resolve_inspection(captured[0])
profile, (resolution, version) = INSPECTION, named
if resolution.get("state") != "ambiguous" or not reviewable(resolution):
return "untraced"
evidence, count, limited = review_evidence(
resolution, profile, version, captured[0]
resolution, profile, version, captured[0], anchored=anchored
)
try:
inserted = db.record_review_item(
@@ -313,6 +388,60 @@ def record_ambiguous_file(
return "limited" if limited else "recorded"
def _default_item_to_anchor(db, queued: tuple, scanned, relative: str) -> bool:
"""True when the bytes' only pending item is a default item recorded
before #390 and the saver anchor alone resolves the scanned records.
The file is then captured again so the anchored item supersedes that
pending, undecided item. Decided bytes and bytes already queued under
the current inspection version are never re-read.
"""
if scanned is None or scanned[0] is None:
return False
version = inspection_review_version(scanned[0])
if version is None or db.review_pending_at(*queued, version):
return False
if not db.review_pending_at(*queued, RESOLVER_VERSION):
return False
return _anchored(scanned[0], _file_name(relative)) is not None
def _captured_inspection(metadata: dict, relative: str, scanned) -> tuple | None:
"""``(resolution, item_version)`` of the captured records under the
inspection profile, anchored on the file's name; None when the table is
unusable.
A result tagged ``anchor: "saver"`` means the saver anchor alone
resolves the file (#390); any other resolved result is returned as
``unresolved`` (no review case, as before). The scan's result is reused
when the capture holds the same records (``inspection_choice`` applies
the same rule).
"""
version = inspection_review_version(metadata)
if version is None:
return None
if (
scanned is not None
and scanned[1] is not None
and _same_records(metadata, scanned[0])
):
return scanned[1], version # the same records: no second run
name = _file_name(relative)
resolution = resolve_inspection(metadata, name)
if resolution.get("state") == "resolved":
# Resolved without the anchor too: no review case, as before.
resolution = _anchor_of(metadata, name, resolution) or {
**resolution,
"state": "unresolved",
}
return resolution, version
def _file_name(relative: str) -> str:
"""The last part of a ``/``-separated relative path (the saved name)."""
return relative.rsplit("/", 1)[-1]
def _already_queued(db, locator: str, size: int, mtime_ns: int, versions) -> str:
"""``duplicate`` while an existing item is pending, else ``decided``."""
for version in versions:
@@ -354,14 +483,19 @@ def _queued_spelling(db, root, relative: str, versions) -> tuple[str, tuple | No
return spelled, None
def captured_review(store, capture, profile: str = DEFAULT) -> tuple[dict, dict] | None:
def captured_review(
store, capture, profile: str = DEFAULT, file_name: str | None = None
) -> tuple[dict, dict] | None:
"""``(metadata, resolution)`` of the capture's own bytes, else None.
``metadata`` is what the scan hands the ingester: prompt and workflow
JSON-decoded and ``file_info`` (size, dimensions, format) from the
capture. ``resolution`` comes from ``profile`` (``DEFAULT`` or
``INSPECTION``). None when the capture is gone or carries no prompt
metadata.
``INSPECTION``; the latter anchored on ``file_name``, the file's own
name, as the item was recorded). With ``file_name`` (Apply), a default
result its saver anchor alone resolves (``saver_anchored``) is that
anchored result, so only the anchored branch is still offered. None
when the capture is gone or carries no prompt metadata.
"""
if store.acquire(capture.token) is None:
return None
@@ -389,8 +523,15 @@ def captured_review(store, capture, profile: str = DEFAULT) -> tuple[dict, dict]
info["format"] = read.format
canonical["file_info"] = info
if profile == INSPECTION:
return canonical, resolve_inspection(canonical)
return canonical, resolve_prompt_pair(canonical)
named = resolve_inspection(canonical, file_name)
return canonical, _anchor_of(canonical, file_name, named) or named
resolution = resolve_prompt_pair(canonical)
if file_name is not None and resolution.get("state") == "ambiguous":
anchored = _anchored(canonical, file_name)
if anchored is not None:
# Its saver names its branch (#390): only that one is offered.
return canonical, anchored
return canonical, resolution
def reviewable(resolution: Any) -> bool:
@@ -416,12 +557,14 @@ def review_evidence(
profile: str = DEFAULT,
version: int | None = None,
metadata: Any = None,
anchored: bool = False,
) -> tuple[dict, int, bool]:
"""``(evidence, candidate_count, limited)`` for one ambiguous result.
Inspection evidence also names its ``profile`` and ``semantics``, the
table fingerprint of ``metadata``'s graph it was resolved with;
``version`` is the item's version.
``version`` is the item's version. ``anchored`` evidence (the saver
anchor's one pair, #390) also names ``anchor: "saver"``.
"""
base = {
"resolver_version": RESOLVER_VERSION if version is None else version,
@@ -430,6 +573,8 @@ def review_evidence(
if profile == INSPECTION:
base["profile"] = INSPECTION
base["semantics"] = semantics_version(metadata)
if anchored:
base["anchor"] = SAVER_ANCHOR
candidates: list[dict] = []
seen: set[str] = set()
total = 0