* feat(review): offer inspection-profile choices for untraceable batches Qwen image-edit batches run several KSamplers, each with its own TextEncodeQwenImageEdit prompt, all saved under one prefix. The scan's default profile knows neither the encoder nor the string nodes, so every candidate was unresolved and the files left Needs review as "could not be traced", although the inspection profile resolves every text. When the default profile is ambiguous without a choice and nothing links the file, the scan now resolves it with the inspection profile and, with at least two choosable candidates, queues the item from that evidence. The evidence records profile "inspection" and the semantics table version (SHA-256 of its canonical JSON); the item's resolver_version folds the profile, RESOLVER_VERSION and the table version, so a table change is handled like a resolver change (I2). Apply re-resolves the capture with the recorded profile, re-checks the chosen candidate hash and links through the compatible-only ingestion. The inspection profile never links on its own, ingestion keeps the default profile, and the live monitor never queues. * feat(review): supersede stale items and inspect each file state once A semantics table or resolver version change no longer leaves two pending items for one file. A new item supersedes the other items of the same bytes when every one is pending and undecided (revision 0, no decision row, the self-heal's safe delete). Beside a decided or advanced item nothing new is stored, and the rescan's stat shortcut skips those bytes under any version. The inspection item version now folds a narrower fingerprint: the table restricted to the graph's node classes plus its class-independent parts, so an edit about other classes leaves every item alone. The inspection profile runs once per file state. The scan resolves the untraced file at batch level, skips files already queued or decided at their current size and mtime under their version, and remembers files it found untraced in this process. The capture reuses the scan's result when it holds the same records. * feat(review): contain inspection failures, version the inspection resolver Final-review fixes for the inspection-profile review choices. - An inspection failure on one file is logged, counted in review_errors and reported unresolved; the scan goes on. - INSPECTION_RESOLVER_VERSION and the inspection profile's budgets are folded into the item version, so an inspection-only resolver change re-inspects and supersedes the item. An older version's pending, undecided item of bytes now untraced leaves the queue, so an item never stays stuck at 409. - Fingerprint tests: an edit of the encoder's own entry moves the version, an unused dict-keyed class leaves it. - The untraced memory is a plain dict of 16-byte digests (8.6 MiB measured at its 65,536-entry cap). - Only scans add items: the monitor's pass records nothing and only clears the items of outputs it links. Tests and the browser review fixture record through the scan's recorder. - record_review_item documents the byte-record refresh it commits when it refuses a new item. * fix(review): an unusable semantics table never retires recorded items
649 lines
26 KiB
Python
649 lines
26 KiB
Python
"""Needs review items (#249): persist and read capture-bound ambiguous evidence.
|
|
|
|
An item is one ambiguous resolver result for one annotation byte record (the
|
|
whole-file SHA-256 of one physical file at one locator), keyed by
|
|
``(byte_id, resolver_version, evidence_hash)``. Recording is idempotent: the
|
|
same evidence for the same bytes is a duplicate and never touches an existing
|
|
item, its state, revision or decisions. Changed bytes are a new byte record
|
|
and therefore a new pending item; the old one stays.
|
|
|
|
Callers pass verified facts (the capture's digest, size and ``mtime_ns``);
|
|
nothing here reads the filesystem. Writes run in one ``_pair_transaction``
|
|
fenced by the storage epoch the capture was taken under.
|
|
"""
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
from typing import Iterator
|
|
|
|
try:
|
|
from ..utils.hashing import generate_prompt_hash
|
|
except ImportError:
|
|
from utils.hashing import generate_prompt_hash
|
|
|
|
from .annotation_schema import (
|
|
BYTES_TABLE,
|
|
FILES_TABLE,
|
|
METADATA_TABLE as ANNOTATION_METADATA_TABLE,
|
|
)
|
|
from .annotations import _new_id, _validate_locator, _validate_target
|
|
from .models import normalize_image_path, utc_now_iso
|
|
from .review_changes import notify_review_change
|
|
from .review_schema import (
|
|
DECISIONS_TABLE,
|
|
ITEMS_TABLE,
|
|
MAX_CANDIDATES,
|
|
MAX_SPELLED_PATH_BYTES,
|
|
METADATA_TABLE,
|
|
canonical_evidence,
|
|
evidence_digest,
|
|
evidence_offers_choice,
|
|
)
|
|
|
|
# Only this module's table constants are interpolated into the SQL below.
|
|
_INSERT_ITEM = (
|
|
f"INSERT OR IGNORE INTO {ITEMS_TABLE}(id,byte_id,spelled_path," # nosec B608
|
|
"resolver_version,evidence_hash,evidence,candidate_count,limited,state,"
|
|
"revision,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,'pending',0,?,?)"
|
|
)
|
|
_BUMP_REVIEW_INDEX = (
|
|
f"UPDATE {METADATA_TABLE} SET " # nosec B608
|
|
"review_index_revision=review_index_revision+1 WHERE singleton_id=1"
|
|
)
|
|
# A later spelling of the same locator (a case-only rename on a case-folding
|
|
# filesystem) replaces the recorded one, so targets keep formatting.
|
|
_REFRESH_SPELLING = (
|
|
f"UPDATE {ITEMS_TABLE} SET spelled_path=? " # nosec B608
|
|
f"WHERE spelled_path<>? AND byte_id IN (SELECT b.id FROM {BYTES_TABLE} b "
|
|
f"JOIN {FILES_TABLE} f ON f.id=b.file_id WHERE f.locator=?)"
|
|
)
|
|
_REVIEW_INDEX = (
|
|
f"SELECT review_index_revision FROM {METADATA_TABLE} " # nosec B608
|
|
"WHERE singleton_id=1"
|
|
)
|
|
_BUMP_IMAGE_INDEX = (
|
|
f"UPDATE {ANNOTATION_METADATA_TABLE} SET " # nosec B608
|
|
"image_index_revision=image_index_revision+1 WHERE singleton_id=1"
|
|
)
|
|
_ITEMS_OF_LOCATOR = (
|
|
"SELECT i.id,i.byte_id,i.resolver_version,i.evidence_hash," # nosec B608
|
|
"i.evidence,i.candidate_count,i.limited,i.state,i.revision,i.created_at,"
|
|
f"i.updated_at,b.sha256 FROM {FILES_TABLE} f "
|
|
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
|
f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
|
|
"WHERE f.locator=? ORDER BY i.created_at,i.id"
|
|
)
|
|
_QUEUED = (
|
|
f"SELECT 1 FROM {FILES_TABLE} f " # nosec B608
|
|
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
|
f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
|
|
"WHERE f.locator=? AND b.size=? AND b.mtime_ns=? AND i.resolver_version=? "
|
|
"LIMIT 1"
|
|
)
|
|
_PENDING_AT = (
|
|
f"SELECT 1 FROM {FILES_TABLE} f " # nosec B608
|
|
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
|
f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
|
|
"WHERE f.locator=? AND b.size=? AND b.mtime_ns=? AND i.resolver_version=? "
|
|
"AND i.state='pending' LIMIT 1"
|
|
)
|
|
# Bytes verified at this stat whose item carries a decision (any version).
|
|
_DECIDED_AT = (
|
|
f"SELECT 1 FROM {FILES_TABLE} f " # nosec B608
|
|
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
|
f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
|
|
"WHERE f.locator=? AND b.size=? AND b.mtime_ns=? "
|
|
"AND (i.state<>'pending' OR i.revision<>0 OR EXISTS (SELECT 1 FROM "
|
|
f"{DECISIONS_TABLE} d WHERE d.item_id=i.id)) LIMIT 1"
|
|
)
|
|
# The other items of one byte record (another version or evidence), with
|
|
# whether each may be superseded: pending, revision 0, no decision row.
|
|
_OTHER_ITEMS_OF_BYTES = (
|
|
"SELECT i.id, (i.state='pending' AND i.revision=0 AND NOT EXISTS " # nosec B608
|
|
f"(SELECT 1 FROM {DECISIONS_TABLE} d WHERE d.item_id=i.id)) "
|
|
f"FROM {ITEMS_TABLE} i WHERE i.byte_id=? "
|
|
"AND NOT (i.resolver_version=? AND i.evidence_hash=?)"
|
|
)
|
|
# Pending items of the bytes verified at one stat, with their versions.
|
|
_PENDING_AT_STAT = (
|
|
f"SELECT i.id,i.resolver_version FROM {FILES_TABLE} f " # nosec B608
|
|
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
|
f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
|
|
"WHERE f.locator=? AND b.size=? AND b.mtime_ns=? AND i.state='pending'"
|
|
)
|
|
# Pending items of one file: the candidates a link settles.
|
|
_PENDING_OF_LOCATOR = (
|
|
f"SELECT i.id FROM {FILES_TABLE} f " # nosec B608
|
|
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
|
f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
|
|
"WHERE f.locator=? AND i.state='pending'"
|
|
)
|
|
# Pending items of one file with their evidence: an untraced file's settle.
|
|
_PENDING_EVIDENCE_OF_LOCATOR = (
|
|
f"SELECT i.id,i.evidence FROM {FILES_TABLE} f " # nosec B608
|
|
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
|
f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
|
|
"WHERE f.locator=? AND i.state='pending'"
|
|
)
|
|
# Only an undecided item goes: a decision row (RESTRICT) is never deleted.
|
|
_SETTLE_ITEM = (
|
|
f"DELETE FROM {ITEMS_TABLE} WHERE id=? AND state='pending' " # nosec B608
|
|
f"AND revision=0 AND NOT EXISTS (SELECT 1 FROM {DECISIONS_TABLE} d "
|
|
f"WHERE d.item_id={ITEMS_TABLE}.id)"
|
|
)
|
|
_EPOCH = "SELECT storage_epoch FROM execution_history_metadata WHERE singleton_id=1"
|
|
_PENDING_COUNT = (
|
|
f"SELECT COUNT(*) FROM {ITEMS_TABLE} WHERE state='pending'" # nosec B608
|
|
)
|
|
_QUEUE_COLUMNS = (
|
|
"id",
|
|
"state",
|
|
"revision",
|
|
"created_at",
|
|
"candidate_count",
|
|
"limited",
|
|
"spelled_path",
|
|
"locator",
|
|
)
|
|
_QUEUE_SELECT = (
|
|
"SELECT i.id,i.state,i.revision,i.created_at,i.candidate_count," # nosec B608
|
|
f"i.limited,i.spelled_path,f.locator FROM {ITEMS_TABLE} i "
|
|
f"JOIN {BYTES_TABLE} b ON b.id=i.byte_id "
|
|
f"JOIN {FILES_TABLE} f ON f.id=b.file_id "
|
|
)
|
|
_QUEUE_FIRST = _QUEUE_SELECT + (
|
|
"WHERE i.state='pending' ORDER BY i.created_at,i.id LIMIT ?"
|
|
)
|
|
_QUEUE_AFTER = _QUEUE_SELECT + (
|
|
"WHERE i.state='pending' AND (i.created_at,i.id)>(?,?) "
|
|
"ORDER BY i.created_at,i.id LIMIT ?"
|
|
)
|
|
_DETAIL_COLUMNS = _QUEUE_COLUMNS + (
|
|
"byte_id",
|
|
"resolver_version",
|
|
"evidence_hash",
|
|
"evidence",
|
|
"updated_at",
|
|
"sha256",
|
|
)
|
|
_DETAIL = (
|
|
"SELECT i.id,i.state,i.revision,i.created_at,i.candidate_count," # nosec B608
|
|
"i.limited,i.spelled_path,f.locator,i.byte_id,i.resolver_version,"
|
|
"i.evidence_hash,i.evidence,i.updated_at,b.sha256 "
|
|
f"FROM {ITEMS_TABLE} i JOIN {BYTES_TABLE} b ON b.id=i.byte_id "
|
|
f"JOIN {FILES_TABLE} f ON f.id=b.file_id WHERE i.id=?"
|
|
)
|
|
# The legacy rows and pair links the ingester's compatibility rules read.
|
|
_DEPENDENT_IMAGES = (
|
|
"SELECT gi.id,p.hash,gi.image_path FROM generated_images gi "
|
|
"JOIN prompts p ON p.id=gi.prompt_id "
|
|
"WHERE gi.file_path=? OR gi.image_path=? ORDER BY gi.id"
|
|
)
|
|
_DEPENDENT_LINKS = (
|
|
"SELECT DISTINCT ip.pair_id,n.hash FROM image_prompt_pairs ip "
|
|
"JOIN prompt_pairs pp ON pp.id=ip.pair_id "
|
|
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id "
|
|
"WHERE ip.image_id=? ORDER BY ip.pair_id"
|
|
)
|
|
# Rows whose spelling ends with the file's name: the candidates for another
|
|
# spelling of the same physical file (the link summaries' filename fallback).
|
|
_SAME_NAME_ROWS = (
|
|
"SELECT id,image_path FROM generated_images "
|
|
"WHERE image_path LIKE ? ESCAPE '\\' ORDER BY id"
|
|
)
|
|
_TITLES = (
|
|
"SELECT p.hash,s.title FROM prompts p "
|
|
"JOIN prompt_library_state s ON s.prompt_id=p.id "
|
|
"WHERE p.hash IN ({marks}) AND s.title IS NOT NULL ORDER BY p.id"
|
|
)
|
|
_ITEM_ID = re.compile(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}")
|
|
_ITEM_COLUMNS = (
|
|
"id",
|
|
"byte_id",
|
|
"resolver_version",
|
|
"evidence_hash",
|
|
"evidence",
|
|
"candidate_count",
|
|
"limited",
|
|
"state",
|
|
"revision",
|
|
"created_at",
|
|
"updated_at",
|
|
"sha256",
|
|
)
|
|
|
|
|
|
def _validate_spelling(spelled_path: object, locator: str) -> None:
|
|
if (
|
|
not isinstance(spelled_path, str)
|
|
or not spelled_path
|
|
or len(spelled_path.encode("utf-8", "surrogatepass")) > MAX_SPELLED_PATH_BYTES
|
|
or os.path.normcase(spelled_path) != locator
|
|
):
|
|
raise ValueError("spelled_path must be the file's spelling of its locator")
|
|
|
|
|
|
def dependent_images(conn: sqlite3.Connection, spellings) -> list:
|
|
"""``(id, image_path, positive_hash)`` of every legacy row of the file.
|
|
|
|
``spellings`` are the paths the file is linked under (the Apply path and
|
|
every alias of the same physical file); each is matched as the ingester
|
|
matches one path. Rows are distinct and ordered by id.
|
|
"""
|
|
found = {}
|
|
for spelled in dict.fromkeys(spellings):
|
|
for row in conn.execute(
|
|
_DEPENDENT_IMAGES, (normalize_image_path(spelled), spelled)
|
|
):
|
|
found[row[0]] = (row[0], row[2], row[1])
|
|
return [found[key] for key in sorted(found)]
|
|
|
|
|
|
def same_name_rows(conn: sqlite3.Connection, name: str) -> list:
|
|
"""``(id, image_path)`` of rows whose spelling may end with ``name``."""
|
|
# Imported here: operations composes this module's mixin.
|
|
from .operations import escape_like
|
|
|
|
return [
|
|
(row[0], row[1])
|
|
for row in conn.execute(_SAME_NAME_ROWS, (f"%{escape_like(name)}",))
|
|
]
|
|
|
|
|
|
def dependency_fingerprint(
|
|
conn: sqlite3.Connection, image_path: str, aliases=()
|
|
) -> str:
|
|
"""SHA-256 of what Apply's compatibility depends on for ``image_path``.
|
|
|
|
The exact legacy rows of the file under ``image_path`` and every alias
|
|
spelling of it, with their positive hashes, and each row's pair links
|
|
with the negative hash, read as the ingester reads them. Unrelated prompt
|
|
changes (tags, titles) do not move it.
|
|
"""
|
|
rows = []
|
|
for image_id, _spelled, positive_hash in dependent_images(
|
|
conn, (image_path, *aliases)
|
|
):
|
|
links = conn.execute(_DEPENDENT_LINKS, (image_id,)).fetchall()
|
|
rows.append([image_id, positive_hash, [list(link) for link in links]])
|
|
text = canonical_evidence({"v": 1, "images": rows})
|
|
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
|
|
|
|
class ReviewOperations:
|
|
"""PromptDatabase mixin: insert and read Needs review items."""
|
|
|
|
def record_review_item(
|
|
self,
|
|
*,
|
|
locator: str,
|
|
spelled_path: str,
|
|
sha256: str,
|
|
size: int,
|
|
mtime_ns: int,
|
|
epoch: str,
|
|
resolver_version: int,
|
|
evidence: dict,
|
|
candidate_count: int,
|
|
limited: bool,
|
|
) -> bool:
|
|
"""Store one ambiguous result for these exact bytes; True when new.
|
|
|
|
The byte record is created (or its verified fingerprint refreshed)
|
|
like an observe. ``spelled_path`` is the file's own spelling (it must
|
|
fold to ``locator``); targets are formatted from it. Raises
|
|
``LibraryCommandError("storage-changed")`` when the storage epoch is
|
|
no longer ``epoch`` (a restore since the capture) and ``ValueError``
|
|
for invalid facts or evidence.
|
|
|
|
One file never holds two pending items for the same bytes: the new
|
|
item supersedes the byte record's other items (an older resolver or
|
|
semantics version) when every one is pending and undecided (the safe
|
|
delete of the self-heal); when one carries a decision or a later
|
|
revision, it is kept and nothing new is stored (False).
|
|
"""
|
|
_validate_target(locator, sha256, size, mtime_ns)
|
|
_validate_spelling(spelled_path, locator)
|
|
if type(resolver_version) is not int or resolver_version < 1:
|
|
raise ValueError("resolver_version must be a positive integer")
|
|
if (
|
|
type(candidate_count) is not int
|
|
or not 0 <= candidate_count <= MAX_CANDIDATES
|
|
):
|
|
raise ValueError(f"candidate_count must be between 0 and {MAX_CANDIDATES}")
|
|
if not isinstance(limited, bool) or (limited and candidate_count):
|
|
raise ValueError("A limited item carries no candidates")
|
|
text = canonical_evidence(evidence)
|
|
evidence_hash = evidence_digest(text)
|
|
with self._pair_transaction() as conn:
|
|
self._library_epoch(conn, epoch)
|
|
now = utc_now_iso()
|
|
file_id = self._annotation_owner(conn, locator, now)
|
|
byte_id, _revision, created, moved = self._annotation_byte(
|
|
conn, file_id, sha256, size, mtime_ns, now
|
|
)
|
|
if created or moved:
|
|
conn.execute(_BUMP_IMAGE_INDEX)
|
|
# The byte record is created or refreshed even when the item is
|
|
# refused below (an observe would do the same); that commits
|
|
# with the image index bump, and the review queue is unchanged.
|
|
others = conn.execute(
|
|
_OTHER_ITEMS_OF_BYTES, (byte_id, resolver_version, evidence_hash)
|
|
).fetchall()
|
|
if any(not removable for _id, removable in others):
|
|
return False
|
|
superseded = sum(
|
|
conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
|
|
for item_id, _removable in others
|
|
)
|
|
cursor = conn.execute(
|
|
_INSERT_ITEM,
|
|
(
|
|
_new_id(),
|
|
byte_id,
|
|
spelled_path,
|
|
resolver_version,
|
|
evidence_hash,
|
|
text,
|
|
candidate_count,
|
|
int(limited),
|
|
now,
|
|
now,
|
|
),
|
|
)
|
|
inserted = cursor.rowcount > 0
|
|
refreshed = _refresh_spelling(conn, locator, spelled_path)
|
|
if inserted or refreshed or superseded:
|
|
conn.execute(_BUMP_REVIEW_INDEX)
|
|
if inserted or refreshed or superseded:
|
|
notify_review_change(self.model.db_path)
|
|
return inserted
|
|
|
|
def refresh_review_spelling(self, locator: str, spelled_path: str) -> bool:
|
|
"""Record ``spelled_path`` as the spelling of every item at ``locator``.
|
|
|
|
The rescan's skip calls it for a file it does not copy again: a
|
|
case-only rename keeps size and ``mtime_ns``, and a target formatted
|
|
from the old spelling would be refused. True when a spelling changed
|
|
(the review index revision moves; item revisions do not, so an open
|
|
decision stays valid).
|
|
"""
|
|
_validate_locator(locator)
|
|
_validate_spelling(spelled_path, locator)
|
|
with self._pair_transaction() as conn:
|
|
refreshed = _refresh_spelling(conn, locator, spelled_path)
|
|
if refreshed:
|
|
conn.execute(_BUMP_REVIEW_INDEX)
|
|
if refreshed:
|
|
notify_review_change(self.model.db_path)
|
|
return refreshed
|
|
|
|
def review_index_revision(self) -> int:
|
|
"""The queue's revision (moved by every queue change)."""
|
|
row = self.model.get_connection().execute(_REVIEW_INDEX).fetchone()
|
|
if row is None:
|
|
raise sqlite3.DatabaseError("Missing review queue metadata")
|
|
return row[0]
|
|
|
|
def review_queue_version(self) -> dict:
|
|
"""``{library_epoch, review_revision}`` of the queue, from one snapshot.
|
|
|
|
The pair a list read answers: equal pairs mean the same queue (a
|
|
restore moves the epoch, every queue change moves the revision).
|
|
"""
|
|
with self._annotation_read() as conn:
|
|
epoch = conn.execute(_EPOCH).fetchone()
|
|
revision = conn.execute(_REVIEW_INDEX).fetchone()
|
|
if epoch is None or revision is None:
|
|
raise sqlite3.DatabaseError("Missing review queue metadata")
|
|
return {"library_epoch": epoch[0], "review_revision": revision[0]}
|
|
|
|
def review_items_for(self, locator: str) -> list:
|
|
"""Every item of the file at ``locator``, oldest first, evidence parsed."""
|
|
_validate_locator(locator)
|
|
return list(self._iter_review_items(locator))
|
|
|
|
def _iter_review_items(self, locator: str) -> Iterator[dict]:
|
|
conn = self.model.get_connection()
|
|
for row in conn.execute(_ITEMS_OF_LOCATOR, (locator,)):
|
|
item = dict(zip(_ITEM_COLUMNS, tuple(row)))
|
|
item["evidence"] = json.loads(item["evidence"])
|
|
item["limited"] = bool(item["limited"])
|
|
yield item
|
|
|
|
# -- queue reads (#249 E3) -------------------------------------------------
|
|
|
|
def review_fingerprint_queued(
|
|
self, locator: str, size: int, mtime_ns: int, resolver_version: int
|
|
) -> bool:
|
|
"""True when an item cites a byte record verified at this exact stat.
|
|
|
|
A rescan uses it to skip the copy and hash of a file it already
|
|
queued: the same ``(size, mtime_ns)`` the bytes were verified with.
|
|
"""
|
|
row = (
|
|
self.model.get_connection()
|
|
.execute(_QUEUED, (locator, size, mtime_ns, resolver_version))
|
|
.fetchone()
|
|
)
|
|
return row is not None
|
|
|
|
def review_decided_at(self, locator: str, size: int, mtime_ns: int) -> bool:
|
|
"""True when bytes verified at this stat carry a decided item.
|
|
|
|
Any version: a newer resolver or table never adds an item beside a
|
|
decision, so the rescan need not copy those bytes again.
|
|
"""
|
|
row = (
|
|
self.model.get_connection()
|
|
.execute(_DECIDED_AT, (locator, size, mtime_ns))
|
|
.fetchone()
|
|
)
|
|
return row is not None
|
|
|
|
def review_pending_at(
|
|
self, locator: str, size: int, mtime_ns: int, resolver_version: int
|
|
) -> bool:
|
|
"""True when a *pending* item cites bytes verified at this exact stat.
|
|
|
|
``review_fingerprint_queued`` also matches decided items; a file whose
|
|
only item is decided is not in the queue.
|
|
"""
|
|
row = (
|
|
self.model.get_connection()
|
|
.execute(_PENDING_AT, (locator, size, mtime_ns, resolver_version))
|
|
.fetchone()
|
|
)
|
|
return row is not None
|
|
|
|
def review_file_linked(self, spellings) -> bool:
|
|
"""True when a saved row links the file under any of ``spellings``."""
|
|
with self._annotation_read() as conn:
|
|
return bool(dependent_images(conn, spellings))
|
|
|
|
def settle_linked_review_items(self, files) -> int:
|
|
"""Remove the pending, undecided items of files that are now linked.
|
|
|
|
``files`` holds ``(locator, spellings)`` per file. An item leaves the
|
|
queue only when it is still ``pending`` at revision 0 with no decision
|
|
row and a saved row links its file under one of ``spellings``; a
|
|
decided item is never touched. Files without a pending item cost one
|
|
indexed read and open no write transaction. Returns how many items
|
|
were removed (the review index revision moves when any was).
|
|
"""
|
|
files = [(locator, tuple(spellings)) for locator, spellings in files]
|
|
for locator, _spellings in files:
|
|
_validate_locator(locator)
|
|
conn = self.model.get_connection()
|
|
candidates = [
|
|
(locator, spellings)
|
|
for locator, spellings in files
|
|
if conn.execute(_PENDING_OF_LOCATOR, (locator,)).fetchone() is not None
|
|
]
|
|
if not candidates:
|
|
return 0
|
|
removed = 0
|
|
with self._pair_transaction() as conn:
|
|
for locator, spellings in candidates:
|
|
if not dependent_images(conn, spellings):
|
|
continue
|
|
for (item_id,) in conn.execute(
|
|
_PENDING_OF_LOCATOR, (locator,)
|
|
).fetchall():
|
|
removed += conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
|
|
if removed:
|
|
conn.execute(_BUMP_REVIEW_INDEX)
|
|
if removed:
|
|
notify_review_change(self.model.db_path)
|
|
return removed
|
|
|
|
def settle_stale_review_items(
|
|
self, locator: str, size: int, mtime_ns: int, versions
|
|
) -> int:
|
|
"""Remove pending, undecided items of the bytes verified at this stat
|
|
whose ``resolver_version`` is not one of ``versions``.
|
|
|
|
The caller found these bytes untraced under the current versions, so
|
|
an older version's item could only refuse Apply. The same safe delete
|
|
as the other settles; one indexed read when nothing is pending.
|
|
"""
|
|
_validate_locator(locator)
|
|
current = set(versions)
|
|
conn = self.model.get_connection()
|
|
stale = [
|
|
item_id
|
|
for item_id, version in conn.execute(
|
|
_PENDING_AT_STAT, (locator, size, mtime_ns)
|
|
)
|
|
if version not in current
|
|
]
|
|
if not stale:
|
|
return 0
|
|
removed = 0
|
|
with self._pair_transaction() as conn:
|
|
for item_id in stale:
|
|
removed += conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
|
|
if removed:
|
|
conn.execute(_BUMP_REVIEW_INDEX)
|
|
if removed:
|
|
notify_review_change(self.model.db_path)
|
|
return removed
|
|
|
|
def settle_untraced_review_items(self, locators) -> int:
|
|
"""Remove the pending, undecided items that offer nothing to choose.
|
|
|
|
``locators`` are files the caller found ambiguous without a choice
|
|
(fewer than two candidates carry a prompt). Of their items, one leaves
|
|
the queue only when its own stored evidence offers no choice either
|
|
(``evidence_offers_choice``) and it is still ``pending`` at revision 0
|
|
with no decision row, the same safe delete as
|
|
``settle_linked_review_items``. Files without a pending item cost one
|
|
indexed read and open no write transaction. Returns how many items
|
|
were removed (the review index revision moves when any was).
|
|
"""
|
|
locators = list(locators)
|
|
for locator in locators:
|
|
_validate_locator(locator)
|
|
conn = self.model.get_connection()
|
|
pending = [
|
|
locator
|
|
for locator in locators
|
|
if conn.execute(_PENDING_OF_LOCATOR, (locator,)).fetchone() is not None
|
|
]
|
|
if not pending:
|
|
return 0
|
|
removed = 0
|
|
with self._pair_transaction() as conn:
|
|
for locator in pending:
|
|
rows = conn.execute(_PENDING_EVIDENCE_OF_LOCATOR, (locator,)).fetchall()
|
|
for item_id, evidence in rows:
|
|
if evidence_offers_choice(json.loads(evidence)):
|
|
continue
|
|
removed += conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
|
|
if removed:
|
|
conn.execute(_BUMP_REVIEW_INDEX)
|
|
return removed
|
|
|
|
def review_queue_page(self, *, limit: int, after: tuple | None = None) -> dict:
|
|
"""One page of pending items, oldest first, from one snapshot.
|
|
|
|
``after`` is the ``(created_at, id)`` keyset of the last item already
|
|
shown. Returns ``{epoch, revision, total, items, more}``; ``total`` is
|
|
the exact pending count and ``items`` carry their ``spelled_path`` and
|
|
``locator`` (server-side only).
|
|
"""
|
|
if type(limit) is not int or limit < 1:
|
|
raise ValueError("limit must be a positive integer")
|
|
with self._annotation_read() as conn:
|
|
epoch = conn.execute(_EPOCH).fetchone()
|
|
revision = conn.execute(_REVIEW_INDEX).fetchone()
|
|
if epoch is None or revision is None:
|
|
raise sqlite3.DatabaseError("Missing review queue metadata")
|
|
total = conn.execute(_PENDING_COUNT).fetchone()[0]
|
|
if after is None:
|
|
rows = conn.execute(_QUEUE_FIRST, (limit + 1,)).fetchall()
|
|
else:
|
|
rows = conn.execute(_QUEUE_AFTER, (*after, limit + 1)).fetchall()
|
|
items = [_queue_row(row, _QUEUE_COLUMNS) for row in rows[:limit]]
|
|
return {
|
|
"epoch": epoch[0],
|
|
"revision": revision[0],
|
|
"total": total,
|
|
"items": items,
|
|
"more": len(rows) > limit,
|
|
}
|
|
|
|
def review_item(self, item_id: str) -> dict | None:
|
|
"""One item with its parsed evidence and the pending total, or None."""
|
|
if not isinstance(item_id, str) or _ITEM_ID.fullmatch(item_id) is None:
|
|
return None
|
|
with self._annotation_read() as conn:
|
|
epoch = conn.execute(_EPOCH).fetchone()
|
|
if epoch is None:
|
|
raise sqlite3.DatabaseError("Missing execution history metadata")
|
|
row = conn.execute(_DETAIL, (item_id,)).fetchone()
|
|
if row is None:
|
|
return None
|
|
item = _queue_row(row, _DETAIL_COLUMNS)
|
|
item["evidence"] = json.loads(item["evidence"])
|
|
item["epoch"] = epoch[0]
|
|
item["total"] = conn.execute(_PENDING_COUNT).fetchone()[0]
|
|
return item
|
|
|
|
def review_dependency_fingerprint(self, image_path: str, aliases=()) -> str:
|
|
with self._annotation_read() as conn:
|
|
return dependency_fingerprint(conn, image_path, aliases)
|
|
|
|
def review_same_name_rows(self, name: str) -> list:
|
|
"""``(id, image_path)`` of legacy rows that may spell a file ``name``."""
|
|
with self._annotation_read() as conn:
|
|
return same_name_rows(conn, name)
|
|
|
|
def stored_prompt_titles(self, texts) -> dict:
|
|
"""Library titles of saved prompts by text (dedup hash), when set."""
|
|
hashes = {generate_prompt_hash(text): text for text in texts if text.strip()}
|
|
if not hashes:
|
|
return {}
|
|
marks = ",".join("?" * len(hashes))
|
|
found: dict[str, tuple] = {}
|
|
conn = self.model.get_connection()
|
|
for prompt_hash, title in conn.execute(
|
|
_TITLES.format(marks=marks), list(hashes)
|
|
):
|
|
text = hashes[prompt_hash]
|
|
found[text] = found.get(text, ()) + (title,)
|
|
return found
|
|
|
|
|
|
def _refresh_spelling(conn: sqlite3.Connection, locator: str, spelled: str) -> bool:
|
|
return conn.execute(_REFRESH_SPELLING, (spelled, spelled, locator)).rowcount > 0
|
|
|
|
|
|
def _queue_row(row, columns) -> dict:
|
|
item = dict(zip(columns, tuple(row)))
|
|
item["limited"] = bool(item["limited"])
|
|
return item
|