nodes/ComfyAssets-ComfyUI_PromptManager
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(review): explicit association decisions with compatible linking

Browse Source 
Needs review gains its queue reads and decisions:

- GET /prompt_manager/review/items lists pending items oldest first with
  the exact pending total and an HMAC keyset cursor bound to the storage
  epoch and the review index revision.
- GET /prompt_manager/review/items/{id} adds the candidates (role texts,
  label, selectability), the dependency fingerprint and the digest.
- POST /prompt_manager/review/items/{id}/decision applies a candidate only
  for a capture of the item's bytes that the current file still hashes to
  and whose evidence is unchanged, through the ingester's compatible-only
  rules in the same transaction as the decision row; a conflict writes
  nothing. Leave unresolved records the decision and links nothing.

ingest_resolved_pair keeps its contract; its body moves into
_ingest_resolved_pair_in, which runs in the caller's transaction.

Items keep the file's recorded spelling so targets keep their case on
Windows, and a rescan skips the copy and hash of a file whose size and
mtime match the bytes an item already cites.
This commit is contained in:
Vito Sansevero
2026-10-22 16:27:49 -07:00
parent d25c2cb4f2
commit 3e20c533fa
14 changed files with 2161 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
database/operations.py
+2
View File
@@ -14,6 +14,7 @@ from .library import LibraryOperations, titled
from .image_favorites import ImageFavoriteOperations
from .annotations import AnnotationOperations
from .review import ReviewOperations
from .review_decisions import ReviewDecisionOperations
from .saved_views import SavedViewOperations
from .library_commands import LibraryCommandOperations
from .vocabulary import VocabularyOperations
@@ -343,6 +344,7 @@ class PromptDatabase(
ImageFavoriteOperations,
AnnotationOperations,
ReviewOperations,
ReviewDecisionOperations,
SavedViewOperations,
):
"""Database operations class for managing prompts."""
database/pair_ingestion.py
+130 -104
View File
@@ -19,118 +19,144 @@ def ingest_resolved_pair(
state = resolution.get("state", "unresolved")
if state != "resolved":
return {"status": state, "pairs_added": 0, "links_added": 0}
with db._pair_transaction() as conn:
return _ingest_resolved_pair_in(conn, db, image_path, metadata, resolution)
def _conflict() -> Dict[str, Any]:
return {"status": "conflict", "pairs_added": 0, "links_added": 0}
def _ingest_resolved_pair_in(
conn: Any,
db: Any,
image_path: str,
metadata: Dict[str, Any],
resolution: Dict[str, Any],
*,
provenance: Dict[str, Any] | None = None,
) -> Dict[str, Any]:
"""``ingest_resolved_pair`` inside the caller's open ``_pair_transaction``.
The compatible-only rules are the public ingester's: an image with more
than one exact row, another positive, or a prior pair with another
negative is a ``conflict`` and nothing is written. The caller owns the
transaction, so its own later failure rolls the link back too.
``provenance`` replaces the link's default resolver provenance (Needs
review records the user's selection there).
"""
positive, negative = resolution["positive"], resolution["negative"]
positive_text = positive.get("storage_text", positive["text"])
negative_text = negative.get("storage_text", negative["text"])
normalized = normalize_image_path(image_path)
provenance = {"resolver_version": 1, "positive": positive, "negative": negative}
if provenance is None:
provenance = {
"resolver_version": 1,
"positive": positive,
"negative": negative,
}
encoded = json.dumps(
provenance, sort_keys=True, separators=(",", ":"), allow_nan=False
)
with db._pair_transaction() as conn:
images = conn.execute(
"SELECT gi.*,p.hash AS positive_hash FROM generated_images gi "
"JOIN prompts p ON p.id=gi.prompt_id "
"WHERE gi.file_path=? OR gi.image_path=?",
(normalized, image_path),
images = conn.execute(
"SELECT gi.*,p.hash AS positive_hash FROM generated_images gi "
"JOIN prompts p ON p.id=gi.prompt_id "
"WHERE gi.file_path=? OR gi.image_path=?",
(normalized, image_path),
).fetchall()
if len(images) > 1 or (
images and images[0]["positive_hash"] != generate_prompt_hash(positive_text)
):
return _conflict()
existing = images[0] if images else None
if existing:
prior = conn.execute(
"SELECT n.hash AS negative_hash FROM image_prompt_pairs ip "
"JOIN prompt_pairs pp ON pp.id=ip.pair_id "
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id WHERE ip.image_id=?",
(existing["id"],),
).fetchall()
if len(images) > 1 or (
images and images[0]["positive_hash"] != generate_prompt_hash(positive_text)
):
return {"status": "conflict", "pairs_added": 0, "links_added": 0}
existing = images[0] if images else None
if existing:
prior = conn.execute(
"SELECT n.hash AS negative_hash FROM image_prompt_pairs ip "
"JOIN prompt_pairs pp ON pp.id=ip.pair_id "
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id WHERE ip.image_id=?",
(existing["id"],),
).fetchall()
expected = (
generate_prompt_hash(negative_text) if negative_text.strip() else None
)
if any(row["negative_hash"] != expected for row in prior):
return {"status": "conflict", "pairs_added": 0, "links_added": 0}
prior_pair = conn.execute(
"SELECT pp.id FROM prompt_pairs pp "
"JOIN prompts p ON p.id=pp.positive_prompt_id "
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id "
"WHERE p.hash=? AND n.hash IS ?",
(
generate_prompt_hash(positive_text),
generate_prompt_hash(negative_text) if negative_text.strip() else None,
),
).fetchone()
pair_id = db._save_pair(conn, positive_text, negative_text)
pair = conn.execute(
"SELECT * FROM prompt_pairs WHERE id=?", (pair_id,)
).fetchone()
pairs_added = int(prior_pair is None)
file_size, width, height, image_format = _coerce_file_info(metadata)
snapshots = {
column: json.dumps(metadata[key], sort_keys=True, allow_nan=False)
for column, key in (
("workflow_data", "workflow"),
("prompt_metadata", "prompt"),
("parameters", "parameters"),
)
if key in metadata
}
info = {
"file_size": file_size,
"width": width,
"height": height,
"format": image_format,
}
snapshots.update(
{key: value for key, value in info.items() if value is not None}
expected = (
generate_prompt_hash(negative_text) if negative_text.strip() else None
)
changed = False
if existing:
image_id = existing["id"]
updates = {
key: value for key, value in snapshots.items() if existing[key] != value
}
if updates:
conn.execute(
"UPDATE generated_images SET "
+ ",".join(f"{key}=?" for key in updates)
+ " WHERE id=?",
[*updates.values(), image_id],
)
changed = True
else:
columns = ["prompt_id", "image_path", "filename", "file_path", *snapshots]
values = [
pair["positive_prompt_id"],
image_path,
os.path.basename(image_path),
normalized,
*snapshots.values(),
]
image_id = conn.execute(
"INSERT INTO generated_images("
+ ",".join(columns)
+ ") VALUES("
+ ",".join("?" for _ in columns)
+ ")",
values,
).lastrowid
linked = conn.execute(
"SELECT 1 FROM image_prompt_pairs WHERE pair_id=? "
"AND image_id=? AND provenance=?",
(pair_id, image_id, encoded),
).fetchone()
db._link_image_to_pair(conn, pair_id, image_id, provenance)
status = (
"added"
if existing is None
else "enriched" if changed or not linked else "unchanged"
if any(row["negative_hash"] != expected for row in prior):
return _conflict()
prior_pair = conn.execute(
"SELECT pp.id FROM prompt_pairs pp "
"JOIN prompts p ON p.id=pp.positive_prompt_id "
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id "
"WHERE p.hash=? AND n.hash IS ?",
(
generate_prompt_hash(positive_text),
generate_prompt_hash(negative_text) if negative_text.strip() else None,
),
).fetchone()
pair_id = db._save_pair(conn, positive_text, negative_text)
pair = conn.execute("SELECT * FROM prompt_pairs WHERE id=?", (pair_id,)).fetchone()
pairs_added = int(prior_pair is None)
file_size, width, height, image_format = _coerce_file_info(metadata)
snapshots = {
column: json.dumps(metadata[key], sort_keys=True, allow_nan=False)
for column, key in (
("workflow_data", "workflow"),
("prompt_metadata", "prompt"),
("parameters", "parameters"),
)
return {
"status": status,
"pairs_added": pairs_added,
"links_added": int(not linked),
"pair_id": pair_id,
"image_id": image_id,
if key in metadata
}
info = {
"file_size": file_size,
"width": width,
"height": height,
"format": image_format,
}
snapshots.update({key: value for key, value in info.items() if value is not None})
changed = False
if existing:
image_id = existing["id"]
updates = {
key: value for key, value in snapshots.items() if existing[key] != value
}
if updates:
conn.execute(
"UPDATE generated_images SET "
+ ",".join(f"{key}=?" for key in updates)
+ " WHERE id=?",
[*updates.values(), image_id],
)
changed = True
else:
columns = ["prompt_id", "image_path", "filename", "file_path", *snapshots]
values = [
pair["positive_prompt_id"],
image_path,
os.path.basename(image_path),
normalized,
*snapshots.values(),
]
image_id = conn.execute(
"INSERT INTO generated_images("
+ ",".join(columns)
+ ") VALUES("
+ ",".join("?" for _ in columns)
+ ")",
values,
).lastrowid
linked = conn.execute(
"SELECT 1 FROM image_prompt_pairs WHERE pair_id=? "
"AND image_id=? AND provenance=?",
(pair_id, image_id, encoded),
).fetchone()
db._link_image_to_pair(conn, pair_id, image_id, provenance)
status = (
"added"
if existing is None
else "enriched" if changed or not linked else "unchanged"
)
return {
"status": status,
"pairs_added": pairs_added,
"links_added": int(not linked),
"pair_id": pair_id,
"image_id": image_id,
}
database/review.py
+204 -7
View File
@@ -12,20 +12,29 @@ nothing here reads the filesystem. Writes run in one ``_pair_transaction``
fenced by the storage epoch the capture was taken under.
"""
import hashlib
import json
import os
import re
import sqlite3
from typing import Iterator
try:
from ..utils.hashing import generate_prompt_hash
except ImportError:
from utils.hashing import generate_prompt_hash
from .annotation_schema import (
BYTES_TABLE,
FILES_TABLE,
METADATA_TABLE as ANNOTATION_METADATA_TABLE,
)
from .annotations import _new_id, _validate_locator, _validate_target
from .models import utc_now_iso
from .models import normalize_image_path, utc_now_iso
from .review_schema import (
ITEMS_TABLE,
MAX_CANDIDATES,
MAX_SPELLED_PATH_BYTES,
METADATA_TABLE,
canonical_evidence,
evidence_digest,
@@ -33,9 +42,9 @@ from .review_schema import (
# Only this module's table constants are interpolated into the SQL below.
_INSERT_ITEM = (
f"INSERT OR IGNORE INTO {ITEMS_TABLE}(id,byte_id,resolver_version," # nosec B608
"evidence_hash,evidence,candidate_count,limited,state,revision,created_at,"
"updated_at) VALUES(?,?,?,?,?,?,?,'pending',0,?,?)"
f"INSERT OR IGNORE INTO {ITEMS_TABLE}(id,byte_id,spelled_path," # nosec B608
"resolver_version,evidence_hash,evidence,candidate_count,limited,state,"
"revision,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,'pending',0,?,?)"
)
_BUMP_REVIEW_INDEX = (
f"UPDATE {METADATA_TABLE} SET " # nosec B608
@@ -57,6 +66,73 @@ _ITEMS_OF_LOCATOR = (
f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
"WHERE f.locator=? ORDER BY i.created_at,i.id"
)
_QUEUED = (
f"SELECT 1 FROM {FILES_TABLE} f " # nosec B608
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
"WHERE f.locator=? AND b.size=? AND b.mtime_ns=? AND i.resolver_version=? "
"LIMIT 1"
)
_EPOCH = "SELECT storage_epoch FROM execution_history_metadata WHERE singleton_id=1"
_PENDING_COUNT = (
f"SELECT COUNT(*) FROM {ITEMS_TABLE} WHERE state='pending'" # nosec B608
)
_QUEUE_COLUMNS = (
"id",
"state",
"revision",
"created_at",
"candidate_count",
"limited",
"spelled_path",
"locator",
)
_QUEUE_SELECT = (
"SELECT i.id,i.state,i.revision,i.created_at,i.candidate_count," # nosec B608
f"i.limited,i.spelled_path,f.locator FROM {ITEMS_TABLE} i "
f"JOIN {BYTES_TABLE} b ON b.id=i.byte_id "
f"JOIN {FILES_TABLE} f ON f.id=b.file_id "
)
_QUEUE_FIRST = _QUEUE_SELECT + (
"WHERE i.state='pending' ORDER BY i.created_at,i.id LIMIT ?"
)
_QUEUE_AFTER = _QUEUE_SELECT + (
"WHERE i.state='pending' AND (i.created_at,i.id)>(?,?) "
"ORDER BY i.created_at,i.id LIMIT ?"
)
_DETAIL_COLUMNS = _QUEUE_COLUMNS + (
"byte_id",
"resolver_version",
"evidence_hash",
"evidence",
"updated_at",
"sha256",
)
_DETAIL = (
"SELECT i.id,i.state,i.revision,i.created_at,i.candidate_count," # nosec B608
"i.limited,i.spelled_path,f.locator,i.byte_id,i.resolver_version,"
"i.evidence_hash,i.evidence,i.updated_at,b.sha256 "
f"FROM {ITEMS_TABLE} i JOIN {BYTES_TABLE} b ON b.id=i.byte_id "
f"JOIN {FILES_TABLE} f ON f.id=b.file_id WHERE i.id=?"
)
# The legacy rows and pair links the ingester's compatibility rules read.
_DEPENDENT_IMAGES = (
"SELECT gi.id,p.hash FROM generated_images gi "
"JOIN prompts p ON p.id=gi.prompt_id "
"WHERE gi.file_path=? OR gi.image_path=? ORDER BY gi.id"
)
_DEPENDENT_LINKS = (
"SELECT DISTINCT ip.pair_id,n.hash FROM image_prompt_pairs ip "
"JOIN prompt_pairs pp ON pp.id=ip.pair_id "
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id "
"WHERE ip.image_id=? ORDER BY ip.pair_id"
)
_TITLES = (
"SELECT p.hash,s.title FROM prompts p "
"JOIN prompt_library_state s ON s.prompt_id=p.id "
"WHERE p.hash IN ({marks}) AND s.title IS NOT NULL ORDER BY p.id"
)
_ITEM_ID = re.compile(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}")
_ITEM_COLUMNS = (
"id",
"byte_id",
@@ -73,6 +149,34 @@ _ITEM_COLUMNS = (
)
def _validate_spelling(spelled_path: object, locator: str) -> None:
if (
not isinstance(spelled_path, str)
or not spelled_path
or len(spelled_path.encode("utf-8", "surrogatepass")) > MAX_SPELLED_PATH_BYTES
or os.path.normcase(spelled_path) != locator
):
raise ValueError("spelled_path must be the file's spelling of its locator")
def dependency_fingerprint(conn: sqlite3.Connection, image_path: str) -> str:
"""SHA-256 of what Apply's compatibility depends on for ``image_path``.
The exact legacy rows of the file with their positive hashes, and each
row's pair links with the negative hash, read as the ingester reads them.
Unrelated prompt changes (tags, titles) do not move it.
"""
images = conn.execute(
_DEPENDENT_IMAGES, (normalize_image_path(image_path), image_path)
).fetchall()
rows = []
for image_id, positive_hash in images:
links = conn.execute(_DEPENDENT_LINKS, (image_id,)).fetchall()
rows.append([image_id, positive_hash, [list(link) for link in links]])
text = canonical_evidence({"v": 1, "images": rows})
return hashlib.sha256(text.encode("utf-8")).hexdigest()
class ReviewOperations:
"""PromptDatabase mixin: insert and read Needs review items."""
@@ -80,6 +184,7 @@ class ReviewOperations:
self,
*,
locator: str,
spelled_path: str,
sha256: str,
size: int,
mtime_ns: int,
@@ -92,11 +197,14 @@ class ReviewOperations:
"""Store one ambiguous result for these exact bytes; True when new.
The byte record is created (or its verified fingerprint refreshed)
like an observe. Raises ``LibraryCommandError("storage-changed")`` when
the storage epoch is no longer ``epoch`` (a restore since the capture)
and ``ValueError`` for invalid facts or evidence.
like an observe. ``spelled_path`` is the file's own spelling (it must
fold to ``locator``); targets are formatted from it. Raises
``LibraryCommandError("storage-changed")`` when the storage epoch is
no longer ``epoch`` (a restore since the capture) and ``ValueError``
for invalid facts or evidence.
"""
_validate_target(locator, sha256, size, mtime_ns)
_validate_spelling(spelled_path, locator)
if type(resolver_version) is not int or resolver_version < 1:
raise ValueError("resolver_version must be a positive integer")
if (
@@ -122,6 +230,7 @@ class ReviewOperations:
(
_new_id(),
byte_id,
spelled_path,
resolver_version,
evidence_hash,
text,
@@ -155,3 +264,91 @@ class ReviewOperations:
item["evidence"] = json.loads(item["evidence"])
item["limited"] = bool(item["limited"])
yield item
# -- queue reads (#249 E3) -------------------------------------------------
def review_fingerprint_queued(
self, locator: str, size: int, mtime_ns: int, resolver_version: int
) -> bool:
"""True when an item cites a byte record verified at this exact stat.
A rescan uses it to skip the copy and hash of a file it already
queued: the same ``(size, mtime_ns)`` the bytes were verified with.
"""
row = (
self.model.get_connection()
.execute(_QUEUED, (locator, size, mtime_ns, resolver_version))
.fetchone()
)
return row is not None
def review_queue_page(self, *, limit: int, after: tuple | None = None) -> dict:
"""One page of pending items, oldest first, from one snapshot.
``after`` is the ``(created_at, id)`` keyset of the last item already
shown. Returns ``{epoch, revision, total, items, more}``; ``total`` is
the exact pending count and ``items`` carry their ``spelled_path`` and
``locator`` (server-side only).
"""
if type(limit) is not int or limit < 1:
raise ValueError("limit must be a positive integer")
with self._annotation_read() as conn:
epoch = conn.execute(_EPOCH).fetchone()
revision = conn.execute(_REVIEW_INDEX).fetchone()
if epoch is None or revision is None:
raise sqlite3.DatabaseError("Missing review queue metadata")
total = conn.execute(_PENDING_COUNT).fetchone()[0]
if after is None:
rows = conn.execute(_QUEUE_FIRST, (limit + 1,)).fetchall()
else:
rows = conn.execute(_QUEUE_AFTER, (*after, limit + 1)).fetchall()
items = [_queue_row(row, _QUEUE_COLUMNS) for row in rows[:limit]]
return {
"epoch": epoch[0],
"revision": revision[0],
"total": total,
"items": items,
"more": len(rows) > limit,
}
def review_item(self, item_id: str) -> dict | None:
"""One item with its parsed evidence and the pending total, or None."""
if not isinstance(item_id, str) or _ITEM_ID.fullmatch(item_id) is None:
return None
with self._annotation_read() as conn:
epoch = conn.execute(_EPOCH).fetchone()
if epoch is None:
raise sqlite3.DatabaseError("Missing execution history metadata")
row = conn.execute(_DETAIL, (item_id,)).fetchone()
if row is None:
return None
item = _queue_row(row, _DETAIL_COLUMNS)
item["evidence"] = json.loads(item["evidence"])
item["epoch"] = epoch[0]
item["total"] = conn.execute(_PENDING_COUNT).fetchone()[0]
return item
def review_dependency_fingerprint(self, image_path: str) -> str:
with self._annotation_read() as conn:
return dependency_fingerprint(conn, image_path)
def stored_prompt_titles(self, texts) -> dict:
"""Library titles of saved prompts by text (dedup hash), when set."""
hashes = {generate_prompt_hash(text): text for text in texts if text.strip()}
if not hashes:
return {}
marks = ",".join("?" * len(hashes))
found: dict[str, tuple] = {}
conn = self.model.get_connection()
for prompt_hash, title in conn.execute(
_TITLES.format(marks=marks), list(hashes)
):
text = hashes[prompt_hash]
found[text] = found.get(text, ()) + (title,)
return found
def _queue_row(row, columns) -> dict:
item = dict(zip(columns, tuple(row)))
item["limited"] = bool(item["limited"])
return item
database/review_decisions.py
+252
View File
@@ -0,0 +1,252 @@
"""Needs review decisions (#249 E3): explicit, auditable and compatible-only.
A decision is one row of ``media_review_decisions`` for one item revision.
``selected`` links the chosen candidate's pair to the file through the public
ingester's compatible-only rules (``_ingest_resolved_pair_in``) in the same
``_pair_transaction`` as the decision row, the item's new state and revision
and the review index revision; any conflict or later failure rolls all of it
back. ``unresolved`` records the decision and links nothing. No decision
counts a run, touches usage or calls an execution writer.
Callers verify the file first (the capture must be the item's bytes and the
current file must still hash to them); nothing here reads the filesystem.
"""
import sqlite3
from .annotation_schema import BYTES_TABLE
from .annotations import _new_id
from .library_commands import LibraryCommandError
from .models import utc_now_iso
from .review import (
_BUMP_REVIEW_INDEX,
_DETAIL,
_DETAIL_COLUMNS,
_PENDING_COUNT,
_queue_row,
dependency_fingerprint,
)
from .review_schema import DECISIONS_TABLE, ITEMS_TABLE, canonical_evidence
DECISION_KIND = "user-selection"
_INSERT_DECISION = (
f"INSERT INTO {DECISIONS_TABLE}(id,item_id,decision_revision," # nosec B608
"created_at,outcome,candidate_hash,evidence,pair_id,image_id) "
"VALUES(?,?,?,?,?,?,?,?,?)"
)
_ADVANCE_ITEM = (
f"UPDATE {ITEMS_TABLE} SET state=?,revision=revision+1," # nosec B608
"updated_at=? WHERE id=? AND revision=?"
)
_ITEM_BYTES = (
f"SELECT b.sha256 FROM {ITEMS_TABLE} i " # nosec B608
f"JOIN {BYTES_TABLE} b ON b.id=i.byte_id WHERE i.id=?"
)
class ReviewCommandError(LibraryCommandError):
"""A decision lost a race or would rewrite an association.
``code`` is ``review-conflict`` (the item was decided or changed since it
was read), ``association-conflict`` (the choice is not compatible with
the file's saved links, or those links changed), ``file-changed`` (the
item is not about these bytes) or ``review-missing``. ``item`` is the
current item read inside the transaction, ``total`` the pending count and
``fingerprint`` the current dependency fingerprint, when known;
``changed`` marks an association conflict caused by links that changed
since the reviewer read them (rather than an incompatible choice).
"""
def __init__(
self, code, epoch, *, item=None, total=None, fingerprint=None, changed=False
):
super().__init__(code, epoch)
self.item = item
self.total = total
self.fingerprint = fingerprint
self.changed = changed
class _Rollback(Exception):
"""Raised inside the transaction so the context manager rolls back."""
def __init__(self, error: ReviewCommandError):
super().__init__(error.code)
self.error = error
class ReviewDecisionOperations:
"""PromptDatabase mixin: record one decision on one Needs review item."""
def decide_review_item(
self,
*,
item_id: str,
expected_epoch: str,
expected_revision: int,
sha256: str,
outcome: str,
candidate: dict | None = None,
image_path: str | None = None,
metadata: dict | None = None,
expected_fingerprint: str | None = None,
) -> dict:
"""Record the decision; return ``{item, total, pair}``.
``selected`` needs the evidence ``candidate`` (with its ``hash``), the
``image_path`` the scan and the monitor link under, the captured
``metadata`` and the ``expected_fingerprint`` the reviewer saw.
Raises ``LibraryCommandError("storage-changed")`` after a restore and
:class:`ReviewCommandError` for every refused decision, with zero
writes either way.
"""
if outcome not in ("selected", "unresolved"):
raise ValueError("outcome must be selected or unresolved")
if outcome == "selected" and (
not isinstance(candidate, dict) or not image_path
):
raise ValueError("A selected decision needs a candidate and a file")
try:
with self._pair_transaction() as conn:
return self._decide_in(
conn,
item_id=item_id,
expected_epoch=expected_epoch,
expected_revision=expected_revision,
sha256=sha256,
outcome=outcome,
candidate=candidate,
image_path=image_path,
metadata=metadata or {},
expected_fingerprint=expected_fingerprint,
)
except _Rollback as rollback:
raise rollback.error from None
def _decide_in(self, conn: sqlite3.Connection, **command) -> dict:
epoch = self._library_epoch(conn, command["expected_epoch"])
item_id = command["item_id"]
current = _current_item(conn, item_id)
if current is None:
raise _Rollback(ReviewCommandError("review-missing", epoch))
total = conn.execute(_PENDING_COUNT).fetchone()[0]
if (
current["state"] != "pending"
or current["revision"] != command["expected_revision"]
):
raise _Rollback(
ReviewCommandError("review-conflict", epoch, item=current, total=total)
)
if conn.execute(_ITEM_BYTES, (item_id,)).fetchone()[0] != command["sha256"]:
raise _Rollback(ReviewCommandError("file-changed", epoch))
decision_id = _new_id()
pair = None
candidate = command["candidate"] if command["outcome"] == "selected" else None
if candidate is not None:
pair = self._apply_candidate(
conn, epoch, current, decision_id, total, **command
)
evidence = {
"kind": DECISION_KIND,
"decision_id": decision_id,
"outcome": command["outcome"],
"reviewed_digest": command["sha256"],
"item_evidence_hash": current["evidence_hash"],
"resolver_version": current["resolver_version"],
"candidate": candidate,
"dependency_fingerprint": command["expected_fingerprint"],
"pair": pair,
}
now = utc_now_iso()
self._insert_review_decision(
conn,
(
decision_id,
item_id,
current["revision"] + 1,
now,
command["outcome"],
candidate["hash"] if candidate is not None else None,
canonical_evidence(evidence),
pair["id"] if pair else None,
pair["image_id"] if pair else None,
),
)
conn.execute(
_ADVANCE_ITEM, (command["outcome"], now, item_id, current["revision"])
)
conn.execute(_BUMP_REVIEW_INDEX)
return {
"item": _current_item(conn, item_id),
"total": conn.execute(_PENDING_COUNT).fetchone()[0],
"pair": pair,
"decision_id": decision_id,
}
def _apply_candidate(
self, conn, epoch, current, decision_id, total, **command
) -> dict:
"""Link the candidate's pair, compatible-only; raise on any conflict."""
# Imported here: pair_ingestion imports operations, which composes this.
from .pair_ingestion import _ingest_resolved_pair_in
image_path = command["image_path"]
fingerprint = dependency_fingerprint(conn, image_path)
if fingerprint != command["expected_fingerprint"]:
raise _Rollback(
ReviewCommandError(
"association-conflict",
epoch,
item=current,
total=total,
fingerprint=fingerprint,
changed=True,
)
)
candidate = command["candidate"]
positive, negative = candidate["positive"], candidate["negative"]
provenance = {
"kind": DECISION_KIND,
"decision_id": decision_id,
"reviewed_digest": command["sha256"],
"candidate_hash": candidate["hash"],
"resolver_version": current["resolver_version"],
"positive": positive,
"negative": negative,
}
result = _ingest_resolved_pair_in(
conn,
self,
image_path,
command["metadata"],
{"state": "resolved", "positive": positive, "negative": negative},
provenance=provenance,
)
if result["status"] == "conflict":
raise _Rollback(
ReviewCommandError(
"association-conflict",
epoch,
item=current,
total=total,
fingerprint=fingerprint,
)
)
return {
"id": result["pair_id"],
"image_id": result["image_id"],
"status": result["status"],
}
@staticmethod
def _insert_review_decision(conn: sqlite3.Connection, values: tuple) -> None:
conn.execute(_INSERT_DECISION, values)
def _current_item(conn: sqlite3.Connection, item_id: str) -> dict | None:
row = conn.execute(_DETAIL, (item_id,)).fetchone()
if row is None:
return None
item = _queue_row(row, _DETAIL_COLUMNS)
item.pop("evidence")
return item
database/review_schema.py
+12 -2
View File
@@ -7,8 +7,10 @@ annotation group, because every item cites one annotation byte record. It owns
- ``media_review_metadata``: one row, the group's schema version and the
review index revision (moved by every queue change);
- ``media_review_items``: one ambiguous resolver result for one byte record,
keyed by ``(byte_id, resolver_version, evidence_hash)``, with its canonical
evidence, candidate count, ``limited`` flag, state and revision;
keyed by ``(byte_id, resolver_version, evidence_hash)``, with the file's
spelling when it was recorded (the locator is case-folded on Windows, a
target is not), its canonical evidence, candidate count, ``limited`` flag,
state and revision;
- ``media_review_decisions``: the auditable decision history of an item, one
row per decision revision, with optional navigation to the pair and image
the decision linked (nulled, never cascaded, when those are deleted).
@@ -40,6 +42,9 @@ DECISIONS_TABLE = "media_review_decisions"
TABLES = (METADATA_TABLE, ITEMS_TABLE, DECISIONS_TABLE)
MAX_EVIDENCE_BYTES = 8 * 1024 * 1024
MAX_CANDIDATES = 16
# The file's own spelling when it was recorded (bytes of UTF-8 text). Locators
# are case-folded on Windows, so a target is formatted from this instead.
MAX_SPELLED_PATH_BYTES = 16384
ITEM_STATES = ("pending", "selected", "unresolved")
DECISION_OUTCOMES = ("selected", "unresolved")
_TIMESTAMP_BYTES = 64
@@ -57,6 +62,8 @@ _TABLE_DDL = {
ITEMS_TABLE: (
f"CREATE TABLE {ITEMS_TABLE} (id TEXT PRIMARY KEY CHECK(length(id)=36), "
f"byte_id TEXT NOT NULL REFERENCES {BYTES_TABLE}(id) ON DELETE RESTRICT, "
"spelled_path TEXT NOT NULL CHECK(length(CAST(spelled_path AS BLOB)) "
f"BETWEEN 1 AND {MAX_SPELLED_PATH_BYTES}), "
"resolver_version INTEGER NOT NULL, evidence_hash TEXT NOT NULL "
f"CHECK({_HEX64.format(c='evidence_hash')}), "
f"evidence TEXT NOT NULL {_EVIDENCE_CHECK}, "
@@ -109,6 +116,7 @@ _COLUMNS = {
ITEMS_TABLE: (
("id", "TEXT", 0, None, 1),
("byte_id", "TEXT", 1, None, 0),
("spelled_path", "TEXT", 1, None, 0),
("resolver_version", "INTEGER", 1, None, 0),
("evidence_hash", "TEXT", 1, None, 0),
("evidence", "TEXT", 1, None, 0),
@@ -243,6 +251,8 @@ _ROW_PREDICATES = {
(
_uuid("id"),
_uuid("byte_id"),
"typeof(spelled_path) = 'text' AND length(CAST(spelled_path AS BLOB)) "
f"BETWEEN 1 AND {MAX_SPELLED_PATH_BYTES}",
"typeof(resolver_version) = 'integer'",
_hex64("evidence_hash"),
_evidence("evidence"),