From f30db7713df8e707a0f688400d55a5a3bb37b745 Mon Sep 17 00:00:00 2001 From: Vito Sansevero Date: Wed, 28 Oct 2026 18:26:27 -0700 Subject: [PATCH] feat(library): archive paired and delete unpaired selected prompts on the server (#443) A prompt_archive table marks Library parents hidden from the shelf; older builds ignore it. New routes /prompt_manager/library/delete/preview, /library/delete and /library/restore classify a selection (delete, archive, protected, already archived, not found), execute exactly what was reviewed under a records hold, take a verified safety backup (disk checked) before any delete, and restore archived prompts. The shelf, search and the shelf-facing counts honor the archive (?shelf=active|archived); archiving repairs a stale prompt hash so scans find the archived row. --- database/library.py | 53 ++- database/library_archive.py | 289 +++++++++++++++++ database/maintenance.py | 9 +- database/models.py | 8 + database/operations.py | 12 +- database/prompt_archive.py | 107 ++++++ database/restore_stage.py | 9 +- database/vocabulary.py | 29 +- py/api/__init__.py | 4 + py/api/jobs.py | 3 + py/api/library.py | 10 + py/api/library_delete.py | 263 +++++++++++++++ tests/test_bulk_library_delete.py | 484 ++++++++++++++++++++++++++++ tests/test_history_reads.py | 26 ++ tests/test_library.py | 4 +- tests/test_library_delete_routes.py | 353 ++++++++++++++++++++ tests/test_prompt_archive_schema.py | 196 +++++++++++ 17 files changed, 1826 insertions(+), 33 deletions(-) create mode 100644 database/library_archive.py create mode 100644 database/prompt_archive.py create mode 100644 py/api/library_delete.py create mode 100644 tests/test_bulk_library_delete.py create mode 100644 tests/test_library_delete_routes.py create mode 100644 tests/test_prompt_archive_schema.py diff --git a/database/library.py b/database/library.py index d91677be..812b3faa 100644 --- a/database/library.py +++ b/database/library.py @@ -6,6 +6,7 @@ from collections.abc import Iterator from contextlib import contextmanager from .library_tokens import library_edit_token +from .prompt_archive import ARCHIVED, SHELVED _ELIGIBLE = """(EXISTS(SELECT 1 FROM prompt_pairs ep WHERE ep.positive_prompt_id=p.id) OR EXISTS(SELECT 1 FROM generated_images eg WHERE eg.prompt_id=p.id) @@ -105,6 +106,12 @@ ARCHIVE_SCOPES = { } +# Shelves (#443): the parents on the shelf, or the archived ones. +PARENT_SCOPES = {"active": SHELVED, "archived": ARCHIVED} +# A positive pair owner that is on the shelf, as a condition on alias pp. +SHELVED_PAIR = SHELVED.replace("p.id", "pp.positive_prompt_id") + + def _escape(value: str) -> str: return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") @@ -216,6 +223,8 @@ class LibraryOperations: rows = conn.execute( "SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes,p.created_at," "p.run_count,p.last_used_at," + + ARCHIVED + + " AS archived," + (f"{sort_expression} AS _sort," if sort_expression else "") + "(SELECT COUNT(*) FROM prompt_pairs v WHERE v.positive_prompt_id=p.id " "AND v.archived=0) AS variant_count," @@ -239,6 +248,7 @@ class LibraryOperations: # Usage is shown, never compared: the token hashes owner fields only. row["run_count"] = row["run_count"] if type(row["run_count"]) is int else 0 row["last_used_at"] = utc_iso(row["last_used_at"]) + row["archived"] = bool(row["archived"]) row["edit_token"] = library_edit_token(epoch, row) return result @@ -254,10 +264,13 @@ class LibraryOperations: sort: str = "newest", limit: int = 40, after: dict | None = None, + shelf: str = "active", ) -> dict: _limit(limit) if sort not in _ORDERS: raise ValueError("Unknown library sort order") + if not isinstance(shelf, str) or shelf not in PARENT_SCOPES: + raise ValueError("shelf must be active or archived") where, params = _filter(text, category, [] if tags is None else tags) if after is not None: if ( @@ -270,15 +283,18 @@ class LibraryOperations: ceiling = after["ceiling"] where += " AND p.id<=?" params.append(ceiling) - # One SELECT counts the filtered parents and their active variants. - total, variant_total = conn.execute( - "SELECT COUNT(*),COALESCE(SUM((SELECT COUNT(*) FROM prompt_pairs v " - "WHERE v.positive_prompt_id=p.id AND v.archived=0)),0) FROM " - + _FROM - + " WHERE " - + where, + # One SELECT counts the filtered parents of this scope, their active + # variants, and the filtered archived parents (Show archived, #443). + scope = PARENT_SCOPES[shelf] + total, variant_total, archived_total = conn.execute( + f"SELECT COALESCE(SUM({scope}),0)," + f"COALESCE(SUM(CASE WHEN {scope} THEN (SELECT COUNT(*) " + "FROM prompt_pairs v WHERE v.positive_prompt_id=p.id " + "AND v.archived=0) ELSE 0 END),0)," + f"COALESCE(SUM({ARCHIVED}),0) FROM " + _FROM + " WHERE " + where, params, ).fetchone() + where += f" AND {scope}" expression, direction = _ORDERS[sort] if after is not None: operator = ">" if direction == "ASC" else "<" @@ -311,6 +327,7 @@ class LibraryOperations: "items": rows, "total": total, "variant_total": variant_total, + "archived_total": archived_total, "has_more": has_more, "next_key": next_key, } @@ -324,6 +341,7 @@ class LibraryOperations: sort: str = "newest", limit: int = 40, after: dict | None = None, + shelf: str = "active", ) -> dict: with self.library_snapshot(include_ceiling=True) as (conn, epoch, ceiling): return self._list_library( @@ -336,6 +354,7 @@ class LibraryOperations: sort=sort, limit=limit, after=after, + shelf=shelf, ) def _is_library_positive(self, conn: sqlite3.Connection, positive_id: int) -> bool: @@ -576,7 +595,9 @@ class LibraryOperations: owners = {} order = [] for row in conn.execute( - "SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes FROM " + "SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes," + + ARCHIVED + + " AS archived FROM " + _FROM + " WHERE " + _ELIGIBLE @@ -589,6 +610,7 @@ class LibraryOperations: "tags": [], "rating": row["rating"], "notes": row["notes"], + "archived": bool(row["archived"]), "variants": [], } order.append(row["id"]) @@ -633,8 +655,9 @@ class LibraryOperations: """System › Overview counts from one snapshot, writing nothing. ``positive_prompts`` and ``negative_variants`` are the unfiltered - Library's ``total`` and ``variant_total``: eligible parents and active - pairs (every pair's positive is eligible). ``images`` counts indexed + Library's ``total`` and ``variant_total``: eligible parents on the + shelf (not archived, #443) and their active pairs (every pair's + positive is eligible). ``images`` counts indexed ``generated_images`` rows, not files on disk. An image has no rating of its own (it shows its linked prompt's), so ``rated_images`` counts the indexed images whose linked prompt is rated. ``review_pending`` is @@ -642,8 +665,14 @@ class LibraryOperations: """ with self.library_snapshot() as (conn, _epoch, _ceiling): row = conn.execute( - "SELECT (SELECT COUNT(*) FROM prompts p WHERE " + _ELIGIBLE + ")," - "(SELECT COUNT(*) FROM prompt_pairs WHERE archived=0)," + "SELECT (SELECT COUNT(*) FROM prompts p WHERE " + + _ELIGIBLE + + " AND " + + SHELVED + + ")," + "(SELECT COUNT(*) FROM prompt_pairs pp WHERE pp.archived=0 AND " + + SHELVED_PAIR + + ")," "(SELECT COUNT(*) FROM generated_images)," "(SELECT COUNT(*) FROM generated_images gi JOIN prompts r " "ON r.id=gi.prompt_id WHERE r.rating IS NOT NULL)," diff --git a/database/library_archive.py b/database/library_archive.py new file mode 100644 index 00000000..c44aa5e0 --- /dev/null +++ b/database/library_archive.py @@ -0,0 +1,289 @@ +"""Delete selected Library prompts; archive the paired ones (#443, D152). + +A selected Library parent is classified, in this order, as ``not_found`` +(missing, or not a Library parent), ``protected`` (tagged ``__protected__``; +never changed), ``already_archived``, ``archive`` (the positive or negative of +any saved pair, archived pair or not: archiving hides it from the shelf and +removes nothing) or ``delete`` (in no pair: deleted exactly as the legacy +delete does, through ``delete_prompt_rows``). Every command checks the +storage epoch first and runs in one ``BEGIN IMMEDIATE`` transaction. + +The write executes what the user reviewed: it is given the reviewed archive +and delete lists, and deletes only reviewed deletes the route backed up +(``candidates``) whose fingerprint is unchanged since that backup. +""" + +import sqlite3 +from typing import Iterable + +from .library import _ELIGIBLE +from .library_commands import _validate_bulk_ids, _validate_epoch +from .models import utc_now_iso +from .prompt_archive import ARCHIVED as _ARCHIVED +from .prompt_archive import TABLE as ARCHIVE +from .workflow_documents import documents_of_rows, release_documents + +try: + from ..utils.hashing import generate_prompt_hash +except ImportError: + from utils.hashing import generate_prompt_hash + +# Ids bound per statement: the pair check binds each id twice, which stays +# under SQLite's historical 999-variable limit (maintenance.py ID_BATCH). +ID_BATCH = 400 +_PROTECTED = ( + "EXISTS(SELECT 1 FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id " + "WHERE pt.prompt_id=p.id AND t.name='__protected__')" +) +_PAIRED = ( + "EXISTS(SELECT 1 FROM prompt_pairs pp " + "WHERE pp.positive_prompt_id=p.id OR pp.negative_prompt_id=p.id)" +) +# Classes in the order a prompt is tested; the first that holds wins. +CLASSES = ("not_found", "protected", "already_archived", "archive", "delete") + + +def _batches(ids: list) -> Iterable[list]: + for start in range(0, len(ids), ID_BATCH): + yield ids[start : start + ID_BATCH] + + +def delete_prompt_rows(conn: sqlite3.Connection, prompt_ids: list[int]) -> int: + """Delete prompts and their image link rows; return the prompts deleted. + + The legacy delete's statements (``delete_prompt``, ``bulk_delete_prompts``) + in the caller's transaction: image link rows first (``image_prompt_pairs`` + cascades, review decisions keep a NULL image), then the prompts (tags and + Library state cascade), then the workflow documents no row references. + Image files are never touched. The caller checks pairs first. + """ + documents = documents_of_rows(conn, "prompt_id", prompt_ids) + deleted = 0 + for chunk in _batches(list(prompt_ids)): + marks = ",".join("?" for _ in chunk) + conn.execute( + f"DELETE FROM generated_images WHERE prompt_id IN ({marks})", # nosec + chunk, + ) + deleted += conn.execute( + f"DELETE FROM prompts WHERE id IN ({marks})", chunk # nosec B608 + ).rowcount + release_documents(conn, documents) + return deleted + + +class LibraryArchiveOperations: + """PromptDatabase mixin: classify, archive or delete, and restore parents.""" + + def _classify_library_delete( + self, conn: sqlite3.Connection, prompt_ids: list[int] + ) -> dict: + found = {} + for chunk in _batches(prompt_ids): + marks = ",".join("?" for _ in chunk) + for row in conn.execute( + f"SELECT p.id,{_PROTECTED},{_ARCHIVED},{_PAIRED} " # nosec B608 + f"FROM prompts p WHERE p.id IN ({marks}) AND {_ELIGIBLE}", + chunk, + ): + protected, archived, paired = row[1], row[2], row[3] + if protected: + found[row[0]] = "protected" + elif archived: + found[row[0]] = "already_archived" + else: + found[row[0]] = "archive" if paired else "delete" + classes = {name: [] for name in CLASSES} + for pid in prompt_ids: + classes[found.get(pid, "not_found")].append(pid) + return classes + + @staticmethod + def _image_links(conn: sqlite3.Connection, prompt_ids: list[int]) -> int: + total = 0 + for chunk in _batches(prompt_ids): + marks = ",".join("?" for _ in chunk) + total += conn.execute( + "SELECT COUNT(*) FROM generated_images " # nosec B608 + f"WHERE prompt_id IN ({marks})", + chunk, + ).fetchone()[0] + return total + + @staticmethod + def _fingerprints(conn: sqlite3.Connection, prompt_ids: list[int]) -> dict: + """What a safety backup holds of each prompt a delete would remove. + + The prompt row (usage included), its tag ids, its Library title and + its image link rows (count and newest id), as a JSON-safe list. + """ + prints = {} + for chunk in _batches(prompt_ids): + marks = ",".join("?" for _ in chunk) + for row in conn.execute( + "SELECT p.id,p.text,p.hash,p.category,p.rating,p.notes," # nosec + "p.updated_at,p.run_count,p.last_used_at," + "(SELECT group_concat(tag_id) FROM (SELECT tag_id FROM prompt_tags " + "WHERE prompt_id=p.id ORDER BY tag_id))," + "(SELECT title FROM prompt_library_state WHERE prompt_id=p.id)," + "(SELECT COUNT(*) FROM generated_images WHERE prompt_id=p.id)," + "(SELECT MAX(id) FROM generated_images WHERE prompt_id=p.id) " + f"FROM prompts p WHERE p.id IN ({marks})", + chunk, + ): + prints[row[0]] = [ + value if isinstance(value, (int, str, type(None))) else repr(value) + for value in tuple(row)[1:] + ] + return prints + + @staticmethod + def _checked_ids(prompt_ids: list[int], expected_epoch: str) -> list[int]: + _validate_epoch(expected_epoch) + _validate_bulk_ids(prompt_ids) + return list(dict.fromkeys(prompt_ids)) + + def classify_library_delete( + self, prompt_ids: list[int], *, expected_epoch: str + ) -> dict: + """What a delete of ``prompt_ids`` would do now (reads only). + + Returns the id lists per class, in request order, ``image_links`` + (the image link rows the ``delete`` class would remove) and + ``fingerprints`` of the ``delete`` class (``_fingerprints``). + """ + prompt_ids = self._checked_ids(prompt_ids, expected_epoch) + with self.library_snapshot() as (conn, _epoch, _ceiling): + epoch = self._library_epoch(conn, expected_epoch) + classes = self._classify_library_delete(conn, prompt_ids) + return { + "library_epoch": epoch, + **classes, + "image_links": self._image_links(conn, classes["delete"]), + "fingerprints": self._fingerprints(conn, classes["delete"]), + } + + @staticmethod + def _repair_hash(conn: sqlite3.Connection, prompt_id: int) -> None: + """Give an archived prompt its canonical hash, so ingestion finds it. + + Scans, the monitor and node saves resolve prompts by hash; a stale or + missing one would make them insert a visible copy (Q3). When another + row already holds the canonical hash, lookups find that row and no new + copy is made, so nothing is written. + """ + text, stored = conn.execute( + "SELECT text,hash FROM prompts WHERE id=?", (prompt_id,) + ).fetchone() + canonical = generate_prompt_hash(text) if isinstance(text, str) else None + if canonical is None or stored == canonical: + return + if conn.execute( + "SELECT 1 FROM prompts WHERE hash=? AND id!=?", (canonical, prompt_id) + ).fetchone(): + return + conn.execute("UPDATE prompts SET hash=? WHERE id=?", (canonical, prompt_id)) + + def archive_or_delete_library_prompts( + self, + archive_ids: list[int], + delete_ids: list[int], + *, + expected_epoch: str, + candidates: dict | None, + ) -> dict: + """Archive and delete what the user reviewed, all or nothing. + + ``archive_ids`` and ``delete_ids`` are the reviewed classes. + ``candidates`` maps each reviewed delete the route backed up to its + fingerprint then (None: every reviewed delete, unchecked; tests and + callers without a backup). A reviewed delete is deleted only when it + still classifies ``delete`` and its fingerprint is unchanged; one that + is now paired, and a reviewed archive now unpaired, are archived (the + restorable direction); anything else is kept and counted. + """ + _validate_epoch(expected_epoch) + for ids in (archive_ids, delete_ids): + if not isinstance(ids, list) or any( + type(pid) is not int or not 1 <= pid <= 2**63 - 1 for pid in ids + ): + raise ValueError("prompt ids must be lists of positive integers") + reviewed_delete = set(delete_ids) + if not archive_ids and not delete_ids: + raise ValueError("Nothing to delete or archive") + if reviewed_delete & set(archive_ids): + raise ValueError("A prompt cannot be both archived and deleted") + prompt_ids = list(dict.fromkeys([*archive_ids, *delete_ids])) + with self._pair_transaction() as conn: + epoch = self._library_epoch(conn, expected_epoch) + classes = self._classify_library_delete(conn, prompt_ids) + now_prints = self._fingerprints(conn, classes["delete"]) + deletes, archives, changed = [], list(classes["archive"]), 0 + for pid in classes["delete"]: + if pid not in reviewed_delete: + archives.append(pid) # reviewed as archive, now unpaired + elif candidates is None or candidates.get(pid) == now_prints[pid]: + deletes.append(pid) + else: + changed += 1 + changed += len(classes["protected"]) + image_links = self._image_links(conn, deletes) + now = utc_now_iso() + for pid in archives: + self._repair_hash(conn, pid) + conn.executemany( + f"INSERT INTO {ARCHIVE}(prompt_id, archived_at) VALUES (?, ?)", + [(pid, now) for pid in archives], + ) + # A changed row is the newest edit the restore fingerprint reads. + conn.executemany( + "UPDATE prompts SET updated_at=? WHERE id=?", + [(now, pid) for pid in archives], + ) + deleted = delete_prompt_rows(conn, deletes) + return { + "library_epoch": epoch, + "deleted": deleted, + "archived": len(archives), + "already_archived": len(classes["already_archived"]), + "not_found": len(classes["not_found"]), + "changed": changed, + "image_links": image_links, + } + + def restore_library_prompts( + self, prompt_ids: list[int], *, expected_epoch: str + ) -> dict: + """Put archived prompts back on the shelf, all or nothing. + + Every requested id with an archive row is restored, whatever its + protection or Library parent state: restoring only un-hides it. + """ + prompt_ids = self._checked_ids(prompt_ids, expected_epoch) + with self._pair_transaction() as conn: + epoch = self._library_epoch(conn, expected_epoch) + restored, existing = [], set() + for chunk in _batches(prompt_ids): + marks = ",".join("?" for _ in chunk) + for pid, archived in conn.execute( + f"SELECT p.id,{_ARCHIVED} FROM prompts p " # nosec B608 + f"WHERE p.id IN ({marks})", + chunk, + ): + existing.add(pid) + if archived: + restored.append(pid) + now = utc_now_iso() + conn.executemany( + f"DELETE FROM {ARCHIVE} WHERE prompt_id=?", [(pid,) for pid in restored] + ) + conn.executemany( + "UPDATE prompts SET updated_at=? WHERE id=?", + [(now, pid) for pid in restored], + ) + return { + "library_epoch": epoch, + "restored": len(restored), + "unchanged": len(existing) - len(restored), + "not_found": len(prompt_ids) - len(existing), + } diff --git a/database/maintenance.py b/database/maintenance.py index 76a7abdc..4d6f39a2 100644 --- a/database/maintenance.py +++ b/database/maintenance.py @@ -80,6 +80,12 @@ PROTECTIONS = ( "with Library state (a title or an explicit parent)", "with Library state (a title or an explicit parent)", ), + ( + # #443: an archived prompt is restorable, never pruned. + "EXISTS (SELECT 1 FROM prompt_archive pa WHERE pa.prompt_id = p.id)", + "archived in the Library", + "archived in the Library", + ), ( "p.id IN (SELECT prompt_id FROM live_hint_prompts)", "used by recorded runs", @@ -88,7 +94,8 @@ PROTECTIONS = ( ) # Exempt references: images, pairs, the __protected__ tag, Library parent -# state, and live recorded-run hints (computed once, see LIVE_HINT_CTES). +# state, the Library archive, and live recorded-run hints (computed once, +# see LIVE_HINT_CTES). ORPHANED_PROMPTS_SQL = ( "WITH " + LIVE_HINT_CTES diff --git a/database/models.py b/database/models.py index 805d665f..5946d4a6 100644 --- a/database/models.py +++ b/database/models.py @@ -39,6 +39,7 @@ from .review_schema import ( ) from .saved_views import migrate_saved_views, saved_views_schema_error from .image_categories import image_categories_schema_error, migrate_image_categories +from .prompt_archive import migrate_prompt_archive, prompt_archive_schema_error from .workflow_documents import ( DOCUMENT_ID_COLUMNS, migrate_workflow_documents, @@ -358,6 +359,8 @@ class PromptModel: migrate_autotag_state(conn) # Image categories (#332): independent of every table above. migrate_image_categories(conn) + # The Library parent archive (#443): independent of every table above. + migrate_prompt_archive(conn) # Last: the older generated_images rebuilds run only before file_path. migrate_workflow_documents(conn) @@ -1234,6 +1237,11 @@ class PromptModel: ) if categories_error: return False, categories_error + archive_error = prompt_archive_schema_error( + conn, allow_missing=not require_library_state + ) + if archive_error: + return False, archive_error views_error = saved_views_schema_error( conn, allow_missing=not require_library_state ) diff --git a/database/operations.py b/database/operations.py index a40651b4..f6b46508 100644 --- a/database/operations.py +++ b/database/operations.py @@ -24,6 +24,7 @@ from .review import ReviewOperations from .review_decisions import ReviewDecisionOperations from .saved_views import SavedViewOperations from .library_commands import LibraryCommandOperations +from .library_archive import LibraryArchiveOperations, delete_prompt_rows from .vocabulary import VocabularyOperations from .execution_history import ExecutionHistoryOperations from .execution_contexts import ExecutionContextOperations @@ -448,6 +449,7 @@ class PromptDatabase( PairOperations, LibraryOperations, LibraryCommandOperations, + LibraryArchiveOperations, VocabularyOperations, ExecutionHistoryOperations, ExecutionContextOperations, @@ -2344,16 +2346,10 @@ class PromptDatabase( Returns: Number of prompts actually deleted. """ - count = 0 with self._pair_transaction() as conn: self._ensure_prompts_unpaired(conn, prompt_ids) - documents = documents_of_rows(conn, "prompt_id", prompt_ids) - for pid in prompt_ids: - conn.execute("DELETE FROM generated_images WHERE prompt_id = ?", (pid,)) - cursor = conn.execute("DELETE FROM prompts WHERE id = ?", (pid,)) - if cursor.rowcount > 0: - count += 1 - release_documents(conn, documents) + # The workspace Library's delete runs the same statements (#443). + count = delete_prompt_rows(conn, list(dict.fromkeys(prompt_ids))) conn.commit() return count diff --git a/database/prompt_archive.py b/database/prompt_archive.py new file mode 100644 index 00000000..9f824361 --- /dev/null +++ b/database/prompt_archive.py @@ -0,0 +1,107 @@ +"""Library parent archive (#443, D152): prompts hidden from the shelf, restorable. + +A row means the Library parent is archived; no row means it is on the shelf. +Nothing else about the prompt changes: its text, metadata, tags, run count, +image links, pairs and recorded runs stay. Restore deletes the row. Deleting +the prompt (any path) drops the row by cascade. + +A table of its own, not a column: ``prompt_library_state`` is validated by its +exact definition, so a new column there would make older builds refuse the +upgraded database, while older builds ignore an unknown table (they show the +archived prompts again and keep the rows). Optional in old backups, required +once a database is current. +""" + +import re +import sqlite3 + +TABLE = "prompt_archive" +MAX_STAMP_BYTES = 64 +# What ``utc_now_iso`` writes starts with a date; anything else was not written here. +_STAMP_GLOB = "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T*" +_DDL = ( + f"CREATE TABLE {TABLE} (prompt_id INTEGER PRIMARY KEY, " + "archived_at TEXT NOT NULL, " + "FOREIGN KEY(prompt_id) REFERENCES prompts(id) ON DELETE CASCADE)" +) +_COLUMNS = (("prompt_id", "INTEGER", 0, 1), ("archived_at", "TEXT", 1, 0)) +_FOREIGN_KEYS = {("prompt_id", "prompts", "id", "CASCADE")} +# A predicate on ``p`` (the prompts alias): the parent is not archived. +SHELVED = f"NOT EXISTS(SELECT 1 FROM {TABLE} pa WHERE pa.prompt_id=p.id)" +ARCHIVED = f"EXISTS(SELECT 1 FROM {TABLE} pa WHERE pa.prompt_id=p.id)" + + +def _compact(sql: str) -> str: + return re.sub(r"\s+", "", sql or "").lower() + + +def _structure_error(conn: sqlite3.Connection, allow_missing: bool) -> str | None: + objects = conn.execute( + "SELECT type,sql FROM sqlite_master WHERE name=? " + "OR (type IN ('index','trigger') AND tbl_name=?)", + (TABLE, TABLE), + ).fetchall() + if not objects: + return None if allow_missing else "Missing prompt archive schema" + tables = [sql for kind, sql in objects if kind == "table"] + if len(tables) != 1 or len(objects) != 1: + return "Prompt archive name is reserved by an unsupported schema object" + if _compact(tables[0]) != _compact(_DDL): + return "Prompt archive has an invalid table definition" + columns = tuple( + (row[1], row[2].upper(), row[3], row[5]) + for row in conn.execute(f"PRAGMA table_info({TABLE})") + ) + if columns != _COLUMNS: + return "Prompt archive has invalid columns" + foreign_keys = { + (row[3], row[2], row[4], row[6]) + for row in conn.execute(f"PRAGMA foreign_key_list({TABLE})") + } + if foreign_keys != _FOREIGN_KEYS: + return "Prompt archive has invalid foreign key definitions" + return None + + +def prompt_archive_schema_error( + conn: sqlite3.Connection, *, allow_missing: bool = True +) -> str | None: + """Reject an unsupported archive schema or values; reads only. + + A missing table is acceptable for backups that predate it unless + ``allow_missing`` is false. + """ + error = _structure_error(conn, allow_missing) + if error is not None: + return error + if not conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,) + ).fetchone(): + return None + if conn.execute(f"PRAGMA foreign_key_check({TABLE})").fetchone(): + return "Prompt archive contains foreign key violations" + if conn.execute( + f"SELECT 1 FROM {TABLE} WHERE typeof(archived_at)!='text' " # nosec B608 + f"OR length(CAST(archived_at AS BLOB))>{MAX_STAMP_BYTES} " + f"OR archived_at NOT GLOB '{_STAMP_GLOB}' LIMIT 1" + ).fetchone(): + return "Prompt archive contains invalid values" + return None + + +def migrate_prompt_archive(conn: sqlite3.Connection) -> None: + """Install the table atomically; validate before and after (no backfill).""" + conn.execute("SAVEPOINT prompt_archive_schema") + try: + error = _structure_error(conn, True) + if error: + raise sqlite3.DatabaseError(error) + conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1)) + error = _structure_error(conn, False) + if error: + raise sqlite3.DatabaseError(error) + conn.execute("RELEASE prompt_archive_schema") + except BaseException: + conn.execute("ROLLBACK TO prompt_archive_schema") + conn.execute("RELEASE prompt_archive_schema") + raise diff --git a/database/restore_stage.py b/database/restore_stage.py index 704435bf..1a3a3762 100644 --- a/database/restore_stage.py +++ b/database/restore_stage.py @@ -149,6 +149,7 @@ def library_summary(conn: sqlite3.Connection) -> Dict[str, int]: collections (#331). """ from .library import _ELIGIBLE + from .prompt_archive import SHELVED, TABLE as ARCHIVE has_state = conn.execute( "SELECT 1 FROM sqlite_master WHERE type='table' AND " @@ -157,9 +158,15 @@ def library_summary(conn: sqlite3.Connection) -> Dict[str, int]: has_pairs = conn.execute( "SELECT 1 FROM sqlite_master WHERE type='table' AND name='prompt_pairs'" ).fetchone() + has_archive = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (ARCHIVE,) + ).fetchone() if has_state and has_pairs: + # Archived parents (#443) are off the shelf, as in library_counts. positives = conn.execute( - "SELECT COUNT(*) FROM prompts p WHERE " + _ELIGIBLE + "SELECT COUNT(*) FROM prompts p WHERE " + + _ELIGIBLE + + (" AND " + SHELVED if has_archive else "") ).fetchone()[0] else: positives = _count(conn, "prompts") diff --git a/database/vocabulary.py b/database/vocabulary.py index 9dd5f42a..79e76461 100644 --- a/database/vocabulary.py +++ b/database/vocabulary.py @@ -20,7 +20,8 @@ counted, hashed into the token and renamed with the prompts and pairs. import sqlite3 -from .library import _ELIGIBLE, _limit +from .library import _ELIGIBLE, SHELVED_PAIR, _limit +from .prompt_archive import SHELVED from .library_commands import _TOKEN, LibraryCommandError, _validate_epoch from .library_tokens import canonical_token from .models import utc_now_iso @@ -40,17 +41,21 @@ _IMAGE_FILES = ( def _tag_rows(page: str) -> str: """Count rows for the registry rows selected by ``page`` (id, name).""" - # Eligible parent components carrying a tag in the page, plus the - # positive parent of every active pair carrying it (always eligible). + # Eligible parent components on the shelf carrying a tag in the page, + # plus the positive parent of every active pair carrying it (always + # eligible) when that parent is on the shelf: archived parents (#443) + # are hidden with their pairs, so Browse lists what was counted. return ( "WITH page AS (" + page + ")," "owners(tag_id,pid) AS (" "SELECT pt.tag_id,p.id FROM prompts p " "JOIN prompt_tags pt ON pt.prompt_id=p.id " "WHERE pt.tag_id IN (SELECT id FROM page) AND " + _ELIGIBLE + " " + "AND " + SHELVED + " " "UNION SELECT ppt.tag_id,pp.positive_prompt_id FROM prompt_pairs pp " "JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id " - "WHERE pp.archived=0 AND ppt.tag_id IN (SELECT id FROM page))," + "WHERE pp.archived=0 AND " + SHELVED_PAIR + " " + "AND ppt.tag_id IN (SELECT id FROM page))," "images(tag_id,n) AS (" + _IMAGE_FILES + " " "WHERE mit.tag_id IN (SELECT id FROM page) GROUP BY mit.tag_id) " "SELECT page.id,page.name,COUNT(DISTINCT owners.pid) AS prompt_count," @@ -100,7 +105,9 @@ def _validate_tag_id(tag_id: object) -> None: # Category values in Tags & categories (#332 §7): the prompt-backed values -# (nonblank values on eligible parents or on active pairs, grouped exactly, so +# (nonblank values on eligible parents or on active pairs; ``prompt_count`` +# counts only parents on the shelf, so a value only archived parents (#443) +# hold stays listed with 0, as a tag does; grouped exactly, so # whitespace and case variants stay distinct; /library/options lists the same # prompt-backed set with its own query) plus the values only image rows hold. # A prompt-backed value is ordered by its oldest owning positive; an image-only @@ -109,15 +116,17 @@ def _validate_tag_id(tag_id: object) -> None: # files missing on disk. No FULL OUTER JOIN, so it runs on SQLite 3.31+. _IMAGE_ONLY_FIRST_PID = 2**63 - 1 _CATEGORY_ROWS = ( - "WITH owners(value,pid) AS (" - "SELECT p.category,p.id FROM prompts p WHERE p.category IS NOT NULL " - "AND TRIM(p.category)!='' AND " + _ELIGIBLE + " " - "UNION SELECT pp.category,pp.positive_prompt_id FROM prompt_pairs pp " + "WITH owners(value,pid,shelved) AS (" + "SELECT p.category,p.id," + SHELVED + " FROM prompts p " + "WHERE p.category IS NOT NULL AND TRIM(p.category)!='' AND " + _ELIGIBLE + " " + "UNION SELECT pp.category,pp.positive_prompt_id," + SHELVED_PAIR + " " + "FROM prompt_pairs pp " "WHERE pp.archived=0 AND pp.category IS NOT NULL AND TRIM(pp.category)!='')," "image_values(value,image_count) AS (" "SELECT category,COUNT(*) FROM image_categories GROUP BY category)," "grouped(value,first_pid,prompt_count) AS (" - "SELECT value,MIN(pid),COUNT(DISTINCT pid) FROM owners GROUP BY value " + "SELECT value,MIN(pid),COUNT(DISTINCT CASE WHEN shelved THEN pid END) " + "FROM owners GROUP BY value " "UNION ALL SELECT value," + str(_IMAGE_ONLY_FIRST_PID) + ",0 FROM image_values " "WHERE value NOT IN (SELECT value FROM owners))," "counted AS (SELECT g.value,g.first_pid,g.prompt_count," diff --git a/py/api/__init__.py b/py/api/__init__.py index b2376662..6a85f779 100644 --- a/py/api/__init__.py +++ b/py/api/__init__.py @@ -19,6 +19,7 @@ Split into domain-specific mixins for maintainability: - SystemRoutesMixin: System health check, Overview and usage statistics - LibraryLocationRoutesMixin: where the library lives, Move library, Keep it here - MaintenanceRoutesMixin: reviewed maintenance plans (preview, apply) +- LibraryDeleteRoutesMixin: the Library's Delete… (archive paired) and Restore - WorkflowStorageRoutesMixin: store each workflow once, then Reclaim space - DuplicateFilesMixin: duplicate image files as find and removal jobs - LoraImportRoutesMixin: the reviewed, staged LoRA reference import job @@ -42,6 +43,7 @@ from PIL import Image from .prompts import PromptRoutesMixin from .pairs import PairRoutesMixin from .library import LibraryRoutesMixin +from .library_delete import LibraryDeleteRoutesMixin from .vocabulary import VocabularyRoutesMixin from .annotations import AnnotationRoutesMixin from .review import ReviewRoutesMixin @@ -313,6 +315,7 @@ async def _gzip_middleware(request, handler): class PromptManagerAPI( PairRoutesMixin, LibraryRoutesMixin, + LibraryDeleteRoutesMixin, VocabularyRoutesMixin, AnnotationRoutesMixin, ReviewRoutesMixin, @@ -687,6 +690,7 @@ class PromptManagerAPI( self._register_pair_routes(routes) self._register_library_routes(routes) + self._register_library_delete_routes(routes) self._register_vocabulary_routes(routes) self._register_annotation_routes(routes) self._register_review_routes(routes) diff --git a/py/api/jobs.py b/py/api/jobs.py index de92fba8..c538b996 100644 --- a/py/api/jobs.py +++ b/py/api/jobs.py @@ -123,6 +123,9 @@ MODEL_DOWNLOAD_JOB_KIND = "model_download" DUPLICATES_JOB_KIND = "duplicates" DUPLICATE_GROUP_NAMES = ("groups_found",) DUPLICATE_CLEANUP_GROUP_NAMES = ("removed", "kept", "errors") +# The workspace Library's Delete… (#443, py/api/library_delete.py): a +# ``records`` hold around its safety backup and write. +PROMPT_DELETE_JOB_KIND = "prompt_delete" # The System health check (py/api/system.py): a read-only ``read`` job with # no file outcomes, so it may run beside a scan. HEALTH_JOB_KIND = "health" diff --git a/py/api/library.py b/py/api/library.py index 2e3dc4b5..97453a76 100644 --- a/py/api/library.py +++ b/py/api/library.py @@ -155,6 +155,13 @@ def _archive_scope(request: web.Request) -> str: return value +def _shelf(request: web.Request) -> str: + value = request.query.get("shelf", "active") + if value not in ("active", "archived"): + raise ValueError("shelf must be active or archived") + return value + + def _missing() -> web.Response: return web.json_response( {"success": False, "error": "Positive prompt not found"}, status=404 @@ -295,6 +302,9 @@ class LibraryRoutesMixin: "category": request.query.get("category"), "tags": sorted(set(request.query.getall("tag", []))), "sort": request.query.get("sort", "newest"), + # #443: which shelf; part of the cursor scope, so a cursor never + # crosses shelves. Not the variant ``archive`` scope of /{id}. + "shelf": _shelf(request), } limit = _page_limit(request) cursor = request.query.get("cursor") diff --git a/py/api/library_delete.py b/py/api/library_delete.py new file mode 100644 index 00000000..34038e04 --- /dev/null +++ b/py/api/library_delete.py @@ -0,0 +1,263 @@ +"""Delete or archive selected Library prompts, and restore archived ones (#443). + +The workspace Prompt library's Delete… (D152). The legacy ``POST +/prompt_manager/bulk/delete`` and ``DELETE /prompt_manager/delete/{id}`` are +separate and unchanged. + +- ``POST /prompt_manager/library/delete/preview {prompt_ids, expected_epoch}`` + classifies the selection (reads only, under a lease): the ids to ``delete`` + (in no saved pair), to ``archive`` (paired), the ``protected`` ones (tagged + ``__protected__``, kept), counts of ``already_archived`` and ``not_found``, + the ``image_links`` a delete removes, the safety ``backup`` policy and, + when a backup would be taken, the ``disk`` it needs. +- ``POST /prompt_manager/library/delete {archive_ids, delete_ids, + expected_epoch, backup}`` executes that review under a ``records`` hold + (refused 409 ``busy`` beside a scan, restore, maintenance, move or + Optimize). When a delete remains and a backup was promised (the reviewed + ``backup`` was "on" or the policy is on now) it checks the disk (507 + ``insufficient-space``) and takes a verified safety backup (500 + ``backup-failed``); then one transaction deletes the reviewed deletes whose + fingerprint is unchanged since the backup and archives the paired ones. +- ``POST /prompt_manager/library/restore {prompt_ids, expected_epoch}`` + removes the archive mark (under a lease). + +A changed storage epoch answers 409 ``storage-changed``; nothing is written. +""" + +import sqlite3 + +from aiohttp import web + +try: + from ...database.library_commands import LibraryCommandError + from ...database.safety_backup import SafetyBackupFailed +except ImportError: + from database.library_commands import LibraryCommandError + from database.safety_backup import SafetyBackupFailed + +from . import maintenance as _maintenance +from .admin import _prune_safety_backups +from .job_routes import busy_response +from .jobs import PROMPT_DELETE_JOB_KIND, JobBusy +from .prompts import parse_bulk_prompt_ids + +PREVIEW_URL = "/prompt_manager/library/delete/preview" +DELETE_URL = "/prompt_manager/library/delete" +RESTORE_URL = "/prompt_manager/library/restore" +DELETE_TITLE = "Delete prompts (Library)" +_SELECTION_KEYS = frozenset({"prompt_ids", "expected_epoch"}) +_DELETE_KEYS = frozenset({"archive_ids", "delete_ids", "expected_epoch", "backup"}) +STORAGE_CHANGED = "Prompt storage was restored; refresh the library" + + +def _failure(status, code, error, **extra): + return web.json_response( + {"success": False, "code": code, "error": error, **extra}, status=status + ) + + +def _invalid(message): + return _failure(400, "invalid-request", message) + + +def _storage_changed(exc): + return _failure(409, exc.code, STORAGE_CHANGED, library_epoch=exc.epoch) + + +async def _body(request, keys): + """The JSON object with exactly ``keys``, or raise ValueError.""" + try: + data = await request.json() + except ValueError: + raise ValueError("Expected a JSON object") from None + if not isinstance(data, dict): + raise ValueError("Expected a JSON object") + if set(data) != keys: + raise ValueError("Request must contain exactly " + ", ".join(sorted(keys))) + return data + + +def _reviewed_ids(value, name): + """A reviewed id list: possibly empty, else as ``parse_bulk_prompt_ids``.""" + if value == []: + return [] + try: + return parse_bulk_prompt_ids(value) + except ValueError as exc: + raise ValueError(str(exc).replace("prompt_ids", name)) from None + + +class LibraryDeleteRoutesMixin: + """Mixin providing the workspace Library's delete, archive and restore.""" + + def _register_library_delete_routes(self, routes): + routes.post(PREVIEW_URL)(self.preview_library_delete) + routes.post(DELETE_URL)(self.delete_library_prompts) + routes.post(RESTORE_URL)(self.restore_library_prompts) + + async def preview_library_delete(self, request): + try: + data = await _body(request, _SELECTION_KEYS) + ids = parse_bulk_prompt_ids(data["prompt_ids"]) + except ValueError as exc: + return _invalid(str(exc)) + db = self.db + policy = self._safety_backup_policy() + try: + async with self._jobs.lease("Library delete review"): + review = await self._run_in_executor( + db.classify_library_delete, + ids, + expected_epoch=data["expected_epoch"], + ) + disk = None + if review["delete"] and policy == "on": + public, short, _size = await self._run_in_executor( + _maintenance.disk_plan, + db.model.db_path, + backup=True, + vacuum=False, + ) + disk = { + **public, + "message": ( + None + if short is None + else _maintenance.short_space_message(short) + ), + } + except JobBusy as busy: + return busy_response(busy) + except LibraryCommandError as exc: + return _storage_changed(exc) + except ValueError as exc: + return _invalid(str(exc)) + except (OSError, sqlite3.Error): + self.logger.exception("Library delete review failed") + return _failure(500, "review-failed", "The delete could not be reviewed") + return web.json_response( + { + "success": True, + "library_epoch": review["library_epoch"], + "delete": review["delete"], + "archive": review["archive"], + "protected": review["protected"], + "already_archived": len(review["already_archived"]), + "not_found": len(review["not_found"]), + "image_links": review["image_links"], + "backup": policy, + "disk": disk, + } + ) + + async def delete_library_prompts(self, request): + try: + data = await _body(request, _DELETE_KEYS) + archive_ids = _reviewed_ids(data["archive_ids"], "archive_ids") + delete_ids = _reviewed_ids(data["delete_ids"], "delete_ids") + if not archive_ids and not delete_ids: + raise ValueError("Nothing to delete or archive") + if len(archive_ids) + len(delete_ids) > 1000: + raise ValueError("At most 1000 prompts can be changed at once") + if set(archive_ids) & set(delete_ids): + raise ValueError("A prompt cannot be both archived and deleted") + if data["backup"] not in ("on", "off"): + raise ValueError("backup must be on or off") + except ValueError as exc: + return _invalid(str(exc)) + busy = self._jobs.blocker(PROMPT_DELETE_JOB_KIND, admission="records") + if busy is not None: + return busy_response(JobBusy(busy)) + try: + async with self._jobs.hold( + PROMPT_DELETE_JOB_KIND, DELETE_TITLE, admission="records" + ) as job: + response = await self._held_library_delete( + self.db, data, archive_ids, delete_ids + ) + if response.status >= 400: + job.finish("failed") + return response + except JobBusy as busy: + return busy_response(busy) + + async def _held_library_delete(self, db, data, archive_ids, delete_ids): + epoch = data["expected_epoch"] + promised = data["backup"] == "on" or self._safety_backup_policy() == "on" + backup = None + try: + candidates = None + if delete_ids: + review = await self._run_in_executor( + db.classify_library_delete, delete_ids, expected_epoch=epoch + ) + candidates = review["fingerprints"] + if candidates and promised: + _disk, short, _size = await self._run_in_executor( + _maintenance.disk_plan, + db.model.db_path, + backup=True, + vacuum=False, + ) + if short is not None: + return _failure( + 507, + "insufficient-space", + _maintenance.short_space_message(short), + ) + backup = await self._run_in_executor( + _maintenance.take_safety_backup, db.model + ) + result = await self._run_in_executor( + db.archive_or_delete_library_prompts, + archive_ids, + delete_ids, + expected_epoch=epoch, + candidates=candidates if candidates is not None else {}, + ) + except LibraryCommandError as exc: + return _storage_changed(exc) + except SafetyBackupFailed as error: + self.logger.error("Safety backup before a Library delete failed") + return _failure( + 500, "backup-failed", _maintenance.backup_failed_message(error) + ) + except ValueError as exc: + return _invalid(str(exc)) + except (OSError, sqlite3.Error): + self.logger.exception("Library delete failed") + return _failure(500, "delete-failed", "The prompts could not be changed") + finally: + if backup is not None: + await self._prune_after_delete(db) + return web.json_response({"success": True, **result, "backup": backup}) + + async def _prune_after_delete(self, db): + try: + await self._run_in_executor(_prune_safety_backups, db.model.db_path) + except Exception: # pruning old copies never fails the delete + self.logger.exception("Could not prune old safety backups") + + async def restore_library_prompts(self, request): + try: + data = await _body(request, _SELECTION_KEYS) + ids = parse_bulk_prompt_ids(data["prompt_ids"]) + except ValueError as exc: + return _invalid(str(exc)) + try: + async with self._jobs.lease("Library restore"): + result = await self._run_in_executor( + self.db.restore_library_prompts, + ids, + expected_epoch=data["expected_epoch"], + ) + except JobBusy as busy: + return busy_response(busy) + except LibraryCommandError as exc: + return _storage_changed(exc) + except ValueError as exc: + return _invalid(str(exc)) + except (OSError, sqlite3.Error): + self.logger.exception("Library restore failed") + return _failure(500, "restore-failed", "The prompts could not be restored") + return web.json_response({"success": True, **result}) diff --git a/tests/test_bulk_library_delete.py b/tests/test_bulk_library_delete.py new file mode 100644 index 00000000..26c4cece --- /dev/null +++ b/tests/test_bulk_library_delete.py @@ -0,0 +1,484 @@ +"""#443 (D152): delete selected Library prompts; paired ones are archived. + +The DB commands, and what every reader does with an archived parent. +""" + +import re +import sqlite3 +import tempfile +import unittest +from pathlib import Path + +from database.execution_history_schema import rotate_history_epoch +from database.library_commands import LibraryCommandError +from database.operations import PromptDatabase +from database.prompt_archive import TABLE as ARCHIVE +from utils.hashing import generate_prompt_hash + + +def save(db, text, **metadata): + return db.save_prompt(text, prompt_hash=generate_prompt_hash(text), **metadata) + + +def rows(db, sql, params=()): + with db.model.get_connection() as conn: + return [tuple(row) for row in conn.execute(sql, params).fetchall()] + + +def archived_ids(db): + return sorted(row[0] for row in rows(db, f"SELECT prompt_id FROM {ARCHIVE}")) + + +def prompt_ids(db): + return sorted(row[0] for row in rows(db, "SELECT id FROM prompts")) + + +_STAMP = re.compile(r"^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}") + + +def _stable(value): + return "