Merge remote-tracking branch 'origin/main' into fix/501-removal-mark-gaps

database/library.py

@@ -7,6 +7,7 @@ from contextlib import contextmanager
 
 from .library_folders import folder_condition, list_library_folders
 from .library_tokens import library_edit_token
+from .prompt_archive import ARCHIVED, SHELVED
 
 _ELIGIBLE = """(EXISTS(SELECT 1 FROM prompt_pairs ep WHERE ep.positive_prompt_id=p.id)
  OR EXISTS(SELECT 1 FROM generated_images eg WHERE eg.prompt_id=p.id)
@@ -106,6 +107,12 @@ ARCHIVE_SCOPES = {
 }
 
 
+# Shelves (#443): the parents on the shelf, or the archived ones.
+PARENT_SCOPES = {"active": SHELVED, "archived": ARCHIVED}
+# A positive pair owner that is on the shelf, as a condition on alias pp.
+SHELVED_PAIR = SHELVED.replace("p.id", "pp.positive_prompt_id")
+
+
 def _escape(value: str) -> str:
     return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
 
@@ -270,6 +277,8 @@ class LibraryOperations:
         rows = conn.execute(
             "SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes,p.created_at,"
             "p.run_count,p.last_used_at,"
+            + ARCHIVED
+            + " AS archived,"
             + (f"{sort_expression} AS _sort," if sort_expression else "")
             + "(SELECT COUNT(*) FROM prompt_pairs v WHERE v.positive_prompt_id=p.id "
             "AND v.archived=0) AS variant_count,"
@@ -293,6 +302,7 @@ class LibraryOperations:
             # Usage is shown, never compared: the token hashes owner fields only.
             row["run_count"] = row["run_count"] if type(row["run_count"]) is int else 0
             row["last_used_at"] = utc_iso(row["last_used_at"])
+            row["archived"] = bool(row["archived"])
             row["edit_token"] = library_edit_token(epoch, row)
         return result
 
@@ -310,10 +320,13 @@ class LibraryOperations:
         sort: str = "newest",
         limit: int = 40,
         after: dict | None = None,
+        shelf: str = "active",
     ) -> dict:
         _limit(limit)
         if sort not in _ORDERS:
             raise ValueError("Unknown library sort order")
+        if not isinstance(shelf, str) or shelf not in PARENT_SCOPES:
+            raise ValueError("shelf must be active or archived")
         where, params = _filter(
             text, category, [] if tags is None else tags, tag_mode, folder_prefixes
         )
@@ -328,15 +341,18 @@ class LibraryOperations:
             ceiling = after["ceiling"]
         where += " AND p.id<=?"
         params.append(ceiling)
-        # One SELECT counts the filtered parents and their active variants.
-        total, variant_total = conn.execute(
-            "SELECT COUNT(*),COALESCE(SUM((SELECT COUNT(*) FROM prompt_pairs v "
-            "WHERE v.positive_prompt_id=p.id AND v.archived=0)),0) FROM "
-            + _FROM
-            + " WHERE "
-            + where,
+        # One SELECT counts the filtered parents of this scope, their active
+        # variants, and the filtered archived parents (Show archived, #443).
+        scope = PARENT_SCOPES[shelf]
+        total, variant_total, archived_total = conn.execute(
+            f"SELECT COALESCE(SUM({scope}),0),"
+            f"COALESCE(SUM(CASE WHEN {scope} THEN (SELECT COUNT(*) "
+            "FROM prompt_pairs v WHERE v.positive_prompt_id=p.id "
+            "AND v.archived=0) ELSE 0 END),0),"
+            f"COALESCE(SUM({ARCHIVED}),0) FROM " + _FROM + " WHERE " + where,
             params,
         ).fetchone()
+        where += f" AND {scope}"
         expression, direction = _ORDERS[sort]
         if after is not None:
             operator = ">" if direction == "ASC" else "<"
@@ -369,6 +385,7 @@ class LibraryOperations:
             "items": rows,
             "total": total,
             "variant_total": variant_total,
+            "archived_total": archived_total,
             # Echoed so a client can tell this read applied its tag match.
             "tag_mode": tag_mode,
             "has_more": has_more,
@@ -386,6 +403,7 @@ class LibraryOperations:
         sort: str = "newest",
         limit: int = 40,
         after: dict | None = None,
+        shelf: str = "active",
     ) -> dict:
         with self.library_snapshot(include_ceiling=True) as (conn, epoch, ceiling):
             return self._list_library(
@@ -400,6 +418,7 @@ class LibraryOperations:
                 sort=sort,
                 limit=limit,
                 after=after,
+                shelf=shelf,
             )
 
     def _is_library_positive(self, conn: sqlite3.Connection, positive_id: int) -> bool:
@@ -644,7 +663,9 @@ class LibraryOperations:
             owners = {}
             order = []
             for row in conn.execute(
-                "SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes FROM "
+                "SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes,"
+                + ARCHIVED
+                + " AS archived FROM "
                 + _FROM
                 + " WHERE "
                 + _ELIGIBLE
@@ -657,6 +678,7 @@ class LibraryOperations:
                     "tags": [],
                     "rating": row["rating"],
                     "notes": row["notes"],
+                    "archived": bool(row["archived"]),
                     "variants": [],
                 }
                 order.append(row["id"])
@@ -701,8 +723,9 @@ class LibraryOperations:
         """System › Overview counts from one snapshot, writing nothing.
 
         ``positive_prompts`` and ``negative_variants`` are the unfiltered
-        Library's ``total`` and ``variant_total``: eligible parents and active
-        pairs (every pair's positive is eligible). ``images`` counts indexed
+        Library's ``total`` and ``variant_total``: eligible parents on the
+        shelf (not archived, #443) and their active pairs (every pair's
+        positive is eligible). ``images`` counts indexed
         ``generated_images`` rows, not files on disk. An image has no rating
         of its own (it shows its linked prompt's), so ``rated_images`` counts
         the indexed images whose linked prompt is rated. ``review_pending`` is
@@ -710,8 +733,14 @@ class LibraryOperations:
         """
         with self.library_snapshot() as (conn, _epoch, _ceiling):
             row = conn.execute(
-                "SELECT (SELECT COUNT(*) FROM prompts p WHERE " + _ELIGIBLE + "),"
-                "(SELECT COUNT(*) FROM prompt_pairs WHERE archived=0),"
+                "SELECT (SELECT COUNT(*) FROM prompts p WHERE "
+                + _ELIGIBLE
+                + " AND "
+                + SHELVED
+                + "),"
+                "(SELECT COUNT(*) FROM prompt_pairs pp WHERE pp.archived=0 AND "
+                + SHELVED_PAIR
+                + "),"
                 "(SELECT COUNT(*) FROM generated_images),"
                 "(SELECT COUNT(*) FROM generated_images gi JOIN prompts r "
                 "ON r.id=gi.prompt_id WHERE r.rating IS NOT NULL),"

database/library_archive.py

@@ -0,0 +1,289 @@
+"""Delete selected Library prompts; archive the paired ones (#443, D152).
+
+A selected Library parent is classified, in this order, as ``not_found``
+(missing, or not a Library parent), ``protected`` (tagged ``__protected__``;
+never changed), ``already_archived``, ``archive`` (the positive or negative of
+any saved pair, archived pair or not: archiving hides it from the shelf and
+removes nothing) or ``delete`` (in no pair: deleted exactly as the legacy
+delete does, through ``delete_prompt_rows``). Every command checks the
+storage epoch first and runs in one ``BEGIN IMMEDIATE`` transaction.
+
+The write executes what the user reviewed: it is given the reviewed archive
+and delete lists, and deletes only reviewed deletes the route backed up
+(``candidates``) whose fingerprint is unchanged since that backup.
+"""
+
+import sqlite3
+from typing import Iterable
+
+from .library import _ELIGIBLE
+from .library_commands import _validate_bulk_ids, _validate_epoch
+from .models import utc_now_iso
+from .prompt_archive import ARCHIVED as _ARCHIVED
+from .prompt_archive import TABLE as ARCHIVE
+from .workflow_documents import documents_of_rows, release_documents
+
+try:
+    from ..utils.hashing import generate_prompt_hash
+except ImportError:
+    from utils.hashing import generate_prompt_hash
+
+# Ids bound per statement: the pair check binds each id twice, which stays
+# under SQLite's historical 999-variable limit (maintenance.py ID_BATCH).
+ID_BATCH = 400
+_PROTECTED = (
+    "EXISTS(SELECT 1 FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id "
+    "WHERE pt.prompt_id=p.id AND t.name='__protected__')"
+)
+_PAIRED = (
+    "EXISTS(SELECT 1 FROM prompt_pairs pp "
+    "WHERE pp.positive_prompt_id=p.id OR pp.negative_prompt_id=p.id)"
+)
+# Classes in the order a prompt is tested; the first that holds wins.
+CLASSES = ("not_found", "protected", "already_archived", "archive", "delete")
+
+
+def _batches(ids: list) -> Iterable[list]:
+    for start in range(0, len(ids), ID_BATCH):
+        yield ids[start : start + ID_BATCH]
+
+
+def delete_prompt_rows(conn: sqlite3.Connection, prompt_ids: list[int]) -> int:
+    """Delete prompts and their image link rows; return the prompts deleted.
+
+    The legacy delete's statements (``delete_prompt``, ``bulk_delete_prompts``)
+    in the caller's transaction: image link rows first (``image_prompt_pairs``
+    cascades, review decisions keep a NULL image), then the prompts (tags and
+    Library state cascade), then the workflow documents no row references.
+    Image files are never touched. The caller checks pairs first.
+    """
+    documents = documents_of_rows(conn, "prompt_id", prompt_ids)
+    deleted = 0
+    for chunk in _batches(list(prompt_ids)):
+        marks = ",".join("?" for _ in chunk)
+        conn.execute(
+            f"DELETE FROM generated_images WHERE prompt_id IN ({marks})",  # nosec
+            chunk,
+        )
+        deleted += conn.execute(
+            f"DELETE FROM prompts WHERE id IN ({marks})", chunk  # nosec B608
+        ).rowcount
+    release_documents(conn, documents)
+    return deleted
+
+
+class LibraryArchiveOperations:
+    """PromptDatabase mixin: classify, archive or delete, and restore parents."""
+
+    def _classify_library_delete(
+        self, conn: sqlite3.Connection, prompt_ids: list[int]
+    ) -> dict:
+        found = {}
+        for chunk in _batches(prompt_ids):
+            marks = ",".join("?" for _ in chunk)
+            for row in conn.execute(
+                f"SELECT p.id,{_PROTECTED},{_ARCHIVED},{_PAIRED} "  # nosec B608
+                f"FROM prompts p WHERE p.id IN ({marks}) AND {_ELIGIBLE}",
+                chunk,
+            ):
+                protected, archived, paired = row[1], row[2], row[3]
+                if protected:
+                    found[row[0]] = "protected"
+                elif archived:
+                    found[row[0]] = "already_archived"
+                else:
+                    found[row[0]] = "archive" if paired else "delete"
+        classes = {name: [] for name in CLASSES}
+        for pid in prompt_ids:
+            classes[found.get(pid, "not_found")].append(pid)
+        return classes
+
+    @staticmethod
+    def _image_links(conn: sqlite3.Connection, prompt_ids: list[int]) -> int:
+        total = 0
+        for chunk in _batches(prompt_ids):
+            marks = ",".join("?" for _ in chunk)
+            total += conn.execute(
+                "SELECT COUNT(*) FROM generated_images "  # nosec B608
+                f"WHERE prompt_id IN ({marks})",
+                chunk,
+            ).fetchone()[0]
+        return total
+
+    @staticmethod
+    def _fingerprints(conn: sqlite3.Connection, prompt_ids: list[int]) -> dict:
+        """What a safety backup holds of each prompt a delete would remove.
+
+        The prompt row (usage included), its tag ids, its Library title and
+        its image link rows (count and newest id), as a JSON-safe list.
+        """
+        prints = {}
+        for chunk in _batches(prompt_ids):
+            marks = ",".join("?" for _ in chunk)
+            for row in conn.execute(
+                "SELECT p.id,p.text,p.hash,p.category,p.rating,p.notes,"  # nosec
+                "p.updated_at,p.run_count,p.last_used_at,"
+                "(SELECT group_concat(tag_id) FROM (SELECT tag_id FROM prompt_tags "
+                "WHERE prompt_id=p.id ORDER BY tag_id)),"
+                "(SELECT title FROM prompt_library_state WHERE prompt_id=p.id),"
+                "(SELECT COUNT(*) FROM generated_images WHERE prompt_id=p.id),"
+                "(SELECT MAX(id) FROM generated_images WHERE prompt_id=p.id) "
+                f"FROM prompts p WHERE p.id IN ({marks})",
+                chunk,
+            ):
+                prints[row[0]] = [
+                    value if isinstance(value, (int, str, type(None))) else repr(value)
+                    for value in tuple(row)[1:]
+                ]
+        return prints
+
+    @staticmethod
+    def _checked_ids(prompt_ids: list[int], expected_epoch: str) -> list[int]:
+        _validate_epoch(expected_epoch)
+        _validate_bulk_ids(prompt_ids)
+        return list(dict.fromkeys(prompt_ids))
+
+    def classify_library_delete(
+        self, prompt_ids: list[int], *, expected_epoch: str
+    ) -> dict:
+        """What a delete of ``prompt_ids`` would do now (reads only).
+
+        Returns the id lists per class, in request order, ``image_links``
+        (the image link rows the ``delete`` class would remove) and
+        ``fingerprints`` of the ``delete`` class (``_fingerprints``).
+        """
+        prompt_ids = self._checked_ids(prompt_ids, expected_epoch)
+        with self.library_snapshot() as (conn, _epoch, _ceiling):
+            epoch = self._library_epoch(conn, expected_epoch)
+            classes = self._classify_library_delete(conn, prompt_ids)
+            return {
+                "library_epoch": epoch,
+                **classes,
+                "image_links": self._image_links(conn, classes["delete"]),
+                "fingerprints": self._fingerprints(conn, classes["delete"]),
+            }
+
+    @staticmethod
+    def _repair_hash(conn: sqlite3.Connection, prompt_id: int) -> None:
+        """Give an archived prompt its canonical hash, so ingestion finds it.
+
+        Scans, the monitor and node saves resolve prompts by hash; a stale or
+        missing one would make them insert a visible copy (Q3). When another
+        row already holds the canonical hash, lookups find that row and no new
+        copy is made, so nothing is written.
+        """
+        text, stored = conn.execute(
+            "SELECT text,hash FROM prompts WHERE id=?", (prompt_id,)
+        ).fetchone()
+        canonical = generate_prompt_hash(text) if isinstance(text, str) else None
+        if canonical is None or stored == canonical:
+            return
+        if conn.execute(
+            "SELECT 1 FROM prompts WHERE hash=? AND id!=?", (canonical, prompt_id)
+        ).fetchone():
+            return
+        conn.execute("UPDATE prompts SET hash=? WHERE id=?", (canonical, prompt_id))
+
+    def archive_or_delete_library_prompts(
+        self,
+        archive_ids: list[int],
+        delete_ids: list[int],
+        *,
+        expected_epoch: str,
+        candidates: dict | None,
+    ) -> dict:
+        """Archive and delete what the user reviewed, all or nothing.
+
+        ``archive_ids`` and ``delete_ids`` are the reviewed classes.
+        ``candidates`` maps each reviewed delete the route backed up to its
+        fingerprint then (None: every reviewed delete, unchecked; tests and
+        callers without a backup). A reviewed delete is deleted only when it
+        still classifies ``delete`` and its fingerprint is unchanged; one that
+        is now paired, and a reviewed archive now unpaired, are archived (the
+        restorable direction); anything else is kept and counted.
+        """
+        _validate_epoch(expected_epoch)
+        for ids in (archive_ids, delete_ids):
+            if not isinstance(ids, list) or any(
+                type(pid) is not int or not 1 <= pid <= 2**63 - 1 for pid in ids
+            ):
+                raise ValueError("prompt ids must be lists of positive integers")
+        reviewed_delete = set(delete_ids)
+        if not archive_ids and not delete_ids:
+            raise ValueError("Nothing to delete or archive")
+        if reviewed_delete & set(archive_ids):
+            raise ValueError("A prompt cannot be both archived and deleted")
+        prompt_ids = list(dict.fromkeys([*archive_ids, *delete_ids]))
+        with self._pair_transaction() as conn:
+            epoch = self._library_epoch(conn, expected_epoch)
+            classes = self._classify_library_delete(conn, prompt_ids)
+            now_prints = self._fingerprints(conn, classes["delete"])
+            deletes, archives, changed = [], list(classes["archive"]), 0
+            for pid in classes["delete"]:
+                if pid not in reviewed_delete:
+                    archives.append(pid)  # reviewed as archive, now unpaired
+                elif candidates is None or candidates.get(pid) == now_prints[pid]:
+                    deletes.append(pid)
+                else:
+                    changed += 1
+            changed += len(classes["protected"])
+            image_links = self._image_links(conn, deletes)
+            now = utc_now_iso()
+            for pid in archives:
+                self._repair_hash(conn, pid)
+            conn.executemany(
+                f"INSERT INTO {ARCHIVE}(prompt_id, archived_at) VALUES (?, ?)",
+                [(pid, now) for pid in archives],
+            )
+            # A changed row is the newest edit the restore fingerprint reads.
+            conn.executemany(
+                "UPDATE prompts SET updated_at=? WHERE id=?",
+                [(now, pid) for pid in archives],
+            )
+            deleted = delete_prompt_rows(conn, deletes)
+            return {
+                "library_epoch": epoch,
+                "deleted": deleted,
+                "archived": len(archives),
+                "already_archived": len(classes["already_archived"]),
+                "not_found": len(classes["not_found"]),
+                "changed": changed,
+                "image_links": image_links,
+            }
+
+    def restore_library_prompts(
+        self, prompt_ids: list[int], *, expected_epoch: str
+    ) -> dict:
+        """Put archived prompts back on the shelf, all or nothing.
+
+        Every requested id with an archive row is restored, whatever its
+        protection or Library parent state: restoring only un-hides it.
+        """
+        prompt_ids = self._checked_ids(prompt_ids, expected_epoch)
+        with self._pair_transaction() as conn:
+            epoch = self._library_epoch(conn, expected_epoch)
+            restored, existing = [], set()
+            for chunk in _batches(prompt_ids):
+                marks = ",".join("?" for _ in chunk)
+                for pid, archived in conn.execute(
+                    f"SELECT p.id,{_ARCHIVED} FROM prompts p "  # nosec B608
+                    f"WHERE p.id IN ({marks})",
+                    chunk,
+                ):
+                    existing.add(pid)
+                    if archived:
+                        restored.append(pid)
+            now = utc_now_iso()
+            conn.executemany(
+                f"DELETE FROM {ARCHIVE} WHERE prompt_id=?", [(pid,) for pid in restored]
+            )
+            conn.executemany(
+                "UPDATE prompts SET updated_at=? WHERE id=?",
+                [(now, pid) for pid in restored],
+            )
+            return {
+                "library_epoch": epoch,
+                "restored": len(restored),
+                "unchanged": len(existing) - len(restored),
+                "not_found": len(prompt_ids) - len(existing),
+            }

database/maintenance.py

@@ -89,6 +89,12 @@ PROTECTIONS = (
         "with Library state (a title or an explicit parent)",
         "with Library state (a title or an explicit parent)",
     ),
+    (
+        # #443: an archived prompt is restorable, never pruned.
+        "EXISTS (SELECT 1 FROM prompt_archive pa WHERE pa.prompt_id = p.id)",
+        "archived in the Library",
+        "archived in the Library",
+    ),
     (
         "p.id IN (SELECT prompt_id FROM live_hint_prompts)",
         "used by recorded runs",
@@ -97,7 +103,8 @@ PROTECTIONS = (
 )
 
 # Exempt references: images, pairs, the __protected__ tag, Library parent
-# state, and live recorded-run hints (computed once, see LIVE_HINT_CTES).
+# state, the Library archive, and live recorded-run hints (computed once,
+# see LIVE_HINT_CTES).
 ORPHANED_PROMPTS_SQL = (
     "WITH "
     + LIVE_HINT_CTES

database/models.py

@@ -45,6 +45,7 @@ except ImportError:
     from utils.hashing import register_prompt_functions
 from .image_categories import image_categories_schema_error, migrate_image_categories
 from .image_removal_schema import image_removals_schema_error, migrate_image_removals
+from .prompt_archive import migrate_prompt_archive, prompt_archive_schema_error
 from .workflow_documents import (
     DOCUMENT_ID_COLUMNS,
     migrate_workflow_documents,
@@ -367,6 +368,8 @@ class PromptModel:
         migrate_autotag_state(conn)
         # Image categories (#332): independent of every table above.
         migrate_image_categories(conn)
+        # The Library parent archive (#443): independent of every table above.
+        migrate_prompt_archive(conn)
         # Removal marks (#480): independent of every table above.
         migrate_image_removals(conn)
         # Last: the older generated_images rebuilds run only before file_path.
@@ -1245,6 +1248,11 @@ class PromptModel:
             )
             if categories_error:
                 return False, categories_error
+            archive_error = prompt_archive_schema_error(
+                conn, allow_missing=not require_library_state
+            )
+            if archive_error:
+                return False, archive_error
             removals_error = image_removals_schema_error(
                 conn, allow_missing=not require_library_state
             )

database/operations.py

@@ -25,6 +25,7 @@ from .review import ReviewOperations
 from .review_decisions import ReviewDecisionOperations
 from .saved_views import SavedViewOperations
 from .library_commands import LibraryCommandOperations
+from .library_archive import LibraryArchiveOperations, delete_prompt_rows
 from .vocabulary import VocabularyOperations
 from .execution_history import ExecutionHistoryOperations
 from .execution_contexts import ExecutionContextOperations
@@ -449,6 +450,7 @@ class PromptDatabase(
     PairOperations,
     LibraryOperations,
     LibraryCommandOperations,
+    LibraryArchiveOperations,
     VocabularyOperations,
     ExecutionHistoryOperations,
     ExecutionContextOperations,
@@ -2351,16 +2353,10 @@ class PromptDatabase(
         Returns:
             Number of prompts actually deleted.
         """
-        count = 0
         with self._pair_transaction() as conn:
             self._ensure_prompts_unpaired(conn, prompt_ids)
-            documents = documents_of_rows(conn, "prompt_id", prompt_ids)
-            for pid in prompt_ids:
-                conn.execute("DELETE FROM generated_images WHERE prompt_id = ?", (pid,))
-                cursor = conn.execute("DELETE FROM prompts WHERE id = ?", (pid,))
-                if cursor.rowcount > 0:
-                    count += 1
-            release_documents(conn, documents)
+            # The workspace Library's delete runs the same statements (#443).
+            count = delete_prompt_rows(conn, list(dict.fromkeys(prompt_ids)))
             conn.commit()
         return count
 

database/prompt_archive.py

@@ -0,0 +1,107 @@
+"""Library parent archive (#443, D152): prompts hidden from the shelf, restorable.
+
+A row means the Library parent is archived; no row means it is on the shelf.
+Nothing else about the prompt changes: its text, metadata, tags, run count,
+image links, pairs and recorded runs stay. Restore deletes the row. Deleting
+the prompt (any path) drops the row by cascade.
+
+A table of its own, not a column: ``prompt_library_state`` is validated by its
+exact definition, so a new column there would make older builds refuse the
+upgraded database, while older builds ignore an unknown table (they show the
+archived prompts again and keep the rows). Optional in old backups, required
+once a database is current.
+"""
+
+import re
+import sqlite3
+
+TABLE = "prompt_archive"
+MAX_STAMP_BYTES = 64
+# What ``utc_now_iso`` writes starts with a date; anything else was not written here.
+_STAMP_GLOB = "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T*"
+_DDL = (
+    f"CREATE TABLE {TABLE} (prompt_id INTEGER PRIMARY KEY, "
+    "archived_at TEXT NOT NULL, "
+    "FOREIGN KEY(prompt_id) REFERENCES prompts(id) ON DELETE CASCADE)"
+)
+_COLUMNS = (("prompt_id", "INTEGER", 0, 1), ("archived_at", "TEXT", 1, 0))
+_FOREIGN_KEYS = {("prompt_id", "prompts", "id", "CASCADE")}
+# A predicate on ``p`` (the prompts alias): the parent is not archived.
+SHELVED = f"NOT EXISTS(SELECT 1 FROM {TABLE} pa WHERE pa.prompt_id=p.id)"
+ARCHIVED = f"EXISTS(SELECT 1 FROM {TABLE} pa WHERE pa.prompt_id=p.id)"
+
+
+def _compact(sql: str) -> str:
+    return re.sub(r"\s+", "", sql or "").lower()
+
+
+def _structure_error(conn: sqlite3.Connection, allow_missing: bool) -> str | None:
+    objects = conn.execute(
+        "SELECT type,sql FROM sqlite_master WHERE name=? "
+        "OR (type IN ('index','trigger') AND tbl_name=?)",
+        (TABLE, TABLE),
+    ).fetchall()
+    if not objects:
+        return None if allow_missing else "Missing prompt archive schema"
+    tables = [sql for kind, sql in objects if kind == "table"]
+    if len(tables) != 1 or len(objects) != 1:
+        return "Prompt archive name is reserved by an unsupported schema object"
+    if _compact(tables[0]) != _compact(_DDL):
+        return "Prompt archive has an invalid table definition"
+    columns = tuple(
+        (row[1], row[2].upper(), row[3], row[5])
+        for row in conn.execute(f"PRAGMA table_info({TABLE})")
+    )
+    if columns != _COLUMNS:
+        return "Prompt archive has invalid columns"
+    foreign_keys = {
+        (row[3], row[2], row[4], row[6])
+        for row in conn.execute(f"PRAGMA foreign_key_list({TABLE})")
+    }
+    if foreign_keys != _FOREIGN_KEYS:
+        return "Prompt archive has invalid foreign key definitions"
+    return None
+
+
+def prompt_archive_schema_error(
+    conn: sqlite3.Connection, *, allow_missing: bool = True
+) -> str | None:
+    """Reject an unsupported archive schema or values; reads only.
+
+    A missing table is acceptable for backups that predate it unless
+    ``allow_missing`` is false.
+    """
+    error = _structure_error(conn, allow_missing)
+    if error is not None:
+        return error
+    if not conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
+    ).fetchone():
+        return None
+    if conn.execute(f"PRAGMA foreign_key_check({TABLE})").fetchone():
+        return "Prompt archive contains foreign key violations"
+    if conn.execute(
+        f"SELECT 1 FROM {TABLE} WHERE typeof(archived_at)!='text' "  # nosec B608
+        f"OR length(CAST(archived_at AS BLOB))>{MAX_STAMP_BYTES} "
+        f"OR archived_at NOT GLOB '{_STAMP_GLOB}' LIMIT 1"
+    ).fetchone():
+        return "Prompt archive contains invalid values"
+    return None
+
+
+def migrate_prompt_archive(conn: sqlite3.Connection) -> None:
+    """Install the table atomically; validate before and after (no backfill)."""
+    conn.execute("SAVEPOINT prompt_archive_schema")
+    try:
+        error = _structure_error(conn, True)
+        if error:
+            raise sqlite3.DatabaseError(error)
+        conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
+        error = _structure_error(conn, False)
+        if error:
+            raise sqlite3.DatabaseError(error)
+        conn.execute("RELEASE prompt_archive_schema")
+    except BaseException:
+        conn.execute("ROLLBACK TO prompt_archive_schema")
+        conn.execute("RELEASE prompt_archive_schema")
+        raise

database/restore_stage.py

@@ -149,6 +149,7 @@ def library_summary(conn: sqlite3.Connection) -> Dict[str, int]:
     collections (#331).
     """
     from .library import _ELIGIBLE
+    from .prompt_archive import SHELVED, TABLE as ARCHIVE
 
     has_state = conn.execute(
         "SELECT 1 FROM sqlite_master WHERE type='table' AND "
@@ -157,9 +158,15 @@ def library_summary(conn: sqlite3.Connection) -> Dict[str, int]:
     has_pairs = conn.execute(
         "SELECT 1 FROM sqlite_master WHERE type='table' AND name='prompt_pairs'"
     ).fetchone()
+    has_archive = conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (ARCHIVE,)
+    ).fetchone()
     if has_state and has_pairs:
+        # Archived parents (#443) are off the shelf, as in library_counts.
         positives = conn.execute(
-            "SELECT COUNT(*) FROM prompts p WHERE " + _ELIGIBLE
+            "SELECT COUNT(*) FROM prompts p WHERE "
+            + _ELIGIBLE
+            + (" AND " + SHELVED if has_archive else "")
         ).fetchone()[0]
     else:
         positives = _count(conn, "prompts")

database/vocabulary.py

@@ -20,7 +20,8 @@ counted, hashed into the token and renamed with the prompts and pairs.
 
 import sqlite3
 
-from .library import _ELIGIBLE, UNTAGGED, _escape, _limit
+from .library import _ELIGIBLE, SHELVED_PAIR, UNTAGGED, _escape, _limit
+from .prompt_archive import SHELVED
 from .library_commands import _TOKEN, LibraryCommandError, _validate_epoch
 from .library_tokens import canonical_token
 from .models import utc_now_iso
@@ -40,17 +41,21 @@ _IMAGE_FILES = (
 
 def _tag_rows(page: str, order: str = "page.id") -> str:
     """Count rows for the registry rows selected by ``page`` (id, name)."""
-    # Eligible parent components carrying a tag in the page, plus the
-    # positive parent of every active pair carrying it (always eligible).
+    # Eligible parent components on the shelf carrying a tag in the page,
+    # plus the positive parent of every active pair carrying it (always
+    # eligible) when that parent is on the shelf: archived parents (#443)
+    # are hidden with their pairs, so Browse lists what was counted.
     return (
         "WITH page AS (" + page + "),"
         "owners(tag_id,pid) AS ("
         "SELECT pt.tag_id,p.id FROM prompts p "
         "JOIN prompt_tags pt ON pt.prompt_id=p.id "
         "WHERE pt.tag_id IN (SELECT id FROM page) AND " + _ELIGIBLE + " "
+        "AND " + SHELVED + " "
         "UNION SELECT ppt.tag_id,pp.positive_prompt_id FROM prompt_pairs pp "
         "JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id "
-        "WHERE pp.archived=0 AND ppt.tag_id IN (SELECT id FROM page)),"
+        "WHERE pp.archived=0 AND " + SHELVED_PAIR + " "
+        "AND ppt.tag_id IN (SELECT id FROM page)),"
         "images(tag_id,n) AS (" + _IMAGE_FILES + " "
         "WHERE mit.tag_id IN (SELECT id FROM page) GROUP BY mit.tag_id) "
         "SELECT page.id,page.name,COUNT(DISTINCT owners.pid) AS prompt_count,"
@@ -79,8 +84,10 @@ _RANKED_TAG_PAGE = (
     "WITH owners(tag_id,pid) AS ("
     "SELECT pt.tag_id,p.id FROM prompts p "
     "JOIN prompt_tags pt ON pt.prompt_id=p.id WHERE " + _ELIGIBLE + " "
+    "AND " + SHELVED + " "
     "UNION SELECT ppt.tag_id,pp.positive_prompt_id FROM prompt_pairs pp "
-    "JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id WHERE pp.archived=0),"
+    "JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id WHERE pp.archived=0 "
+    "AND " + SHELVED_PAIR + "),"
     "counted AS (SELECT t.id,t.name,COUNT(DISTINCT o.pid) AS prompt_count "
     "FROM tags t LEFT JOIN owners o ON o.tag_id=t.id WHERE {search} "
     "GROUP BY t.id),"
@@ -94,7 +101,13 @@ _RANKED_TAG_PAGE = (
     "ORDER BY page.prompt_count {direction},page.id"
 )
 _UNTAGGED_COUNT = (
-    "SELECT COUNT(*) FROM prompts p WHERE (" + _ELIGIBLE + ") AND " + UNTAGGED
+    "SELECT COUNT(*) FROM prompts p WHERE ("
+    + _ELIGIBLE
+    + ") AND "
+    + UNTAGGED
+    # Untagged prompts opens the shelf, which hides archived parents (#443).
+    + " AND "
+    + SHELVED
 )
 _TAG_ONE = _tag_rows("SELECT id,name FROM tags WHERE id=?")
 # Every owner kind a tag can sit on, for the delete confirmation.
@@ -134,7 +147,9 @@ def _validate_tag_id(tag_id: object) -> None:
 
 
 # Category values in Tags & categories (#332 §7): the prompt-backed values
-# (nonblank values on eligible parents or on active pairs, grouped exactly, so
+# (nonblank values on eligible parents or on active pairs; ``prompt_count``
+# counts only parents on the shelf, so a value only archived parents (#443)
+# hold stays listed with 0, as a tag does; grouped exactly, so
 # whitespace and case variants stay distinct; /library/options lists the same
 # prompt-backed set with its own query) plus the values only image rows hold.
 # A prompt-backed value is ordered by its oldest owning positive; an image-only
@@ -143,15 +158,17 @@ def _validate_tag_id(tag_id: object) -> None:
 # files missing on disk. No FULL OUTER JOIN, so it runs on SQLite 3.31+.
 _IMAGE_ONLY_FIRST_PID = 2**63 - 1
 _CATEGORY_ROWS = (
-    "WITH owners(value,pid) AS ("
-    "SELECT p.category,p.id FROM prompts p WHERE p.category IS NOT NULL "
-    "AND TRIM(p.category)!='' AND " + _ELIGIBLE + " "
-    "UNION SELECT pp.category,pp.positive_prompt_id FROM prompt_pairs pp "
+    "WITH owners(value,pid,shelved) AS ("
+    "SELECT p.category,p.id," + SHELVED + " FROM prompts p "
+    "WHERE p.category IS NOT NULL AND TRIM(p.category)!='' AND " + _ELIGIBLE + " "
+    "UNION SELECT pp.category,pp.positive_prompt_id," + SHELVED_PAIR + " "
+    "FROM prompt_pairs pp "
     "WHERE pp.archived=0 AND pp.category IS NOT NULL AND TRIM(pp.category)!=''),"
     "image_values(value,image_count) AS ("
     "SELECT category,COUNT(*) FROM image_categories GROUP BY category),"
     "grouped(value,first_pid,prompt_count) AS ("
-    "SELECT value,MIN(pid),COUNT(DISTINCT pid) FROM owners GROUP BY value "
+    "SELECT value,MIN(pid),COUNT(DISTINCT CASE WHEN shelved THEN pid END) "
+    "FROM owners GROUP BY value "
     "UNION ALL SELECT value," + str(_IMAGE_ONLY_FIRST_PID) + ",0 FROM image_values "
     "WHERE value NOT IN (SELECT value FROM owners)),"
     "counted AS (SELECT g.value,g.first_pid,g.prompt_count,"