chore: release 3.4.0

# Conflicts:
#	README.md

.github/branch-protection.json

@@ -1,22 +1,12 @@
 {
   "required_status_checks": {
-    "strict": true,
+    "strict": false,
     "contexts": [
-      "py3.10 on ubuntu-latest",
-      "py3.11 on ubuntu-latest",
       "py3.12 on ubuntu-latest",
-      "py3.13 on ubuntu-latest",
-      "py3.10 on windows-latest",
-      "py3.11 on windows-latest",
-      "py3.12 on windows-latest",
-      "py3.13 on windows-latest",
-      "py3.10 on macos-latest",
-      "py3.11 on macos-latest",
-      "py3.12 on macos-latest",
-      "py3.13 on macos-latest",
       "coverage gate (85%)",
       "black and flake8",
       "frontend modules",
+      "all tests",
       "viewerjs",
       "real ComfyUI compatibility",
       "bandit (high severity)"

.github/workflows/code-quality.yml

@@ -4,6 +4,7 @@ permissions:
   contents: read
 
 on:
+  merge_group:
   push:
     branches: [main]
   pull_request:
@@ -19,14 +20,17 @@ jobs:
   security:
     name: bandit (high severity)
     runs-on: ubuntu-latest
+    # ceil(1.5 x slowest of three green main runs) + 3, never under 5 (test.yml): runs 40257217665,
+    # 40254137380, 40240706583 took 23s / 16s / 23s -> 5 min.
+    timeout-minutes: 5
 
     steps:
       - uses: actions/checkout@v7
 
-      - name: Set up Python 3.10
+      - name: Set up Python 3.12
         uses: actions/setup-python@v7
         with:
-          python-version: "3.10"
+          python-version: "3.12"
 
       - name: Install bandit
         run: pip install bandit[toml]

.github/workflows/portability.yml

@@ -0,0 +1,145 @@
+name: Windows and macOS
+
+permissions:
+  contents: read
+
+# Non-blocking portability runs (#420): never on pull requests or in the merge queue, and no context
+# here is a required check. A red run is fixed at its cause before the workspace cutover (#334).
+on:
+  push:
+    branches: [main]
+  schedule:
+    - cron: "17 3 * * *"
+  workflow_dispatch:
+
+# Every run gets its own group (run_id), so no main commit's run is ever cancelled or dropped as pending.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.run_id }}
+  cancel-in-progress: false
+
+env:
+  PYTHONFAULTHANDLER: "1"
+
+# Time limit rule as in test.yml: ceil(1.5 x the slowest of three recent green runs on main) + 3
+# minutes, never under 5, timed as whole-job wall time. One matrix job, so the slowest leg of each
+# run counts (tests/vendor/ci-workflow.test.mjs recomputes it, #493).
+# Measured 2026-10-29 on main runs 40935337099 (dd247e3), 40938407252 (dd247e3), 40941240624 (dd247e3):
+#   slowest leg: 7m24s (py3.12 windows) / 7m31s (py3.13 windows) / 7m32s (py3.14 windows) -> 15 min
+#     (windows 4m56s-7m32s, macos 4m39s-6m14s)
+#   native-trash (#499) has no green main runs yet: its probe, run 40949370943 (2026-10-29), took
+#     about 1 minute per macOS leg, so ceil(1.5 x 2) + 3 gives it 6 minutes (step 4). Recalibrate
+#     after three green runs.
+#   native-trash-linux (#543) has no green main runs yet: it installs the dev requirements and runs
+#     one test module, about 1 minute by the same estimate, so ceil(1.5 x 2) + 3 gives it 6 minutes.
+#     Recalibrate after three green runs.
+jobs:
+  test:
+    name: py${{ matrix.python-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest, macos-latest]
+        python-version: ["3.12", "3.13", "3.14"]
+
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: requirements-dev.txt
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-dev.txt
+
+      # The runner's TEMP is on C:, the OS disk, where a flush took about 6 ms
+      # and a test database's create-to-delete about 111 ms; on the D: work
+      # disk the same took 0.5 ms and 23 ms (macOS: 19 ms). Tests write their
+      # databases under TEMP, so it moves to a folder on D: (measured in #487,
+      # run 40915866444). The runner's TEMP was an 8.3 short name
+      # (C:\Users\RUNNER~1\...), so the new one is spelled by its short name too
+      # and paths still have to be resolved. D: makes no short names by default
+      # (run 40922502209), so they are switched on for it first.
+      - name: Keep test files on the work disk (Windows)
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          fsutil 8dot3name set D: 0
+          $folder = New-Item -ItemType Directory -Force -Path (Join-Path $env:RUNNER_TEMP "PromptManagerTests")
+          $short = (New-Object -ComObject Scripting.FileSystemObject).GetFolder($folder.FullName).ShortPath
+          Write-Output "test TEMP: $short (long name $($folder.FullName))"
+          if ($short -eq $folder.FullName) { Write-Output "::warning::D: gave the test TEMP no 8.3 short name" }
+          "TEMP=$short" >> $env:GITHUB_ENV
+          "TMP=$short" >> $env:GITHUB_ENV
+
+      - name: Run tests
+        env:
+          PM_TEST_SLOWEST: "40"
+        run: python -m tests.run_suite -v
+
+  # The real Trash on both macOS CPUs (#499 design §10.2). It runs apart from the suite, so a red
+  # suite never skips the native proof.
+  native-trash:
+    name: native trash on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 6
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, macos-15-intel]
+
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+
+      - name: Native trash proof
+        timeout-minutes: 4
+        env:
+          PM_NATIVE_TRASH: macos
+        run: python -m unittest tests.test_trash_macos_native tests.test_trash_macos -v
+
+  # The same tests as the fake mount table, on a real ext4 mount used as the output root (#543). A
+  # runner can mount with sudo; the tests run as the normal user, so owner and mode checks are real.
+  native-trash-linux:
+    name: native trash on a real mount
+    runs-on: ubuntu-latest
+    timeout-minutes: 6
+
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: requirements-dev.txt
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-dev.txt
+
+      - name: Mount an ext4 image
+        run: |
+          truncate -s 64M "$RUNNER_TEMP/drive.img"
+          mkfs.ext4 -q "$RUNNER_TEMP/drive.img"
+          mkdir "$RUNNER_TEMP/drive"
+          sudo mount -o loop "$RUNNER_TEMP/drive.img" "$RUNNER_TEMP/drive"
+          sudo rmdir "$RUNNER_TEMP/drive/lost+found"
+          sudo chown "$(id -u):$(id -g)" "$RUNNER_TEMP/drive"
+          findmnt "$RUNNER_TEMP/drive"
+
+      - name: Native trash proof
+        timeout-minutes: 4
+        run: PM_TRASH_MOUNT="$RUNNER_TEMP/drive" python -m unittest tests.test_trash_mount_root -v

.github/workflows/test.yml

@@ -4,6 +4,7 @@ permissions:
   contents: read
 
 on:
+  merge_group:
   pull_request:
     branches: [main]
   push:
@@ -19,15 +20,36 @@ concurrency:
 env:
   PYTHONFAULTHANDLER: "1"
 
+# Every job has a time limit: ceil(1.5 x the slowest of three recent green runs on main) + 3
+# minutes, never under 5. The 3 minutes leave room for the Python runner's watchdog (120 s, see
+# tests/run_suite.py) to print its stack dump before GitHub cancels the job. A limit changes only
+# with new measurements written here (TESTING.md "Time limits and hangs").
+# Measured 2026-10-26 on runs 40257217681 (3be0ada), 40240706568 (fb9134f), 40228167328 (f9835aa):
+#   py matrix, slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min
+#     (ubuntu 5m27s-6m27s, macos 6m14s-8m30s, windows 11m42s-15m54s)
+#     3.10 and 3.11 left the matrix and 3.14 joined it in #356; re-measure after three green main runs.
+#     Windows and macOS moved to portability.yml (#420), which has its own measured limit (#493);
+#     this Linux-only matrix keeps 27 minutes until it is re-measured.
+#   real ComfyUI compatibility: 13m32s / 8m55s / 11m46s -> 24 min
+#   coverage gate (85%): 7m26s / 7m22s / 7m54s -> 15 min
+#   perf budget 30s / 29s / 27s, black and flake8 39s / 37s / 39s,
+#   frontend modules 23s / 19s / 29s -> 5 min (the floor)
+#   all tests: a result check without setup, 5 min (the floor)
+#
+# Pull requests run a fast subset (#420): Python 3.12 on ubuntu-latest, coverage, lint and frontend
+# modules. The merge queue (merge_group), pushes to main and manual runs add Python 3.13 and 3.14 on
+# Linux and the real ComfyUI job. "all tests" is the required gate that holds the queue to that set.
+# Windows and macOS run in portability.yml (push to main, nightly, on demand) and never gate a merge.
 jobs:
   test:
     name: py${{ matrix.python-version }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 27
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.10", "3.11", "3.12", "3.13"]
+        os: [ubuntu-latest]
+        python-version: ${{ github.event_name == 'pull_request' && fromJSON('["3.12"]') || fromJSON('["3.12", "3.13", "3.14"]') }}
 
     steps:
       - uses: actions/checkout@v7
@@ -45,21 +67,26 @@ jobs:
           pip install -r requirements-dev.txt
 
       - name: Run tests
-        run: python -m unittest discover -s tests -v
+        run: python -m tests.run_suite -v
 
   comfyui:
     name: real ComfyUI compatibility
+    # Merge queue, main and manual runs only. A job skipped by its condition reports Success, so this
+    # required context never blocks a pull request; "all tests" fails if it is skipped anywhere else.
+    if: github.event_name != 'pull_request'
     runs-on: ubuntu-latest
-    timeout-minutes: 25
+    # Chromium and its system libraries come from the pinned Playwright image (tag = @playwright/test in
+    # tools/vendor/package-lock.json; tests/vendor/browser-image.test.mjs checks it). The runner install
+    # pulled them from Ubuntu's apt mirror on every run, and three runs (#349, #378, #382) crawled at
+    # ~26 kB/s for 11-12 minutes until the job hit its limit after every test had passed.
+    container:
+      image: mcr.microsoft.com/playwright:v1.64.1-noble
+      options: --user 1001
+    timeout-minutes: 24
     steps:
       - uses: actions/checkout@v7
         with:
           persist-credentials: false
-      - uses: actions/setup-node@v7
-        with:
-          node-version: "24"
-          cache: npm
-          cache-dependency-path: tools/vendor/package-lock.json
       - name: Read the reviewed runtime pins
         id: runtime
         run: |
@@ -84,21 +111,42 @@ jobs:
           ref: ${{ steps.runtime.outputs.kikoRevision }}
           path: .ci-kikotools
           persist-credentials: false
-      - uses: actions/setup-python@v7
+      # The image's own Python and Node 24: setup-python/setup-node install the runner's toolcache builds,
+      # whose entry points name host paths that do not exist in the container. The image has no ensurepip
+      # and the job has no root, so a checksum-pinned pip wheel bootstraps a user-space venv. The image
+      # Python must be the reviewed host Python, so a pin or image bump that disagrees fails here.
+      - name: Python environment on the reviewed host Python
+        env:
+          PIP_WHEEL: https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl
+          PIP_SHA256: 71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e
+          PINNED_PYTHON: ${{ steps.runtime.outputs.pythonVersion }}
+        run: |
+          image="$(python3 -c 'import sys; print("%d.%d" % sys.version_info[:2])')"
+          [ "$image" = "$PINNED_PYTHON" ] || { echo "::error::image Python $image is not the pinned $PINNED_PYTHON"; exit 1; }
+          venv="$RUNNER_TEMP/venv" wheel="$RUNNER_TEMP/pip-26.2.1-py3-none-any.whl"
+          python3 -m venv --without-pip "$venv"
+          curl -fsSL --retry 3 -o "$wheel" "$PIP_WHEEL"
+          echo "$PIP_SHA256  $wheel" | sha256sum -c -
+          "$venv/bin/python" "$wheel/pip" install --no-index --no-cache-dir "$wheel"
+          echo "$venv/bin" >> "$GITHUB_PATH"
+      - run: python --version && node --version && node --version | grep -q '^v24\.'
+      # pip's download cache (torch alone is ~200 MB); inside the container ~ is the job's HOME (/github/home).
+      - name: Cache pip downloads
+        uses: actions/cache@v6
         with:
-          python-version: ${{ steps.runtime.outputs.pythonVersion }}
-          cache: pip
-          cache-dependency-path: |
-            requirements-dev.txt
-            .ci-comfyui/requirements.txt
-            tools/vendor/comfyui-runtime.json
+          path: ~/.cache/pip
+          key: pip-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('requirements-dev.txt', '.ci-comfyui/requirements.txt', 'tools/vendor/comfyui-runtime.json') }}
       - name: Install CPU runtime and test dependencies
         run: |
-          python -m pip install --upgrade pip
           pip install "torch==${{ steps.runtime.outputs.torchVersion }}" "torchvision==${{ steps.runtime.outputs.torchvisionVersion }}" --index-url https://download.pytorch.org/whl/cpu
           pip install -r .ci-comfyui/requirements.txt -r requirements-dev.txt "torch==${{ steps.runtime.outputs.torchVersion }}" "torchvision==${{ steps.runtime.outputs.torchvisionVersion }}"
+      # npm's download cache, keyed on the lockfile. node_modules itself is still built fresh by npm ci.
+      - name: Cache npm downloads
+        uses: actions/cache@v6
+        with:
+          path: ~/.npm
+          key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
       - run: npm ci --prefix tools/vendor --ignore-scripts
-      - run: npx --prefix tools/vendor playwright install --with-deps chromium
       - name: Classic and Nodes 2.0 startup and compatibility
         env:
           PM_COMFY_ROOT: ${{ github.workspace }}/.ci-comfyui
@@ -127,6 +175,7 @@ jobs:
   coverage:
     name: coverage gate (85%)
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@v7
 
@@ -143,7 +192,7 @@ jobs:
 
       - name: Measure coverage
         run: |
-          python -m coverage run -m unittest discover -s tests
+          python -m coverage run -m tests.run_suite
           python -m coverage combine
           python -m coverage report --fail-under=85
 
@@ -154,6 +203,7 @@ jobs:
     name: perf budget
     if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
+    timeout-minutes: 5
     steps:
       - uses: actions/checkout@v7
 
@@ -176,24 +226,31 @@ jobs:
   lint:
     name: black and flake8
     runs-on: ubuntu-latest
+    timeout-minutes: 5
     steps:
       - uses: actions/checkout@v7
       - uses: actions/setup-python@v7
         with:
           python-version: "3.12"
-      - run: pip install black flake8
+          cache: pip
+          cache-dependency-path: requirements-dev.txt
+      # The exact black and flake8 pins from requirements-dev.txt, so a new release cannot turn CI red.
+      - run: pip install $(grep -E '^(black|flake8)==' requirements-dev.txt)
       - run: black --check .
       - run: flake8 .
 
   js-test:
     name: frontend modules
     runs-on: ubuntu-latest
+    timeout-minutes: 5
     steps:
       - uses: actions/checkout@v7
 
       - uses: actions/setup-node@v7
         with:
           node-version: "24"
+          cache: npm
+          cache-dependency-path: tools/vendor/package-lock.json
 
       - name: Check shipped extension and page modules parse
         run: |
@@ -207,3 +264,32 @@ jobs:
 
       - name: Run JS tests
         run: node --test tests/js/*.test.js
+
+  # The required context "all tests". A dropped matrix leg never reports a check at all, so the eight
+  # legs that pull requests skip cannot be required by name; this gate requires them instead. Every
+  # job must succeed; real ComfyUI must be skipped on a pull request and succeed on every other event.
+  all-tests:
+    name: all tests
+    needs: [test, comfyui, coverage, lint, js-test]
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Require every test job for this event
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          TEST_RESULT: ${{ needs.test.result }}
+          COMFYUI_RESULT: ${{ needs.comfyui.result }}
+          COVERAGE_RESULT: ${{ needs.coverage.result }}
+          LINT_RESULT: ${{ needs.lint.result }}
+          JS_RESULT: ${{ needs.js-test.result }}
+        run: |
+          queue_only=success
+          if [ "$EVENT_NAME" = pull_request ]; then queue_only=skipped; fi
+          echo "event: $EVENT_NAME (real ComfyUI must be $queue_only)"
+          echo "test matrix: $TEST_RESULT"
+          echo "comfyui: $COMFYUI_RESULT"
+          echo "coverage: $COVERAGE_RESULT"
+          echo "lint: $LINT_RESULT"
+          echo "js-test: $JS_RESULT"
+          test "$TEST_RESULT" = success && test "$COVERAGE_RESULT" = success && test "$LINT_RESULT" = success && test "$JS_RESULT" = success && test "$COMFYUI_RESULT" = "$queue_only"

.github/workflows/viewerjs-check.yml

@@ -1,6 +1,7 @@
 name: Viewer.js vendor and compatibility
 
 on:
+  merge_group:
   pull_request:
     branches: [main]
   push:
@@ -26,6 +27,20 @@ concurrency:
 # coverage shards 7m34s/6m04s/5m22s (tests 6m38s/4m53s/4m04s), merge 54s, firefox 7m57s/5m21s (tests
 # 7m08s/4m34s), webkit 8m16s/5m03s (tests 7m26s/3m58s); a cache miss adds up to 5m to npm ci, which
 # the 20-minute limits absorb.
+# Hangs (#425): a job killed at its limit keeps no log and skips its upload steps, so each browser
+# test step has its own timeout-minutes (18) below the job's 20. At the step limit, SIGTERM makes
+# tools/vendor/running-tests-reporter.mjs print the tests still running and hand over to Playwright's
+# interrupt, which writes the blob report; the always() uploads then run.
+# Measured on main (runs 40725289094, 40719375442, 40700439399): setup before the test step <= 52s,
+# uploads <= 35s; test steps up to 14m42s (webkit shard 1/2) and 13m05s for coverage (shard 1/3:
+# 12m53s, node units and the viewer suite before 11.8m of workflows), so coverage is the tight one
+# in its 18 minutes. This holds only when the npm cache hits: a cold cache adds up to 5m to npm ci,
+# and then the job limit can still end the job before the step limit does.
+# #473: firefox/webkit shard 1/2 reached its 18-minute step limit (queue run 40866252975: webkit 622 tests
+# passed in 17m56s, 48.5 worker-minutes vs 28.0 in shard 2/2; recent shard 1/2 jobs 13.7m-18.9m, shard 2/2
+# 7.3m-12.2m), because --shard splits by test count and the slow concept specs sort into shard 1. The page
+# workflows now run in three shards per browser with the limits unchanged; the per-shard times are
+# re-measured after three green main runs.
 jobs:
   frontend-coverage:
     name: frontend coverage (shard ${{ matrix.shard }}/3)
@@ -78,6 +93,7 @@ jobs:
       - if: matrix.shard == 3
         run: npm run --prefix tools/vendor test
       - name: Page workflows with coverage, shard ${{ matrix.shard }}/3
+        timeout-minutes: 18
         run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/3
       - name: Upload raw coverage
         uses: actions/upload-artifact@v7
@@ -86,6 +102,7 @@ jobs:
           path: coverage/js/raw/
           if-no-files-found: error
           retention-days: 3
+      # Traces and the blob report (test-results/blob/) of a failed, timed-out or cancelled step.
       - name: Upload Chromium and touch traces
         if: always()
         uses: actions/upload-artifact@v7
@@ -163,7 +180,9 @@ jobs:
           retention-days: 14
 
   browser-workflows:
-    name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/2)
+    name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/3)
+    # Merge queue, main and manual runs only (#420); pull requests run the Chromium/touch coverage shards.
+    if: github.event_name != 'pull_request'
     runs-on: ubuntu-latest
     # Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
     # same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
@@ -180,7 +199,7 @@ jobs:
       fail-fast: false
       matrix:
         browser: [firefox, webkit]
-        shard: [1, 2]
+        shard: [1, 2, 3]
     steps:
       - uses: actions/checkout@v7
       # The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
@@ -207,8 +226,10 @@ jobs:
           path: ~/.npm
           key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
       - run: npm ci --prefix tools/vendor --ignore-scripts
-      - name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/2
-        run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/2
+      - name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/3
+        timeout-minutes: 18
+        run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/3
+      # Traces and the blob report (test-results/blob/) of a failed, timed-out or cancelled step.
       - name: Upload browser traces
         if: always()
         uses: actions/upload-artifact@v7
@@ -218,8 +239,9 @@ jobs:
           retention-days: 14
 
   # Branch protection requires the exact context "viewerjs". This job keeps that
-  # name and fails unless every frontend job above succeeded (every shard, the
-  # coverage merge and every browser shard), including when one was cancelled or skipped.
+  # name and fails unless every coverage shard and the coverage merge succeeded and
+  # the browser shards succeeded, or, on a pull request only, were skipped (#420).
+  # Any failure, cancellation or other skip fails it.
   viewerjs:
     needs: [frontend-coverage, frontend-coverage-merge, browser-workflows]
     if: always()
@@ -228,11 +250,15 @@ jobs:
     steps:
       - name: Require every frontend job to succeed
         env:
+          EVENT_NAME: ${{ github.event_name }}
           COVERAGE_RESULT: ${{ needs.frontend-coverage.result }}
           MERGE_RESULT: ${{ needs.frontend-coverage-merge.result }}
           WORKFLOWS_RESULT: ${{ needs.browser-workflows.result }}
         run: |
+          queue_only=success
+          if [ "$EVENT_NAME" = pull_request ]; then queue_only=skipped; fi
+          echo "event: $EVENT_NAME (browser-workflows must be $queue_only)"
           echo "frontend-coverage: $COVERAGE_RESULT"
           echo "frontend-coverage-merge: $MERGE_RESULT"
           echo "browser-workflows: $WORKFLOWS_RESULT"
-          test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success
+          test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = "$queue_only"

.gitignore

@@ -211,3 +211,10 @@ coverage/
 
 # Disposable upstream checkout for real-host CI
 .ci-comfyui/
+
+# Old library files a Move library renamed (never deleted; #249 S6b)
+*.moved-to-*
+
+# Per-install key behind the workspace scopes and cursors (#430)
+workspace-scope.key
+.workspace-scope.key.*.tmp

.pre-commit-config.yaml

@@ -13,16 +13,18 @@ repos:
         language: system
         types: [python]
 
+      # Fails at the severity CI enforces (code-quality.yml: -lll, high only), never stricter.
+      # pre-commit appends the staged files, so no shell wrapper can drop the first one.
       - id: bandit
         name: bandit
-        entry: bash -c 'bandit "$@" -ll -q 2>&1; true'
+        entry: bandit -lll -q
         language: system
         types: [python]
         exclude: ^(tests/|reference/)
 
       - id: tests
         name: tests
-        entry: python -m unittest discover tests/ -v
+        entry: python -m tests.run_suite -q
         language: system
         pass_filenames: false
         stages: [pre-push]

__init__.py

@@ -22,22 +22,21 @@ Features:
 - Comprehensive logging and diagnostics
 """
 
-import re
+import atexit
+import threading
+import time
 from pathlib import Path
 
 
 def get_version():
-    """Parse version from pyproject.toml"""
+    """Parse version from pyproject.toml (``utils/version.py``)."""
+    from .utils.version import project_version
+
     try:
-        pyproject_path = Path(__file__).parent / "pyproject.toml"
-        if pyproject_path.exists():
-            content = pyproject_path.read_text()
-            match = re.search(r'version\s*=\s*["\']([^"\']+)["\']', content)
-            if match:
-                return match.group(1)
+        root = Path(__file__).parent
     except Exception:
-        pass
-    return "unknown"
+        return "unknown"
+    return project_version(root)
 
 
 from .prompt_manager import (  # noqa: E402
@@ -72,6 +71,12 @@ NODE_DISPLAY_NAME_MAPPINGS = {
 # are served by our own routes instead (see py/api).
 WEB_DIRECTORY = "web/comfy"
 
+# Parts the package teardown stops (``_stop_background_work``). Each stays None
+# when its start-up block below fails, and the teardown skips it.
+api = None
+_global_prompt_tracker = None
+_global_image_monitor = None
+
 # Add API routes (same pattern as ComfyUI_Assets)
 try:
     # Set extension URI
@@ -87,10 +92,25 @@ try:
     routes = config.routes
     api = PromptManagerAPI()
     api.add_routes(routes)
+    # Temp files of preview writes a crash interrupted (#317), off the
+    # import path: the walk may cover a large thumbnails folder.
+    threading.Thread(
+        target=api.sweep_thumbnail_temps,
+        name="PromptManagerPreviewTempSweep",
+        daemon=True,
+    ).start()
     host_app = getattr(config.server_instance, "app", None)
     if host_app is not None and hasattr(host_app, "on_cleanup"):
         host_app.on_cleanup.append(api.cleanup_history_reads)
         host_app.on_cleanup.append(api.cleanup_annotation_captures)
+        host_app.on_cleanup.append(api.cleanup_gallery_listings)
+    if host_app is not None and hasattr(host_app, "on_startup"):
+        # Build each output root's Images listing once the server's loop runs,
+        # so the first open does not wait for the walk (#348).
+        host_app.on_startup.append(api.warm_gallery_listings)
+    if host_app is not None and hasattr(host_app, "on_shutdown"):
+        # Open Needs review change streams end before the server waits on them.
+        host_app.on_shutdown.append(api.close_review_events)
 
 except Exception as e:
     # Log to internal logging system without console spam
@@ -122,6 +142,19 @@ try:
     _global_prompt_tracker = get_prompt_tracker(_global_db)
     _global_image_monitor = get_image_monitor(_global_db, _global_prompt_tracker)
 
+    # The Needs review pass for ambiguous new outputs runs through the API's
+    # capture store (the one the scan and the Tag dialog use): a linked output
+    # is never queued and its pending items leave the queue. The package
+    # teardown detaches it before that store is closed at shutdown.
+    if api is not None:
+        _global_image_monitor.set_review_recorder(api.record_review_file)
+        # Every settled output's preview is queued for the API's background
+        # worker (#249): the workspace never asks to generate thumbnails.
+        _global_image_monitor.set_thumbnail_queue(api.queue_capture_thumbnail)
+        # Every settled output walks its Images listing again (#348): the
+        # API hops from the monitor's thread onto the server's loop.
+        _global_image_monitor.set_output_listener(api.notify_output_settled)
+
     # Start monitoring
     _global_image_monitor.start_monitoring()
     _init_logger.info("Global image monitoring system started successfully")
@@ -162,6 +195,26 @@ except Exception as e:
             f"Failed to register usage hook: {e}"
         )
 
+# A cached node counts what its last execution of the same text counted (#347).
+try:
+    from .py.config import server_instance
+    from .utils.cached_runs import register_cached_runs
+
+    register_cached_runs(server_instance, _db_factory)
+except Exception as e:
+    try:
+        from .utils.logging_config import get_logger
+
+        get_logger("prompt_manager.init").error(
+            f"Failed to register cached-run counting: {e}"
+        )
+    except Exception:
+        import logging
+
+        logging.getLogger("prompt_manager.init").error(
+            f"Failed to register cached-run counting: {e}"
+        )
+
 
 def _history_db_factory() -> "PromptDatabase":
     """Initialize history on its worker even if monitoring initialization failed."""
@@ -190,6 +243,193 @@ except Exception:
             "Failed to register execution history"
         )
 
+
+# ── Package teardown (#301, #302) ────────────────────────────────────────────
+# ComfyUI never runs aiohttp's cleanup hooks (it never calls
+# ``runner.cleanup()``), so ``atexit`` is the live host's path: Ctrl+C, and a
+# normal exit. It is registered after the history hook, so at exit (newest
+# first) it runs before the history dispose. Hosts that do run ``on_cleanup``
+# (our test servers) run it there, and the tests' package unload calls it.
+
+TEARDOWN_SECONDS = 10.0
+_clock = time.monotonic
+_background_work_lock = threading.Lock()
+_background_work_result = None
+
+
+def _teardown_step(name, stop, logger):
+    """Run one teardown step; a failure (even Ctrl+C) is logged and is False."""
+    try:
+        return bool(stop())
+    except (Exception, KeyboardInterrupt):
+        logger.error(f"Background work: stopping {name} failed", exc_info=True)
+        return False
+
+
+def _teardown_logger():
+    """PromptManager's init logger; the standard one if that cannot load."""
+    try:
+        from .utils.logging_config import get_logger
+
+        return get_logger("prompt_manager.init")
+    except Exception:
+        import logging
+
+        return logging.getLogger("prompt_manager.init")
+
+
+def _stop_background_work():
+    """Stop the image monitor, the preview worker and the prompt cleanup
+    thread against one TEARDOWN_SECONDS deadline, once.
+
+    Returns ``{"monitor", "thumbnails", "tracker"}`` booleans (True: it
+    stopped), leaving out parts that never started. Later calls return a
+    copy of the first result. A Ctrl+C inside a step is logged and the
+    step is False; one between steps is re-raised after the result (parts
+    not reached are False) is stored and the ``atexit`` entry removed.
+    """
+    global _background_work_result
+    with _background_work_lock:
+        if _background_work_result is not None:
+            return dict(_background_work_result)
+        monitor, thumbnails, tracker = (
+            _global_image_monitor,
+            api,
+            _global_prompt_tracker,
+        )
+        # A part whose global is set has started; False until its step says
+        # otherwise, so an interrupt between steps leaves an honest result.
+        started = (
+            ("monitor", monitor),
+            ("thumbnails", thumbnails),
+            ("tracker", tracker),
+        )
+        result = {name: False for name, part in started if part is not None}
+        try:
+            logger = _teardown_logger()
+            began = _clock()
+            deadline = began + TEARDOWN_SECONDS
+
+            def left(own_bound):
+                return max(0.0, min(own_bound, deadline - _clock()))
+
+            if monitor is not None:
+
+                def stop_monitor():
+                    from .utils.image_monitor import OBSERVER_JOIN_SECONDS
+
+                    # Callbacks first: an image in hand queues no preview into a
+                    # closing preview worker. The worker may have read the queue
+                    # callback already, so the API refuses new previews too (at
+                    # exit, Python 3.12+ refuses to start their worker thread).
+                    if thumbnails is not None:
+                        thumbnails.refuse_auto_thumbnails()
+                    monitor.set_review_recorder(None)
+                    monitor.set_thumbnail_queue(None)
+                    monitor.set_output_listener(None)
+                    return monitor.stop_monitoring(left(OBSERVER_JOIN_SECONDS))
+
+                result["monitor"] = _teardown_step(
+                    "the image monitor", stop_monitor, logger
+                )
+            if thumbnails is not None:
+
+                def close_previews():
+                    from .py.api.thumbnail_auto import CLOSE_WAIT_SECONDS
+
+                    return thumbnails.close_auto_thumbnails(
+                        final=True, timeout=left(CLOSE_WAIT_SECONDS)
+                    )
+
+                result["thumbnails"] = _teardown_step(
+                    "the preview worker", close_previews, logger
+                )
+            if tracker is not None:
+
+                def stop_tracker():
+                    from .utils.prompt_tracker import TRACKER_JOIN_SECONDS
+
+                    return tracker.shutdown(left(TRACKER_JOIN_SECONDS))
+
+                result["tracker"] = _teardown_step(
+                    "the prompt cleanup thread", stop_tracker, logger
+                )
+            elapsed = _clock() - began
+            failed = [name for name, ok in result.items() if not ok]
+            if failed:
+                logger.warning(
+                    f"Background work not stopped in time ({elapsed:.1f} s): "
+                    + ", ".join(failed)
+                )
+            else:
+                parts = " ".join(f"{name}=ok" for name in result)
+                logger.info(f"Background work stopped in {elapsed:.1f} s: {parts}")
+        finally:
+            _background_work_result = result
+            atexit.unregister(_stop_background_work)
+        return dict(result)
+
+
+async def _stop_background_work_on_cleanup(app):
+    """aiohttp ``on_cleanup``: no request is served then, so it runs inline."""
+    _stop_background_work()
+
+
+def _kill_encoders_at_exit():
+    """Run by the exit hook before Python joins the executor threads (#376):
+    a JPEG XL or ffmpeg encoder (no console on Windows, so Ctrl+C never
+    reaches it) is killed instead of being waited for."""
+    from .py.api.thumbnail_encode import shutdown_encoders
+
+    shutdown_encoders()
+
+
+def _abort_downloads_at_exit():
+    """Run by the exit hook (#376): huggingface_hub's Xet transfers ignore
+    the chunk checkpoint, so they are aborted the way hub aborts on Ctrl+C."""
+    import sys
+
+    if "huggingface_hub" not in sys.modules:
+        return  # no download ever ran: import nothing at exit
+    from .py.autotag import abort_hub_downloads
+
+    abort_hub_downloads()
+
+
+try:
+    # Before the executor join: running jobs stop at their next file, item or
+    # chunk (#376). The teardown below still runs from atexit afterwards.
+    from .utils import exit_signal as _exit_signal
+
+    _exit_signal.install(_kill_encoders_at_exit)
+    _exit_signal.install(_abort_downloads_at_exit)
+except Exception as e:
+    try:
+        from .utils.logging_config import get_logger
+
+        get_logger("prompt_manager.init").error(
+            f"Failed to register the exit signal: {e}"
+        )
+    except Exception:
+        pass
+
+try:
+    atexit.register(_stop_background_work)
+    from .py.config import server_instance as _teardown_server
+
+    _teardown_host = getattr(_teardown_server, "app", None)
+    if _teardown_host is not None and hasattr(_teardown_host, "on_cleanup"):
+        _teardown_host.on_cleanup.insert(0, _stop_background_work_on_cleanup)
+except Exception as e:
+    try:
+        from .utils.logging_config import get_logger
+
+        get_logger("prompt_manager.init").error(
+            f"Failed to register the background work teardown: {e}"
+        )
+    except Exception:
+        pass
+
 __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]
 
 # Print startup message with loaded tools

database/annotations.py

@@ -19,6 +19,11 @@ links (``PromptDatabase._move_tag_links``).
 Cheap reads (spec amendment S1) treat a byte record as *current* only while
 the file's present ``(size, mtime_ns)`` equals the record's verified
 fingerprint; a file without a current record shows no image tags.
+
+A write made from a tagging model's result passes ``mark_autotagged`` (#379):
+the byte record is then marked auto-tagged (``media_image_autotags``) in the
+same transaction, so a mark never exists without the write it records. Hand
+tags never mark.
 """
 
 import re
@@ -34,6 +39,8 @@ from .annotation_schema import (
     MAX_LOCATOR_LENGTH,
     METADATA_TABLE,
 )
+from .autotag_schema import AUTOTAG_TABLE
+from .image_removal_schema import clear_marks
 from .library_commands import _UUID, LibraryCommandError, _validate_epoch
 from .models import utc_now_iso
 from .vocabulary import validate_vocabulary_tag_name
@@ -96,12 +103,24 @@ _TAGS_OF = (
     "JOIN tags t ON t.id=mit.tag_id WHERE mit.byte_id IN ({marks}) "
     "ORDER BY mit.byte_id,mit.position"
 )
-_RECORDS = (
-    "SELECT f.locator,b.id,b.size,b.mtime_ns,t.name "  # nosec B608
-    f"FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b ON b.file_id=f.id "
-    f"LEFT JOIN {IMAGE_TAGS_TABLE} mit ON mit.byte_id=b.id "
-    "LEFT JOIN tags t ON t.id=mit.tag_id "
-    "ORDER BY f.locator,b.first_seen_at,b.id,mit.position"
+_MARK = f"INSERT OR IGNORE INTO {AUTOTAG_TABLE}(byte_id) VALUES(?)"  # nosec B608
+_MARKED_OF = (
+    f"SELECT byte_id FROM {AUTOTAG_TABLE} WHERE byte_id IN ({{marks}})"  # nosec B608
+)
+_MARKED_LOCATORS = (
+    f"SELECT DISTINCT f.locator FROM {AUTOTAG_TABLE} m "  # nosec B608
+    f"JOIN {BYTES_TABLE} b ON b.id=m.byte_id JOIN {FILES_TABLE} f ON f.id=b.file_id"
+)
+# Tag search (#379): the names that tag at least one image (the tag index
+# answers the EXISTS), then the locators carrying chosen names (by tag_id).
+_IMAGE_TAG_NAMES = (
+    "SELECT t.id,t.name FROM tags t WHERE EXISTS "  # nosec B608
+    f"(SELECT 1 FROM {IMAGE_TAGS_TABLE} mit WHERE mit.tag_id=t.id)"
+)
+_TAG_LOCATORS = (
+    f"SELECT DISTINCT f.locator FROM {IMAGE_TAGS_TABLE} mit "  # nosec B608
+    f"JOIN {BYTES_TABLE} b ON b.id=mit.byte_id "
+    f"JOIN {FILES_TABLE} f ON f.id=b.file_id WHERE mit.tag_id IN ({{marks}})"
 )
 
 
@@ -218,6 +237,9 @@ class AnnotationOperations:
 
     @staticmethod
     def _annotation_owner(conn: sqlite3.Connection, locator: str, now: str) -> str:
+        # A file with image tags or Needs review state is in the library: a
+        # removal mark never hides it (#480).
+        clear_marks(conn, [locator])
         row = conn.execute(_OWNER, (locator,)).fetchone()
         if row is not None:
             return row[0]
@@ -301,14 +323,17 @@ class AnnotationOperations:
         expected_epoch: str,
         expected_tag_revision: int,
         tags: list,
+        mark_autotagged: bool = False,
     ) -> dict:
         """Replace the set of these exact bytes, guarded by epoch and revision.
 
         Creates the owner and byte record when the bytes are new (an explicit
         empty set is a real save). The identical set at the current revision,
         or one behind it (a lost-response retry), is ``replayed`` and writes
-        nothing; any other revision mismatch is ``tag-conflict`` with the
-        current set and no writes.
+        no tag; any other revision mismatch is ``tag-conflict`` with the
+        current set and no writes. ``mark_autotagged`` marks the bytes
+        auto-tagged on a saved or replayed receipt (the Tag image dialog's
+        save after Generate suggestions).
         """
         _validate_target(locator, sha256, size, mtime_ns)
         _validate_epoch(expected_epoch)
@@ -329,6 +354,8 @@ class AnnotationOperations:
             ):
                 if refingerprinted:
                     conn.execute(_BUMP_INDEX)
+                if mark_autotagged:
+                    conn.execute(_MARK, (byte_id,))
                 return _receipt(epoch, sha256, current, revision, True)
             if expected_tag_revision != revision:
                 raise AnnotationCommandError(
@@ -337,6 +364,8 @@ class AnnotationOperations:
                     {"sha256": sha256, "tags": current, "tag_revision": revision},
                 )
             revision = self._write_image_tags(conn, byte_id, tags)
+            if mark_autotagged:
+                conn.execute(_MARK, (byte_id,))
             return _receipt(epoch, sha256, tags, revision, False)
 
     def add_image_tags(
@@ -348,16 +377,22 @@ class AnnotationOperations:
         mtime_ns: int,
         expected_epoch: str,
         tags: list,
+        mark_autotagged: bool = False,
     ) -> dict:
         """Union ``tags`` into the set of these exact bytes, keeping stored spelling.
 
         ``added`` counts the names that were new (case-insensitively). A union
         over ``MAX_IMAGE_TAGS`` is ``tag-limit`` with the current set and no
         writes; a union that adds nothing to an existing record is ``replayed``.
+        ``mark_autotagged`` (a tagging model's result) also marks the bytes
+        auto-tagged, replayed or not; only such a call may pass no tags (the
+        model found none: the image is marked and its set is unchanged).
         """
         _validate_target(locator, sha256, size, mtime_ns)
         _validate_epoch(expected_epoch)
         tags = normalize_image_tags(tags)
+        if not tags and not mark_autotagged:
+            raise ValueError("tags must name at least one tag")
         with self._pair_transaction() as conn:
             epoch = self._library_epoch(conn, expected_epoch)
             now = utc_now_iso()
@@ -375,6 +410,8 @@ class AnnotationOperations:
                     epoch,
                     {"sha256": sha256, "tags": current, "tag_revision": revision},
                 )
+            if mark_autotagged:
+                conn.execute(_MARK, (byte_id,))
             if not created and not added:
                 if refingerprinted:
                     conn.execute(_BUMP_INDEX)
@@ -403,6 +440,41 @@ class AnnotationOperations:
                 conn.execute(_BUMP_INDEX)
             return cursor.rowcount > 0
 
+    @staticmethod
+    def _validated_stats(stats: object) -> dict:
+        if not isinstance(stats, dict):
+            raise ValueError(
+                "stats must map locators to (size, mtime_ns) integer pairs"
+            )
+        for locator, fingerprint in stats.items():
+            _validate_fingerprint(locator, fingerprint)
+        return stats
+
+    @staticmethod
+    def _current_byte_records(conn: sqlite3.Connection, stats: dict) -> dict:
+        """``{locator: byte_id}`` of each locator's current byte record.
+
+        The one rule every cheap read uses: a record is current while its
+        verified ``(size, mtime_ns)`` equals the file's present fingerprint
+        in ``stats``; the most recently verified wins (ties on ``(verified_at,
+        byte_id)``). A locator without a current record is left out. Lookups
+        go in chunks of 500 locators.
+        """
+        locators = list(stats)
+        chosen: dict[str, tuple[tuple[str, str], str]] = {}
+        for start in range(0, len(locators), _LOOKUP_CHUNK):
+            chunk = locators[start : start + _LOOKUP_CHUNK]
+            marks = ",".join("?" * len(chunk))
+            for locator, byte_id, size, mtime_ns, verified_at in conn.execute(
+                _FINGERPRINTS.format(marks=marks), chunk
+            ):
+                if (size, mtime_ns) != stats[locator]:
+                    continue
+                key = (verified_at, byte_id)
+                if locator not in chosen or key > chosen[locator][0]:
+                    chosen[locator] = (key, byte_id)
+        return {locator: byte_id for locator, (_key, byte_id) in chosen.items()}
+
     def current_image_tags(self, stats: dict) -> dict:
         """Image tags of the files whose current ``(size, mtime_ns)`` is verified.
 
@@ -411,29 +483,11 @@ class AnnotationOperations:
         exactly that verified fingerprint; the most recently verified record
         wins when several do. Lookups go in chunks of 500 locators.
         """
-        if not isinstance(stats, dict):
-            raise ValueError(
-                "stats must map locators to (size, mtime_ns) integer pairs"
-            )
-        for locator, fingerprint in stats.items():
-            _validate_fingerprint(locator, fingerprint)
-        locators = list(stats)
-        chosen: dict[str, tuple[tuple[str, str], str]] = {}
-        result: dict[str, list[str]] = {}
+        stats = self._validated_stats(stats)
         with self._annotation_read() as conn:
-            for start in range(0, len(locators), _LOOKUP_CHUNK):
-                chunk = locators[start : start + _LOOKUP_CHUNK]
-                marks = ",".join("?" * len(chunk))
-                for locator, byte_id, size, mtime_ns, verified_at in conn.execute(
-                    _FINGERPRINTS.format(marks=marks), chunk
-                ):
-                    if (size, mtime_ns) != stats[locator]:
-                        continue
-                    key = (verified_at, byte_id)
-                    if locator not in chosen or key > chosen[locator][0]:
-                        chosen[locator] = (key, byte_id)
-            owners = {byte_id: locator for locator, (_key, byte_id) in chosen.items()}
-            result = {locator: [] for locator in chosen}
+            current = self._current_byte_records(conn, stats)
+            owners = {byte_id: locator for locator, byte_id in current.items()}
+            result: dict[str, list[str]] = {locator: [] for locator in current}
             byte_ids = list(owners)
             for start in range(0, len(byte_ids), _LOOKUP_CHUNK):
                 chunk = byte_ids[start : start + _LOOKUP_CHUNK]
@@ -442,20 +496,60 @@ class AnnotationOperations:
                     result[owners[byte_id]].append(name)
         return result
 
-    def iter_image_tag_records(self) -> Iterator[tuple[str, int, int, list[str]]]:
-        """Stream every byte record as ``(locator, size, mtime_ns, tags)``.
+    def autotag_state(self, stats: dict) -> dict:
+        """``{locator: bool}``: is each file's current byte record auto-tagged?
 
-        One SELECT ordered by locator, then by the bytes' first sighting, with
-        each set's names in position order.
+        ``stats`` as for :meth:`current_image_tags`, whose rule picks the
+        current record; a locator without one is False. One read snapshot.
         """
-        conn = self.model.get_connection()
-        current: tuple | None = None
-        for locator, byte_id, size, mtime_ns, name in conn.execute(_RECORDS):
-            if current is None or current[0] != byte_id:
-                if current is not None:
-                    yield current[1]
-                current = (byte_id, (locator, size, mtime_ns, []))
-            if name is not None:
-                current[1][3].append(name)
-        if current is not None:
-            yield current[1]
+        stats = self._validated_stats(stats)
+        with self._annotation_read() as conn:
+            current = self._current_byte_records(conn, stats)
+            byte_ids = list(dict.fromkeys(current.values()))
+            marked: set[str] = set()
+            for start in range(0, len(byte_ids), _LOOKUP_CHUNK):
+                chunk = byte_ids[start : start + _LOOKUP_CHUNK]
+                marks = ",".join("?" * len(chunk))
+                marked.update(
+                    row[0]
+                    for row in conn.execute(_MARKED_OF.format(marks=marks), chunk)
+                )
+        return {locator: current.get(locator) in marked for locator in stats}
+
+    def autotagged_locators(self) -> set:
+        """Locators with at least one auto-tagged byte record, current or not.
+
+        No fingerprint is checked (the Tag output images review's estimate).
+        """
+        with self._annotation_read() as conn:
+            return {row[0] for row in conn.execute(_MARKED_LOCATORS)}
+
+    def image_tag_ids(self, wanted, query: str) -> list[int]:
+        """Ids of the tag names on at least one image that a search wants.
+
+        A name counts when it is in ``wanted`` exactly (the Tags filter) or
+        contains ``query`` case-folded (the text search; ``query`` is
+        already folded). One read of the names that tag an image.
+        """
+        wanted = set(wanted)
+        with self._annotation_read() as conn:
+            return [
+                tag_id
+                for tag_id, name in conn.execute(_IMAGE_TAG_NAMES)
+                if name in wanted or (query and query in name.casefold())
+            ]
+
+    def image_tag_locators(self, tag_ids) -> set[str]:
+        """Locators with any byte record (current or not) carrying one of
+        ``tag_ids``, through the tag index, in chunks of 500 ids."""
+        ids = list(dict.fromkeys(tag_ids))
+        found: set[str] = set()
+        with self._annotation_read() as conn:
+            for start in range(0, len(ids), _LOOKUP_CHUNK):
+                chunk = ids[start : start + _LOOKUP_CHUNK]
+                marks = ",".join("?" * len(chunk))
+                found.update(
+                    row[0]
+                    for row in conn.execute(_TAG_LOCATORS.format(marks=marks), chunk)
+                )
+        return found

database/autotag_schema.py

@@ -0,0 +1,140 @@
+"""Auto-tagged marks (#379): which image bytes a tagging model has handled.
+
+One table, ``media_image_autotags``: one row per byte record of the image
+annotation group (``media_annotation_bytes``) means "a tagging model looked at
+these exact bytes and its result was stored", or that the bytes were found to
+be untaggable. "Skip already tagged" skips an image whose current byte record
+is marked; hand tags never mark. Rows are written only by the image tag store
+(``database/annotations.py``), in the transaction of the tag write they
+record.
+
+The group installs like ``image_favorites``: optional in old backups,
+required once a database is current, and never backfilled. Like the Needs
+review group it is valid only beside a valid image annotation group, because
+every mark cites one byte record (``ON DELETE CASCADE``). Validation reads
+only: the catalog compared after whitespace compaction, the columns and the
+foreign key through the PRAGMAs, then one bounded value pass and the foreign
+key check.
+"""
+
+import re
+import sqlite3
+from typing import Optional
+
+from .annotation_schema import BYTES_TABLE
+from .annotation_schema import TABLES as ANNOTATION_TABLES
+from .annotation_schema import _catalog as _annotation_catalog
+from .annotation_schema import _uuid, annotation_schema_error
+
+AUTOTAG_TABLE = "media_image_autotags"
+_DDL = (
+    f"CREATE TABLE {AUTOTAG_TABLE} (byte_id TEXT PRIMARY KEY "
+    f"REFERENCES {BYTES_TABLE}(id) ON DELETE CASCADE) WITHOUT ROWID"
+)
+# (cid, name, declared type, not null, default, primary key position): the
+# key of a WITHOUT ROWID table is implicitly NOT NULL.
+_COLUMNS = [(0, "byte_id", "TEXT", 1, None, 1)]
+# (column, parent table, parent column, on delete)
+_FOREIGN_KEYS = {("byte_id", BYTES_TABLE, "id", "CASCADE")}
+
+NEEDS_ANNOTATIONS = "Image auto-tag marks need the image annotation tables"
+INVALID_SCHEMA = "Image auto-tag marks have an invalid schema"
+INVALID_VALUES = "Image auto-tag marks contain invalid values"
+MISSING = "Missing image auto-tag marks schema"
+
+
+def _compact(sql: str) -> str:
+    return re.sub(r"\s+", "", sql).lower()
+
+
+def _catalog(conn: sqlite3.Connection) -> list:
+    return conn.execute(
+        "SELECT type,name,sql FROM sqlite_master WHERE name=? OR tbl_name=?",
+        (AUTOTAG_TABLE, AUTOTAG_TABLE),
+    ).fetchall()
+
+
+def autotag_without_annotations_error(conn: sqlite3.Connection) -> Optional[str]:
+    """Catalog only: the marks table present without its annotation tables.
+
+    Startup asks this before the annotation group installs, which would make
+    a marks table copied without it look valid. It reads no rows.
+    """
+    if not _catalog(conn):
+        return None
+    present = {
+        name for kind, name, _sql in _annotation_catalog(conn) if kind == "table"
+    }
+    if present != set(ANNOTATION_TABLES):
+        return NEEDS_ANNOTATIONS
+    return None
+
+
+def autotag_schema_error(
+    conn: sqlite3.Connection,
+    *,
+    allow_missing: bool = True,
+    annotations_valid: bool = False,
+) -> Optional[str]:
+    """Reject an unsupported marks table or invalid marks; reads only.
+
+    A database without the table is acceptable for backups that predate it
+    unless ``allow_missing`` is false. A present table requires a present,
+    valid image annotation group; ``annotations_valid`` says the caller has
+    just validated it (startup's annotation install), so only its presence is
+    checked again.
+    """
+    objects = _catalog(conn)
+    if not objects:
+        return None if allow_missing else MISSING
+    if annotations_valid:
+        if autotag_without_annotations_error(conn):
+            return NEEDS_ANNOTATIONS
+    elif annotation_schema_error(conn, allow_missing=False):
+        return NEEDS_ANNOTATIONS
+    actual = {(kind, name, _compact(sql or "")) for kind, name, sql in objects}
+    if actual != {("table", AUTOTAG_TABLE, _compact(_DDL))}:
+        return INVALID_SCHEMA
+    columns = [
+        (row[0], row[1], row[2].upper(), row[3], row[4], row[5])
+        for row in conn.execute(f"PRAGMA table_info({AUTOTAG_TABLE})")
+    ]
+    if columns != _COLUMNS:
+        return INVALID_SCHEMA
+    foreign_keys = {
+        (row[3], row[2], row[4], row[6])
+        for row in conn.execute(f"PRAGMA foreign_key_list({AUTOTAG_TABLE})")
+    }
+    if foreign_keys != _FOREIGN_KEYS:
+        return INVALID_SCHEMA
+    # Only this module's constants are interpolated into the SQL below.
+    if conn.execute(
+        f"SELECT 1 FROM {AUTOTAG_TABLE} "  # nosec B608
+        f"WHERE NOT ({_uuid('byte_id')}) LIMIT 1"
+    ).fetchone():
+        return INVALID_VALUES
+    if conn.execute(f"PRAGMA foreign_key_check({AUTOTAG_TABLE})").fetchone():
+        return INVALID_VALUES
+    return None
+
+
+def migrate_autotag_state(conn: sqlite3.Connection) -> None:
+    """Install the table atomically; validate before and after creating it.
+
+    Runs after ``migrate_annotation_state`` (which has just validated the
+    annotation group). Creates one empty table: no row is read or copied.
+    """
+    conn.execute("SAVEPOINT media_autotag_schema")
+    try:
+        error = autotag_schema_error(conn, annotations_valid=True)
+        if error:
+            raise sqlite3.DatabaseError(error)
+        conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
+        error = autotag_schema_error(conn, allow_missing=False, annotations_valid=True)
+        if error:
+            raise sqlite3.DatabaseError(error)
+        conn.execute("RELEASE media_autotag_schema")
+    except BaseException:
+        conn.execute("ROLLBACK TO media_autotag_schema")
+        conn.execute("RELEASE media_autotag_schema")
+        raise

database/category_import.py

@@ -0,0 +1,201 @@
+"""Atomic replacement of one imported category (#221, #249 System S10).
+
+The LoRA reference import used to delete its whole category and then save
+each record in its own transaction, so a failure part way left the previous
+collection deleted and the new one partial. ``replace_category_prompts``
+does both in one ``BEGIN IMMEDIATE`` transaction: the category's current
+prompts (with their tags and image links) are replaced by the staged
+records, or, on any error, nothing changes.
+
+A staged record is ``{text, hash, tags, images}``. A record whose hash a
+prompt outside the category already has links its images to that prompt
+(the legacy import did the same); a second record with the hash of an
+earlier one links its images to that one. Neither is a new prompt.
+"""
+
+from typing import Any, Dict, List, Optional, Sequence
+
+try:
+    from .maintenance import PlanChanged
+    from .pairs import PairDependencyError
+    from .workflow_documents import documents_of_rows, release_documents
+except ImportError:  # pragma: no cover - flat test imports
+    from database.maintenance import PlanChanged
+    from database.pairs import PairDependencyError
+    from database.workflow_documents import documents_of_rows, release_documents
+
+
+class EmptyReplacement(ValueError):
+    """The replacement has no record and would remove every current import."""
+
+
+# The category's prompt ids, as a subquery (one bound parameter).
+_IN_CATEGORY = "SELECT id FROM prompts WHERE category = ?"
+
+# Each record's outcome and the result counter it adds to.
+_COUNTERS = {"imported": "imported", "linked": "linked", "duplicate": "duplicates"}
+
+
+def _placeholders(values: Sequence[Any]) -> str:
+    return ",".join("?" * len(values))
+
+
+class CategoryImportOperations:
+    """Read the state of an imported category and replace it atomically."""
+
+    def category_import_state(self, category: str) -> Dict[str, Any]:
+        """``{ids, paired}``: the category's prompt ids (ascending) and how
+        many of them a saved pair uses (those block a replacement)."""
+        with self.model.get_connection() as conn:
+            ids = [
+                row[0]
+                for row in conn.execute(
+                    "SELECT id FROM prompts WHERE category = ? ORDER BY id",
+                    (category,),
+                ).fetchall()
+            ]
+            paired = conn.execute(
+                "SELECT COUNT(*) FROM prompts WHERE category = ? AND id IN "
+                "(SELECT positive_prompt_id FROM prompt_pairs UNION "
+                "SELECT negative_prompt_id FROM prompt_pairs)",
+                (category,),
+            ).fetchone()[0]
+        return {"ids": ids, "paired": paired}
+
+    def existing_prompt_hashes(
+        self, hashes: Sequence[str], exclude_category: str
+    ) -> set:
+        """The given hashes a prompt outside ``exclude_category`` already has."""
+        found = set()
+        wanted = list(dict.fromkeys(hashes))
+        with self.model.get_connection() as conn:
+            for start in range(0, len(wanted), 500):
+                chunk = wanted[start : start + 500]
+                rows = conn.execute(
+                    f"SELECT hash FROM prompts WHERE hash IN ({_placeholders(chunk)}) "
+                    "AND (category IS NULL OR category != ?)",
+                    [*chunk, exclude_category],
+                ).fetchall()
+                found.update(row[0] for row in rows)
+        return found
+
+    def replace_category_prompts(
+        self,
+        category: str,
+        records: List[Dict[str, Any]],
+        *,
+        expected_ids: Optional[List[int]] = None,
+        allow_empty: bool = False,
+    ) -> Dict[str, Any]:
+        """Replace every prompt of ``category`` with ``records`` in one transaction.
+
+        Raises (and changes nothing): ``PlanChanged`` when ``expected_ids`` is
+        given and the category's prompt ids differ from it;
+        ``PairDependencyError`` when a saved pair uses one of its prompts;
+        ``EmptyReplacement`` when ``records`` is empty while the category has
+        prompts, unless ``allow_empty`` (an empty replacement that was
+        reviewed). Any other error rolls everything back too.
+
+        Returns ``{removed, imported, linked, duplicates, images, rows}``:
+        prompts removed and created, records linked to an existing prompt or
+        to an earlier record with the same text, image links added, and per
+        record ``{name, outcome}`` (``imported``/``linked``/``duplicate``).
+        """
+        with self._pair_transaction() as conn:
+            ids = [
+                row[0]
+                for row in conn.execute(
+                    "SELECT id FROM prompts WHERE category = ? ORDER BY id",
+                    (category,),
+                ).fetchall()
+            ]
+            if expected_ids is not None and ids != list(expected_ids):
+                raise PlanChanged("The imported prompts changed since the review")
+            if not records and ids and not allow_empty:
+                raise EmptyReplacement(
+                    "Nothing to import; the previous imports were kept"
+                )
+            self._ensure_category_unpaired(conn, category)
+            removed = self._delete_category_rows(conn, category, ids)
+            result = {
+                "removed": removed,
+                "imported": 0,
+                "linked": 0,
+                "duplicates": 0,
+                "images": 0,
+                "rows": [],
+            }
+            created = {}
+            for record in records:
+                self._import_record(conn, category, record, created, result)
+            return result
+
+    def _delete_category_rows(self, conn, category: str, ids: List[int]) -> int:
+        """Delete the category's prompts with their image links and tags.
+
+        No commit. The rows are selected by category (just verified against
+        ``ids`` under the write lock), so no statement binds one parameter per
+        prompt and SQLite's variable limit never caps a collection.
+        """
+        if not ids:
+            return 0
+        documents = documents_of_rows(conn, "prompt_id", ids)  # chunked
+        conn.execute(
+            f"DELETE FROM generated_images WHERE prompt_id IN ({_IN_CATEGORY})",
+            (category,),
+        )
+        release_documents(conn, documents)
+        conn.execute(
+            f"DELETE FROM prompt_tags WHERE prompt_id IN ({_IN_CATEGORY})",
+            (category,),
+        )
+        return conn.execute(
+            "DELETE FROM prompts WHERE category = ?", (category,)
+        ).rowcount
+
+    @staticmethod
+    def _ensure_category_unpaired(conn, category: str) -> None:
+        """Raise PairDependencyError when a saved pair uses a category prompt."""
+        row = conn.execute(
+            f"SELECT id FROM prompt_pairs WHERE positive_prompt_id IN ({_IN_CATEGORY}) "
+            f"OR negative_prompt_id IN ({_IN_CATEGORY}) LIMIT 1",
+            (category, category),
+        ).fetchone()
+        if row:
+            raise PairDependencyError(
+                f"Prompt belongs to saved pair {row[0]}; delete the pair first "
+                "or save changed text as a new prompt"
+            )
+
+    def _import_record(self, conn, category, record, created, result) -> None:
+        """Save one staged record on ``conn``, or link it to the prompt it repeats."""
+        prompt_hash = record["hash"]
+        name = record.get("name") or record["text"]
+        if prompt_hash in created:
+            prompt_id, outcome = created[prompt_hash], "duplicate"
+        else:
+            row = conn.execute(
+                "SELECT id FROM prompts WHERE hash = ?", (prompt_hash,)
+            ).fetchone()
+            if row is not None:
+                prompt_id, outcome = row[0], "linked"
+            else:
+                prompt_id = self._insert_prompt(
+                    conn,
+                    record["text"],
+                    category,
+                    list(record.get("tags") or []),
+                    None,
+                    None,
+                    prompt_hash,
+                )
+                outcome = "imported"
+            created[prompt_hash] = prompt_id
+        result[_COUNTERS[outcome]] += 1
+        # Linked last to first: every workspace list shows a prompt's media
+        # newest link first, so the record's first image (the LoRA cover,
+        # #355) is the one a Library card shows.
+        for image in reversed(record.get("images") or []):
+            if self._insert_image_link(conn, prompt_id, image):
+                result["images"] += 1
+        result["rows"].append({"name": name, "outcome": outcome})

database/duplicate_copies.py

@@ -0,0 +1,252 @@
+"""Records of duplicate image files: count them and move them (#249 S6, T17).
+
+A removed copy's records move to its verified kept copy before the copy is
+unlinked (``py/api/duplicate_files.py``), one file per short ``BEGIN
+IMMEDIATE`` transaction, so no row ever names a deleted file:
+
+- its image records (``generated_images``), found under every spelling the
+  file is linked by. A record moves to the kept copy, unless its prompt
+  already links the kept copy: then its saved-pair links join that record
+  and it is deleted;
+- its favorite;
+- its collection memberships (#331), joined onto the kept copy's as their
+  union under the same keys a favorite moves by;
+- its image tags (``media_image_tags`` of its byte record), joined onto the
+  kept copy's byte record as their union, in order;
+- its own image category (#332), copied to the kept copy when that copy has
+  none; the removed copy's row is kept, so a give-back loses nothing;
+- its pending Needs review items, settled: only an item still ``pending`` at
+  revision 0 with no decision row is removed (``review._SETTLE_ITEM``), so a
+  decided item is never touched.
+
+Every statement is an index search (see ``FILE_LINK_SQL``).
+"""
+
+import os
+
+from .annotation_schema import BYTES_TABLE, FILES_TABLE
+from .annotations import _BUMP_INDEX
+from .image_categories import COPY_SQL as COPY_CATEGORY_SQL
+from .image_categories import copy_image_category
+from .models import normalize_image_path, utc_now_iso
+from .review import _BUMP_REVIEW_INDEX, _PENDING_OF_LOCATOR, _SETTLE_ITEM
+from .review_changes import notify_review_change
+from .workflow_documents import documents_of_rows, release_documents
+
+LINKS_BY_FILE_SQL = "SELECT id, prompt_id FROM generated_images WHERE file_path = ?"
+LEGACY_LINKS_SQL = (
+    "SELECT id, prompt_id FROM generated_images "
+    "WHERE image_path = ? AND file_path IS NULL"
+)
+KEEPER_LINK_SQL = (
+    "SELECT id FROM generated_images WHERE prompt_id = ? AND file_path = ?"
+)
+MOVE_PAIR_LINKS_SQL = (
+    "UPDATE OR IGNORE image_prompt_pairs SET image_id = ? WHERE image_id = ?"
+)
+MOVE_LINK_SQL = (
+    "UPDATE generated_images SET image_path = ?, filename = ?, file_path = ? "
+    "WHERE id = ?"
+)
+DELETE_LINK_SQL = "DELETE FROM generated_images WHERE id = ?"
+FAVORITE_SQL = (
+    "SELECT created_at FROM image_favorites WHERE root_key = ? AND relative_path = ?"
+)
+INSERT_FAVORITE_SQL = "INSERT OR IGNORE INTO image_favorites VALUES (?, ?, ?)"
+DELETE_FAVORITE_SQL = (
+    "DELETE FROM image_favorites WHERE root_key = ? AND relative_path = ?"
+)
+COPY_MEMBERSHIPS_SQL = (
+    "INSERT OR IGNORE INTO image_collection_items "
+    "(collection_id, root_key, relative_path, added_at) "
+    "SELECT collection_id, ?, ?, added_at FROM image_collection_items "
+    "WHERE root_key = ? AND relative_path = ?"
+)
+DELETE_MEMBERSHIPS_SQL = (
+    "DELETE FROM image_collection_items WHERE root_key = ? AND relative_path = ?"
+)
+COPY_BYTE_SQL = (
+    f"SELECT b.id FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b "  # nosec B608
+    "ON b.file_id=f.id WHERE f.locator=? AND b.sha256=?"
+)
+COPY_TAG_COUNT_SQL = (
+    "SELECT COUNT(*) FROM media_image_tags WHERE byte_id=?"  # nosec B608
+)
+PENDING_COUNT_SQL = (
+    f"SELECT COUNT(*) FROM {FILES_TABLE} f "  # nosec B608
+    f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
+    "JOIN media_review_items i ON i.byte_id=b.id "
+    "WHERE f.locator=? AND +i.state='pending' AND i.revision=0 "
+    "AND NOT EXISTS (SELECT 1 FROM media_review_decisions d WHERE d.item_id=i.id)"
+)
+FILE_LINK_SQL = (
+    LINKS_BY_FILE_SQL,
+    LEGACY_LINKS_SQL,
+    KEEPER_LINK_SQL,
+    MOVE_PAIR_LINKS_SQL,
+    MOVE_LINK_SQL,
+    DELETE_LINK_SQL,
+    FAVORITE_SQL,
+    DELETE_FAVORITE_SQL,
+    COPY_MEMBERSHIPS_SQL,
+    DELETE_MEMBERSHIPS_SQL,
+    COPY_BYTE_SQL,
+    COPY_TAG_COUNT_SQL,
+    PENDING_COUNT_SQL,
+    COPY_CATEGORY_SQL,
+)
+
+
+def _file_link_rows(conn, spelled):
+    """``(id, prompt_id)`` of the image records linking one spelling."""
+    rows = list(conn.execute(LINKS_BY_FILE_SQL, (normalize_image_path(spelled),)))
+    rows += list(conn.execute(LEGACY_LINKS_SQL, (spelled,)))
+    return [tuple(row) for row in rows]
+
+
+def _copy_byte(conn, locator, sha256):
+    row = conn.execute(COPY_BYTE_SQL, (locator, sha256)).fetchone()
+    return row[0] if row else None
+
+
+class DuplicateCopyOperations:
+    """PromptDatabase mixin: the records of duplicate copies."""
+
+    def count_copy_records(self, copies) -> dict:
+        """``{path: {links, tags, review}}`` for each copy (one read snapshot).
+
+        ``copies`` maps each path to ``{spellings, locator, sha256}``:
+        ``links`` counts the image records under any spelling, ``tags`` the
+        image tags of its bytes and ``review`` its pending, undecided Needs
+        review items.
+        """
+        from .maintenance import _read_snapshot
+
+        def read(conn):
+            found = {}
+            for path, copy in copies.items():
+                ids = {
+                    row[0]
+                    for spelled in copy["spellings"]
+                    for row in _file_link_rows(conn, spelled)
+                }
+                byte_id = _copy_byte(conn, copy["locator"], copy["sha256"])
+                tags = (
+                    conn.execute(COPY_TAG_COUNT_SQL, (byte_id,)).fetchone()[0]
+                    if byte_id
+                    else 0
+                )
+                review = conn.execute(PENDING_COUNT_SQL, (copy["locator"],))
+                found[path] = {
+                    "links": len(ids),
+                    "tags": tags,
+                    "review": review.fetchone()[0],
+                }
+            return found
+
+        return _read_snapshot(self.model.db_path, read)
+
+    def move_copy_records(self, move) -> dict:
+        """Move one removed copy's records to its kept copy (one transaction).
+
+        ``move``: ``removed`` and ``kept`` (the spellings each file is linked
+        by), ``targets`` (``{removed spelling: kept spelling}``, the spelling
+        a moved record takes; else ``kept_path``), ``favorites`` (``(root key,
+        removed key, kept root key, kept key)`` per listed root holding the
+        file; they move its favorite and its collection memberships),
+        ``removed_locator``, ``kept_locator``, ``sha256`` and the kept copy's
+        ``kept_size`` and ``kept_mtime_ns``; ``categories``: ``(removed file
+        key, kept file key)`` pairs of the image categories (#332). Returns
+        ``{moved, merged, favorites, tags, review, collections, categories}``
+        (``collections``: the memberships inserted on the kept copy;
+        ``categories``: the image categories copied to the kept copy).
+        """
+        counts = dict.fromkeys(
+            ("moved", "merged", "favorites", "tags", "review", "collections"), 0
+        )
+        kept_keys = [normalize_image_path(path) for path in move["kept"]]
+        with self._pair_transaction() as conn:
+            for spelled in dict.fromkeys(move["removed"]):
+                target = move["targets"].get(spelled, move["kept_path"])
+                keys = list(dict.fromkeys(kept_keys + [normalize_image_path(target)]))
+                for image_id, prompt_id in _file_link_rows(conn, spelled):
+                    existing = None
+                    for key in keys:
+                        existing = conn.execute(
+                            KEEPER_LINK_SQL, (prompt_id, key)
+                        ).fetchone()
+                        if existing is not None:
+                            break
+                    if existing is not None:
+                        conn.execute(MOVE_PAIR_LINKS_SQL, (existing[0], image_id))
+                        documents = documents_of_rows(conn, "id", [image_id])
+                        conn.execute(DELETE_LINK_SQL, (image_id,))
+                        release_documents(conn, documents)
+                        counts["merged"] += 1
+                        continue
+                    conn.execute(
+                        MOVE_LINK_SQL,
+                        (
+                            target,
+                            os.path.basename(target),
+                            normalize_image_path(target),
+                            image_id,
+                        ),
+                    )
+                    counts["moved"] += 1
+            for favorite in move.get("favorites", ()):
+                counts["favorites"] += _move_favorite(conn, *favorite)
+                counts["collections"] += _move_memberships(conn, *favorite)
+            counts["tags"] = self._move_copy_tags(conn, move)
+            counts["categories"] = sum(
+                copy_image_category(conn, removed, kept)
+                for removed, kept in move.get("categories", ())
+            )
+            for (item_id,) in conn.execute(
+                _PENDING_OF_LOCATOR, (move["removed_locator"],)
+            ).fetchall():
+                counts["review"] += conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
+            if counts["review"]:
+                conn.execute(_BUMP_REVIEW_INDEX)
+        if counts["review"]:
+            notify_review_change(self.model.db_path)
+        return counts
+
+    def _move_copy_tags(self, conn, move):
+        """Join the removed copy's image tags onto the kept copy's; the count added."""
+        removed = _copy_byte(conn, move["removed_locator"], move["sha256"])
+        if removed is None:
+            return 0
+        tags = self._byte_tags(conn, removed)
+        if not tags:
+            return 0
+        now = utc_now_iso()
+        file_id = self._annotation_owner(conn, move["kept_locator"], now)
+        byte_id, _revision, created, moved = self._annotation_byte(
+            conn, file_id, move["sha256"], move["kept_size"], move["kept_mtime_ns"], now
+        )
+        current = self._byte_tags(conn, byte_id)
+        union = current + [tag for tag in tags if tag not in current]
+        if union != current:
+            self._write_image_tags(conn, byte_id, union)
+        elif created or moved:
+            conn.execute(_BUMP_INDEX)
+        return len(union) - len(current)
+
+
+def _move_favorite(conn, root_key, removed_key, kept_root, kept_key):
+    row = conn.execute(FAVORITE_SQL, (root_key, removed_key)).fetchone()
+    if row is None:
+        return 0
+    conn.execute(INSERT_FAVORITE_SQL, (kept_root, kept_key, row[0]))
+    conn.execute(DELETE_FAVORITE_SQL, (root_key, removed_key))
+    return 1
+
+
+def _move_memberships(conn, root_key, removed_key, kept_root, kept_key):
+    """Copy a removed copy's memberships to the kept copy; the rows inserted."""
+    inserted = conn.execute(
+        COPY_MEMBERSHIPS_SQL, (kept_root, kept_key, root_key, removed_key)
+    ).rowcount
+    conn.execute(DELETE_MEMBERSHIPS_SQL, (root_key, removed_key))
+    return inserted

database/execution_contexts.py

@@ -7,6 +7,11 @@ from typing import Callable, ContextManager, Iterator
 
 try:
     from ..utils.execution_history_events import RunFinish, encode_command
+    from ..utils.hashing import (
+        generate_prompt_hash,
+        stored_prompt_text,
+        stored_prompt_text_sql,
+    )
     from ..utils.execution_contexts import (
         CAPTURE_REASONS,
         ContextBundle,
@@ -15,6 +20,11 @@ try:
     )
 except ImportError:
     from utils.execution_history_events import RunFinish, encode_command
+    from utils.hashing import (
+        generate_prompt_hash,
+        stored_prompt_text,
+        stored_prompt_text_sql,
+    )
     from utils.execution_contexts import (
         CAPTURE_REASONS,
         ContextBundle,
@@ -134,7 +144,8 @@ def _insert_exact(
 
 def _role_hints(role: str) -> str:
     return (
-        f"SELECT r.{role}_prompt_id, tx.text_value "
+        f"SELECT r.{role}_prompt_id, tx.text_value, "
+        f"{stored_prompt_text_sql('tx.text_value')} "
         "FROM execution_run_contexts r "
         "JOIN execution_contexts c ON c.context_id = r.context_id "
         f"JOIN execution_prompt_snapshots s ON s.snapshot_id = c.{role}_snapshot_id "
@@ -146,57 +157,78 @@ def _role_hints(role: str) -> str:
 
 # Set form of the read-time prompt-hint rule in get_execution_contexts():
 # a stored role hint is live only in the current storage epoch, and only when
-# its exact raw snapshot text matches exactly one prompt, which is the hinted
-# one (_prompt_hint). Evaluated once per statement, never per prompt row.
+# the hinted prompt is the one match of _prompt_hint: by text (the raw
+# snapshot text or its stored form, outer whitespace stripped, #438), or, only
+# when no prompt has either text, by dedup hash (case folded too, #460).
+# The hash is a fallback, so a hint #438 kept is never made ambiguous.
+# Needs register_prompt_functions (pm_prompt_hash). The hint rows are the
+# outer loop (CROSS JOIN), so each prompts lookup is by index.
 # Defines the CTE ``live_hint_prompts(prompt_id)``; prefix it with WITH.
 LIVE_HINT_CTES = (
-    "stored_hints(prompt_id, raw) AS ("
+    "stored_hints(prompt_id, raw, stored) AS ("
     + _role_hints("positive")
     + " UNION "
     + _role_hints("negative")
-    + "), unique_hint_texts(prompt_id, raw) AS ("
-    "SELECT MIN(id), text COLLATE BINARY FROM prompts "
-    "WHERE text COLLATE BINARY IN (SELECT raw FROM stored_hints) "
-    "GROUP BY text COLLATE BINARY HAVING COUNT(*) = 1"
+    + "), hint_matches(prompt_id, raw, stored, match_id) AS ("
+    "SELECT h.prompt_id, h.raw, h.stored, p.id FROM stored_hints h "
+    "CROSS JOIN prompts p ON p.text COLLATE BINARY IN (h.raw, h.stored) "
+    "UNION ALL "
+    "SELECT h.prompt_id, h.raw, h.stored, p.id FROM stored_hints h "
+    "CROSS JOIN prompts p ON p.hash = pm_prompt_hash(h.raw) "
+    "WHERE NOT EXISTS (SELECT 1 FROM prompts q "
+    "WHERE q.text COLLATE BINARY IN (h.raw, h.stored))"
     "), live_hint_prompts(prompt_id) AS ("
-    "SELECT h.prompt_id FROM stored_hints h JOIN unique_hint_texts u "
-    "ON u.prompt_id = h.prompt_id AND u.raw = h.raw COLLATE BINARY)"
+    "SELECT prompt_id FROM hint_matches GROUP BY prompt_id, raw, stored "
+    "HAVING COUNT(*) = 1 AND MIN(match_id) = prompt_id)"
 )
 LIVE_HINT_PROMPT_IDS_SQL = (
     "WITH " + LIVE_HINT_CTES + " SELECT DISTINCT prompt_id FROM live_hint_prompts"
 )
 
+_BY_TEXT = "SELECT id FROM prompts WHERE text COLLATE BINARY IN (?,?) LIMIT 2"
+
+
+def _text_matches(conn: sqlite3.Connection, raw: str) -> list:
+    return [row[0] for row in conn.execute(_BY_TEXT, (raw, stored_prompt_text(raw)))]
+
 
 def _prompt_hint(conn: sqlite3.Connection, raw: str | None) -> int | None:
+    """The one prompt that stores ``raw`` (#438), else the one it counts toward.
+
+    By text first: ``raw`` or ``raw.strip()`` (the exact form keeps hints of
+    older rows that kept outer whitespace). Only when no prompt has either
+    text, by the dedup hash, which also folds case (#460).
+    """
     if raw is None:
         return None
-    rows = conn.execute(
-        "SELECT id FROM prompts WHERE text=? COLLATE BINARY LIMIT 2", (raw,)
-    ).fetchall()
-    return rows[0][0] if len(rows) == 1 else None
+    matches = _text_matches(conn, raw)
+    if not matches:
+        matches = [
+            row[0]
+            for row in conn.execute(
+                "SELECT id FROM prompts WHERE hash=? LIMIT 2",
+                (generate_prompt_hash(raw),),
+            )
+        ]
+    return matches[0] if len(matches) == 1 else None
 
 
 def _hints(
     conn: sqlite3.Connection, positive: str | None, negative: str | None
 ) -> tuple:
     pos_id, neg_id = _prompt_hint(conn, positive), _prompt_hint(conn, negative)
+    stored = None if negative is None else stored_prompt_text(negative)
     pair_id = None
+    # A known negative that names no prompt because several share its text
     negative_ambiguous = (
-        negative is not None
-        and neg_id is None
-        and conn.execute(
-            "SELECT 1 FROM prompts WHERE text=? COLLATE BINARY LIMIT 1", (negative,)
-        ).fetchone()
-        is not None
+        stored is not None and neg_id is None and bool(_text_matches(conn, negative))
     )
-    if pos_id is not None and negative is not None and not negative_ambiguous:
+    if pos_id is not None and stored is not None and not negative_ambiguous:
         rows = conn.execute(
-            "SELECT pp.id FROM prompt_pairs pp LEFT JOIN prompts n ON "
-            "n.id=pp.negative_prompt_id "
+            "SELECT pp.id FROM prompt_pairs pp "
             "WHERE pp.positive_prompt_id=? AND ((pp.negative_prompt_id IS NULL AND "
-            "?='') "
-            "OR (pp.negative_prompt_id=? AND n.text=? COLLATE BINARY)) LIMIT 2",
-            (pos_id, negative, neg_id, negative),
+            "?='') OR pp.negative_prompt_id=?) LIMIT 2",
+            (pos_id, stored, neg_id),
         ).fetchall()
         if len(rows) == 1:
             pair_id = rows[0][0]

database/execution_history.py

@@ -196,3 +196,14 @@ class ExecutionHistoryOperations:
             .fetchone()
         )
         return dict(row) if row is not None else None
+
+
+def rekey_history_restore(old_path: str, new_path: str) -> None:
+    """Move the observers of ``old_path`` to ``new_path`` (Move library)."""
+    old, new = _database_identity(old_path), _database_identity(new_path)
+    if old == new:
+        return
+    with _restore_lock:
+        observers = _restore_observers.pop(old, None)
+        if observers:
+            _restore_observers.setdefault(new, {}).update(observers)

database/execution_outputs.py

@@ -4,6 +4,8 @@ import sqlite3
 from contextlib import ExitStack
 from typing import Callable, ContextManager
 
+from .run_attribution import notify_outputs_sealed
+
 try:
     from ..utils.execution_history_events import RunFinish, encode_command
     from ..utils.execution_outputs import OutputBundle, validate_output_bundle
@@ -113,6 +115,10 @@ class ExecutionOutputOperations:
                     "VALUES(?,?,?,?,?,?,?,?)",
                     [(run_id, *reference) for reference in prepared],
                 )
+        # Committed: images waiting for this run may now be attributed (#342)
+        notify_outputs_sealed(
+            run_id, [dict(zip(_REFERENCE_FIELDS, row)) for row in prepared]
+        )
         return True
 
     def get_execution_outputs(self, run_id: str) -> dict:

database/health.py

@@ -0,0 +1,376 @@
+"""The read-only health check behind System › Overview and Maintenance.
+
+``run_health_checks`` opens its own ``mode=ro`` connection, reads one
+snapshot and closes the connection in ``finally``; it never writes. Each
+check reports ``passed``, ``issues`` (with an exact ``count``; the
+integrity check stops at ``INTEGRITY_LIMIT`` problems and then says
+``capped``),
+``unavailable`` (it could not be checked here, so ``count`` is None, or
+``missing_files`` checked only some sources and found nothing) or ``failed``
+(it raised; the other checks still run). ``missing_files`` also reports each
+source and whether the check was ``partial``; it reads the image rows inside
+the snapshot and checks their files after the snapshot ended, so a slow source
+never holds the WAL. At most ``MAX_DETAIL`` detail strings are returned per
+check, and none of them carries a server path.
+"""
+
+import datetime
+import logging
+import ntpath
+import os
+import sqlite3
+from pathlib import PurePosixPath, PureWindowsPath
+from urllib.parse import quote
+
+try:
+    from ..utils.hashing import register_prompt_functions
+    from .models import normalize_image_path
+except ImportError:
+    from utils.hashing import register_prompt_functions
+    from database.models import normalize_image_path
+
+logger = logging.getLogger("prompt_manager.database.health")
+
+MAX_DETAIL = 10  # detail strings returned per check
+INTEGRITY_LIMIT = 100  # PRAGMA integrity_check(N): problems SQLite reports
+BATCH_ROWS = 100  # image rows read per fetch
+REVIEW_UNAVAILABLE = "Needs review is not available yet."
+CHECK_FAILED = "The check could not finish."
+# Check ids and the labels of the concept's health report, in its order;
+# pair uniqueness and consistency are the added rows (D63, now D95).
+CHECKS = (
+    ("integrity", "Integrity check"),
+    ("foreign_keys", "Foreign keys & pair references"),
+    ("canonical_hashes", "Canonical hashes"),
+    ("pair_uniqueness", "Pair uniqueness"),
+    ("missing_files", "Missing files"),
+    ("ambiguous_metadata", "Ambiguous metadata"),
+    ("consistency", "Consistency"),
+)
+_DATABASE_CHECKS = (
+    "integrity",
+    "foreign_keys",
+    "canonical_hashes",
+    "pair_uniqueness",
+    "missing_files",
+    "consistency",
+)
+
+
+def iter_consistency_issues(conn):
+    """The legacy consistency issues: tag and image rows of a missing prompt."""
+    for ref in conn.execute(
+        "SELECT pt.prompt_id FROM prompt_tags pt "
+        "LEFT JOIN prompts p ON pt.prompt_id = p.id WHERE p.id IS NULL"
+    ):
+        yield f"prompt_tags entry references non-existent prompt {ref[0]}"
+    for ref in conn.execute(
+        "SELECT gi.id, gi.prompt_id FROM generated_images gi "
+        "LEFT JOIN prompts p ON gi.prompt_id = p.id WHERE p.id IS NULL"
+    ):
+        yield f"Image {ref[0]} references non-existent prompt {ref[1]}"
+
+
+def _result(count, detail, *, status=None):
+    return {
+        "status": status or ("issues" if count else "passed"),
+        "count": count,
+        "detail": detail[:MAX_DETAIL],
+    }
+
+
+def _counted(lines):
+    """(count, first MAX_DETAIL lines) of an iterable, consumed once."""
+    count, detail = 0, []
+    for line in lines:
+        count += 1
+        if len(detail) < MAX_DETAIL:
+            detail.append(line)
+    return _result(count, detail)
+
+
+def readonly_uri(db_path, *, windows=None):
+    """The ``mode=ro`` SQLite URI of *db_path*, on this platform or the given one.
+
+    pathlib spells a UNC path ``\\\\server\\share\\x.db`` as
+    ``file://server/share/x.db``, a remote authority SQLite refuses to open.
+    With an empty authority (``file:////server/share/x.db``) SQLite hands
+    Windows the ``//server/share`` path, which it opens as the UNC path.
+    """
+    if windows is None:
+        windows = os.name == "nt"
+        db_path = os.path.abspath(db_path)
+    if not windows:
+        return PurePosixPath(db_path).as_uri() + "?mode=ro"
+    path = PureWindowsPath(db_path)
+    if ntpath.splitdrive(str(path))[0].startswith("\\\\"):
+        return "file:" + quote("//" + path.as_posix()) + "?mode=ro"
+    return path.as_uri() + "?mode=ro"
+
+
+def _integrity(conn, _roots):
+    rows = [
+        row[0] for row in conn.execute(f"PRAGMA integrity_check({INTEGRITY_LIMIT})")
+    ]
+    if rows == ["ok"]:
+        return {**_result(0, []), "capped": False}
+    # SQLite stops listing at the limit: there may be more problems than that.
+    return {**_result(len(rows), rows), "capped": len(rows) >= INTEGRITY_LIMIT}
+
+
+def _foreign_keys(conn, _roots):
+    return _counted(
+        f"{table} row {rowid} references a missing {parent} row"
+        for table, rowid, parent, _fkid in conn.execute("PRAGMA foreign_key_check")
+    )
+
+
+def _canonical_hashes(conn, _roots):
+    register_prompt_functions(conn)  # pm_prompt_hash, shared with history (#460)
+    return _counted(
+        f"Prompt {row[0]}: the stored hash does not match its text"
+        for row in conn.execute(
+            "SELECT id FROM prompts WHERE hash IS NOT pm_prompt_hash(text) ORDER BY id"
+        )
+    )
+
+
+def _pair_uniqueness(conn, _roots):
+    groups = conn.execute(
+        "SELECT positive_prompt_id,negative_prompt_id,COUNT(*) FROM prompt_pairs "
+        "GROUP BY positive_prompt_id,negative_prompt_id HAVING COUNT(*)>1 "
+        "ORDER BY positive_prompt_id,negative_prompt_id"
+    ).fetchall()
+    detail = [
+        f"Positive prompt {positive} with "
+        + ("no negative" if negative is None else f"negative {negative}")
+        + f": {rows} pair rows"
+        for positive, negative, rows in groups
+    ]
+    # The surplus rows: what keeping one row per pair would leave out.
+    return _result(sum(rows - 1 for _p, _n, rows in groups), detail)
+
+
+def _consistency(conn, _roots):
+    return _counted(iter_consistency_issues(conn))
+
+
+def _readable(root):
+    return os.path.isdir(root) and os.access(root, os.R_OK | os.X_OK)
+
+
+def _root_prefix(root):
+    return normalize_image_path(root).rstrip("\\/") + os.sep
+
+
+def _source_name(root, index):
+    """A source's folder name for the report (never its server path)."""
+    return os.path.basename(str(root).rstrip("\\/")) or f"Source {index + 1}"
+
+
+def _owner(path, prefixes):
+    """Index of the longest source prefix *path* lies under, or None."""
+    owner, longest = None, -1
+    for index, prefix in enumerate(prefixes):
+        if path.startswith(prefix) and len(prefix) > longest:
+            owner, longest = index, len(prefix)
+    return owner
+
+
+def _image_owner(image_path, file_path, prefixes):
+    spelled = file_path or normalize_image_path(image_path)
+    owner = _owner(spelled, prefixes)
+    if owner is None:
+        owner = _owner(normalize_image_path(os.path.realpath(image_path)), prefixes)
+    return owner
+
+
+def readable_roots(roots):
+    """Whether each source can be read now (an unmounted drive cannot)."""
+    return [_readable(root) for root in roots]
+
+
+def link_states(rows, roots, readable):
+    """``(row, owner, state)`` for each image row ``(id, image_path, filename,
+    file_path)``: ``owner`` is the index of the source it lies under (None
+    outside every source) and ``state`` is ``missing``, ``present``,
+    ``unavailable`` (its source cannot be read, so the file is not looked at)
+    or ``outside``. ``readable`` is ``readable_roots(roots)``. The health
+    check's ``missing_files`` and the maintenance missing-link plan both
+    judge links this way, so their counts agree.
+    """
+    prefixes = [_root_prefix(root) for root in roots]
+    for row in rows:
+        owner = _image_owner(row[1], row[3], prefixes)
+        if owner is None:
+            yield row, None, "outside"
+        elif not readable[owner]:
+            yield row, owner, "unavailable"
+        elif os.path.exists(row[1]):
+            yield row, owner, "present"
+        else:
+            yield row, owner, "missing"
+
+
+def _unavailable_line(unavailable, *, partial):
+    noun = "source is" if unavailable == 1 else "sources are"
+    effect = "its files were" if unavailable == 1 else "their files were"
+    if not partial:
+        effect = "missing files were"
+    return f"{unavailable} image {noun} unavailable, so {effect} not counted."
+
+
+def _missing_files(conn, roots):
+    """Links whose file is gone, counted per source (only readable ones).
+
+    Each source reports ``checked`` with its own count, or ``unavailable``
+    (an unmounted drive) with no count: its links are never turned into a
+    count, while the readable sources still are. The check's ``count`` is the
+    readable sources' total (None when none is readable) and ``partial`` says
+    some sources were not checked. Links outside every source are named in the
+    detail, not counted.
+
+    Inside the snapshot this only reads the image rows; it returns the step
+    that checks their files, which runs after the snapshot ended.
+    """
+    if not roots:
+        result = _result(None, ["No image source is available."], status="unavailable")
+        return {**result, "partial": False, "sources": []}
+    rows = []
+    cursor = conn.execute(
+        "SELECT id,image_path,filename,file_path FROM generated_images ORDER BY id"
+    )
+    while batch := cursor.fetchmany(BATCH_ROWS):
+        rows.extend(batch)
+    return lambda: _check_files(rows, roots)
+
+
+def _check_files(rows, roots):
+    """The ``missing_files`` result of the image *rows*, from the file system."""
+    readable = readable_roots(roots)
+    sources = [
+        {
+            "name": _source_name(root, index),
+            "status": "checked" if ok else "unavailable",
+            "count": 0 if ok else None,
+        }
+        for index, (root, ok) in enumerate(zip(roots, readable))
+    ]
+    unavailable = readable.count(False)
+    if not any(readable):
+        result = _result(
+            None, [_unavailable_line(unavailable, partial=False)], status="unavailable"
+        )
+        return {**result, "partial": False, "sources": sources}
+    outside, missing = 0, []
+    for row, owner, state in link_states(rows, roots, readable):
+        if state == "outside":
+            outside += 1
+        elif state == "missing":
+            image_id, image_path, filename, _file_path = row
+            sources[owner]["count"] += 1
+            if len(missing) < MAX_DETAIL:
+                name = filename or os.path.basename(image_path)
+                missing.append(f"Image {image_id}: {name} is missing")
+    count = sum(source["count"] or 0 for source in sources)
+    detail = [_unavailable_line(unavailable, partial=True)] if unavailable else []
+    detail += missing
+    if outside:
+        links = "link" if outside == 1 else "links"
+        verb = "was" if outside == 1 else "were"
+        detail.append(
+            f"{outside} {links} outside the image sources {verb} not checked."
+        )
+    status = None if count or not unavailable else "unavailable"
+    result = _result(count, detail, status=status)
+    return {**result, "partial": bool(unavailable), "sources": sources}
+
+
+def _ambiguous(review_count):
+    if review_count is None:
+        return _result(None, [REVIEW_UNAVAILABLE], status="unavailable")
+    return _result(review_count, [])
+
+
+def overall_status(checks) -> str:
+    """``passed`` only when every check passed; any check not run is incomplete."""
+    statuses = {check["status"] for check in checks}
+    if statuses & {"unavailable", "failed"} or any(
+        check.get("partial") for check in checks
+    ):
+        return "incomplete"
+    return "issues" if "issues" in statuses else "passed"
+
+
+def _failed():
+    return _result(None, [CHECK_FAILED], status="failed")
+
+
+def _run_check(check_id, step, *args):
+    """One check's result, or None (reported as failed) when it raised."""
+    try:
+        return step(*args)
+    except Exception:  # any error fails this check only; the others still run
+        logger.exception(f"Health check {check_id} failed")
+        return None
+
+
+def _run_database_checks(db_path, roots):
+    """Each database check's result (None when it raised), from one snapshot.
+
+    A check may return a step instead of its result: the step runs after the
+    snapshot ended and the connection closed (``missing_files``' file checks).
+    """
+    results = dict.fromkeys(_DATABASE_CHECKS)
+    try:
+        conn = sqlite3.connect(readonly_uri(db_path), uri=True)
+    except sqlite3.Error:
+        logger.exception("Health check could not open the database")
+        return results
+    try:
+        conn.execute("BEGIN")  # one read snapshot for every check
+        for check_id in _DATABASE_CHECKS:
+            runner = globals()["_" + check_id]  # looked up per run (tests patch it)
+            results[check_id] = _run_check(check_id, runner, conn, roots)
+    except sqlite3.Error:
+        logger.exception("Health check could not read the database")
+    finally:
+        conn.close()
+    for check_id, result in results.items():
+        if callable(result):
+            results[check_id] = _run_check(check_id, result)
+    return results
+
+
+class HealthOperations:
+    """Mixin for PromptDatabase: the read-only System health check."""
+
+    def run_health_checks(self, roots, *, review_count=None) -> dict:
+        """Run every check and return a ``HealthReport``.
+
+        Args:
+            roots: The image source directories to judge missing files under:
+                the effective sources plus any configured source that is not
+                available (reported as an unavailable source of ``missing_files``).
+            review_count: The Needs-review pending count, or None while no
+                review queue exists (``ambiguous_metadata`` is then unavailable).
+        """
+        if review_count is not None and (
+            type(review_count) is not int or review_count < 0
+        ):
+            raise ValueError("review_count must be a non-negative integer or None")
+        roots = [str(root) for root in roots]
+        database = _run_database_checks(self.model.db_path, roots)
+        checks = []
+        for check_id, label in CHECKS:
+            if check_id == "ambiguous_metadata":
+                result = _ambiguous(review_count)
+            else:
+                result = database[check_id] or _failed()
+            checks.append({"id": check_id, "label": label, **result})
+        return {
+            "success": True,
+            "checked_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+            "status": overall_status(checks),
+            "checks": checks,
+        }

database/history_reads.py

@@ -15,6 +15,11 @@ from .execution_history import HistoryEpochChanged
 from .execution_outputs import _capture, _REFERENCE_COLUMNS
 from .library import _ELIGIBLE
 
+try:
+    from ..utils.hashing import prompt_key, stored_prompt_text
+except ImportError:
+    from utils.hashing import prompt_key, stored_prompt_text
+
 RunKey = tuple[int, str, int, str]
 
 
@@ -100,14 +105,43 @@ _ORDER = (
 )
 # Stale-hint rows a newest-pair lookup inspects before it reports none.
 _PAIR_RUN_CANDIDATES = 16
+# The recorded texts of one scope, as a JSON array parameter (_scope_text_ids).
+_IN_SCOPE = "IN (SELECT value FROM json_each(?))"
 _MATCHING = (
     "SELECT DISTINCT rc.run_id FROM execution_prompt_snapshots s "
     "JOIN execution_contexts c ON c.positive_snapshot_id=s.snapshot_id "
     "JOIN execution_run_contexts rc ON rc.context_id=c.context_id "
-    "WHERE s.raw_text_id=?"
+    f"WHERE s.raw_text_id {_IN_SCOPE}"
 )
 
 
+def _scope_text_ids(conn: sqlite3.Connection, key: str) -> str:
+    """Every recorded raw text whose prompt key is ``key``, as a JSON array.
+
+    A run keeps its exact raw text, while the Library saves and counts it as
+    the prompt with its key (outer whitespace #438, case #460). The values are
+    compared, never only their digests; ``pm_prompt_key`` is Python's key.
+    """
+    return json.dumps(
+        [
+            row[0]
+            for row in conn.execute(
+                # Only snapshot raw texts: effective and affix texts are skipped
+                "SELECT t.text_id FROM (SELECT DISTINCT raw_text_id AS text_id "
+                "FROM execution_prompt_snapshots) s CROSS JOIN "
+                "execution_context_texts t WHERE t.text_id=s.text_id AND "
+                "pm_prompt_key(t.text_value)=? COLLATE BINARY",
+                (key,),
+            )
+        ]
+    )
+
+
+def _scope_text_id(raw: str | None) -> str | None:
+    """The scope digest of a raw text: that of its prompt key."""
+    return None if raw is None else content_digest("text", (prompt_key(raw),))
+
+
 def _limit(limit: int) -> None:
     if type(limit) is not int or not 1 <= limit <= 100:
         raise ValueError("History limit must be an integer from 1 to 100")
@@ -255,8 +289,15 @@ class HistoryReadOperations:
         if saved is None:
             return None
         text = saved["text"]
-        raw_id = None if text is None else content_digest("text", (text,))
-        if after is not None and after.raw_text_id != raw_id:
+        raw_id = _scope_text_id(text)
+        # A position from before #460 names the scope by the stripped text
+        # alone; it is the same scope, so a page open across the upgrade resumes.
+        legacy_id = (
+            None
+            if text is None
+            else content_digest("text", (stored_prompt_text(text),))
+        )
+        if after is not None and after.raw_text_id not in (raw_id, legacy_id):
             raise ValueError("History position text scope changed")
         group = {
             "positive_id": positive_id,
@@ -273,18 +314,11 @@ class HistoryReadOperations:
             next_position=None,
             resume_position=None,
         )
-        if (
-            raw_id is None
-            or conn.execute(
-                "SELECT 1 FROM execution_context_texts WHERE text_id=? "
-                "AND text_value=? COLLATE BINARY",
-                (raw_id, text),
-            ).fetchone()
-            is None
-        ):
+        if raw_id is None:
             return result
+        scope = _scope_text_ids(conn, prompt_key(text))
         where = f"r.run_id IN ({_MATCHING})"
-        params = [raw_id]
+        params = [scope]
         if after is not None:
             where += f" AND {_KEY}<=(?,?,?,?)"
             params.extend(after.upper)
@@ -338,9 +372,9 @@ class HistoryReadOperations:
             "LEFT JOIN execution_prompt_snapshots n "
             "ON n.snapshot_id=c.negative_snapshot_id "
             "LEFT JOIN execution_context_texts t ON t.text_id=n.raw_text_id "
-            f"WHERE rc.run_id IN ({placeholders}) AND p.raw_text_id=? "
+            f"WHERE rc.run_id IN ({placeholders}) AND p.raw_text_id {_IN_SCOPE} "
             "ORDER BY rc.anchor_id COLLATE BINARY,rc.run_id COLLATE BINARY",
-            (*by_run, raw_id),
+            (*by_run, scope),
         )
         summaries = {}
         evidence_by_context = {}
@@ -438,6 +472,7 @@ class HistoryReadOperations:
         if saved["text"] is None:
             return {"run": None}
         epoch = self.get_execution_epoch()
+        key = prompt_key(saved["text"])
         candidates = conn.execute(
             "SELECT r.run_id,rc.anchor_id,rc.context_id,"
             "c.positive_snapshot_id,c.negative_snapshot_id "
@@ -445,10 +480,11 @@ class HistoryReadOperations:
             "ON rc.run_id=r.run_id JOIN execution_contexts c "
             "ON c.context_id=rc.context_id JOIN execution_prompt_snapshots s "
             "ON s.snapshot_id=c.positive_snapshot_id "
-            "WHERE s.raw_text_id=? AND rc.pair_id=? AND rc.reference_epoch=? "
+            f"WHERE s.raw_text_id {_IN_SCOPE} AND rc.pair_id=? "
+            "AND rc.reference_epoch=? "
             f"ORDER BY {_ORDER},rc.anchor_id COLLATE BINARY LIMIT ?",
             (
-                content_digest("text", (saved["text"],)),
+                _scope_text_ids(conn, key),
                 pair_id,
                 epoch,
                 _PAIR_RUN_CANDIDATES,
@@ -457,7 +493,11 @@ class HistoryReadOperations:
         for row in candidates:
             positive = _values(conn, row["positive_snapshot_id"])
             negative = _values(conn, row["negative_snapshot_id"])
-            if positive is None or positive["raw"] != saved["text"]:
+            if (
+                positive is None
+                or positive["raw"] is None
+                or prompt_key(positive["raw"]) != key
+            ):
                 continue
             hints = _hints(conn, positive["raw"], negative["raw"] if negative else None)
             if hints[2] == pair_id:
@@ -477,11 +517,9 @@ class HistoryReadOperations:
         snapshots = {
             row["anchor_id"]: row
             for row in conn.execute(
-                "SELECT rc.anchor_id,c.positive_snapshot_id,c.negative_snapshot_id,"
-                "p.raw_text_id AS positive_raw_text_id "
+                "SELECT rc.anchor_id,c.positive_snapshot_id,c.negative_snapshot_id "
                 "FROM execution_run_contexts rc JOIN execution_contexts c "
-                "ON c.context_id=rc.context_id LEFT JOIN execution_prompt_snapshots p "
-                "ON p.snapshot_id=c.positive_snapshot_id WHERE rc.run_id=?",
+                "ON c.context_id=rc.context_id WHERE rc.run_id=?",
                 (run_id,),
             )
         }
@@ -491,12 +529,12 @@ class HistoryReadOperations:
             context.update(
                 {
                     key: row[key]
-                    for key in (
-                        "positive_snapshot_id",
-                        "negative_snapshot_id",
-                        "positive_raw_text_id",
-                    )
-                }
+                    for key in ("positive_snapshot_id", "negative_snapshot_id")
+                },
+                # The history scope this context belongs to (#438)
+                positive_raw_text_id=_scope_text_id(
+                    (context["positive"] or {}).get("raw")
+                ),
             )
             pair = context["pair_id"]
             context["navigation"] = {

database/image_categories.py

@@ -0,0 +1,251 @@
+"""Image categories (#332): an image file's own category.
+
+An image shows its own category when it has one, otherwise the category its
+link supplies (its recipe's, else its positive prompt's); this table holds
+only the own values. A row is keyed by the file key: the deepest configured
+output source that contains the file (its real, case-normalized path) and the
+path below it, case-folded where the OS folds case
+(``py/api/images.py`` ``image_category_key``). Without nested sources the key
+equals a favorite's key. Nothing follows a moved file: its row stays and is
+hidden until a file appears at the same path again.
+
+Writes validate strictly (the route strips the value and applies
+``validate_category``). Startup checks the structure only, so a value saved
+under older rules never stops the library opening; restore also checks the
+values. Optional in old backups, required once a database is current.
+"""
+
+import re
+import sqlite3
+from typing import Iterable
+
+TABLE = "image_categories"
+INDEX = "idx_image_categories_category"
+MAX_KEY_LENGTH = 4096
+MAX_CATEGORY_LENGTH = 100
+_DDL = (
+    f"CREATE TABLE {TABLE} (root_key TEXT NOT NULL, relative_path TEXT NOT NULL, "
+    "category TEXT NOT NULL, updated_at TEXT NOT NULL, "
+    "PRIMARY KEY (root_key, relative_path))"
+)
+_INDEX_DDL = f"CREATE INDEX {INDEX} ON {TABLE}(category)"
+_AUTOINDEX = f"sqlite_autoindex_{TABLE}_1"
+_COLUMNS = (
+    ("root_key", "TEXT", 1, 1),
+    ("relative_path", "TEXT", 1, 2),
+    ("category", "TEXT", 1, 0),
+    ("updated_at", "TEXT", 1, 0),
+)
+# A category may not hold a character below U+0020 (validate_category).
+_CONTROL_GLOB = "*[" + "".join(chr(code) for code in range(1, 32)) + "]*"
+
+CATEGORY_OF_SQL = (
+    f"SELECT category FROM {TABLE} WHERE root_key = ? AND relative_path = ?"
+)
+CATEGORIES_OF_ROOT_SQL = (
+    f"SELECT relative_path, category FROM {TABLE} WHERE root_key = ?"
+)
+UPSERT_SQL = (
+    f"INSERT INTO {TABLE} (root_key, relative_path, category, updated_at) "
+    "VALUES (?, ?, ?, ?) ON CONFLICT(root_key, relative_path) "
+    "DO UPDATE SET category = excluded.category, updated_at = excluded.updated_at"
+)
+DELETE_SQL = f"DELETE FROM {TABLE} WHERE root_key = ? AND relative_path = ?"
+# The duplicate cleanup's copy (#332 MC11): the removed copy's row is copied
+# to the kept copy only when the kept copy has none; it is never deleted.
+COPY_SQL = (
+    f"INSERT OR IGNORE INTO {TABLE} (root_key, relative_path, category, updated_at) "
+    f"SELECT ?, ?, category, updated_at FROM {TABLE} "
+    "WHERE root_key = ? AND relative_path = ?"
+)
+STATEMENTS = (CATEGORY_OF_SQL, CATEGORIES_OF_ROOT_SQL, UPSERT_SQL, DELETE_SQL, COPY_SQL)
+
+
+def _compact(sql: str) -> str:
+    return re.sub(r"\s+", "", sql or "").lower()
+
+
+def _structure_error(
+    conn: sqlite3.Connection, allow_missing: bool, allow_missing_index: bool = False
+) -> str | None:
+    objects = conn.execute(
+        "SELECT type,name,sql FROM sqlite_master WHERE name=? "
+        "OR (type IN ('index','trigger') AND tbl_name=?)",
+        (TABLE, TABLE),
+    ).fetchall()
+    if not objects:
+        return None if allow_missing else "Missing image categories schema"
+    tables = [sql for kind, _name, sql in objects if kind == "table"]
+    others = {(kind, name) for kind, name, _sql in objects if kind != "table"}
+    expected = {("index", _AUTOINDEX), ("index", INDEX)}
+    # Before its install, a table without the category index is repaired by
+    # CREATE INDEX IF NOT EXISTS rather than refused.
+    if allow_missing_index and others == {("index", _AUTOINDEX)}:
+        expected = others
+    if len(tables) != 1 or others != expected:
+        return "Image categories name is reserved by an unsupported schema object"
+    if _compact(tables[0]) != _compact(_DDL):
+        return "Image categories has an invalid table definition"
+    index_sql = next((sql for kind, name, sql in objects if name == INDEX), None)
+    if ("index", INDEX) in expected and _compact(index_sql) != _compact(_INDEX_DDL):
+        return "Image categories has an invalid index"
+    autoindex_sql = next(sql for kind, name, sql in objects if name == _AUTOINDEX)
+    if autoindex_sql is not None:
+        return "Image categories has an invalid index"
+    columns = tuple(
+        (row[1], row[2].upper(), row[3], row[5])
+        for row in conn.execute(f"PRAGMA table_info({TABLE})")
+    )
+    if columns != _COLUMNS:
+        return "Image categories has invalid columns"
+    return None
+
+
+def _values_error(conn: sqlite3.Connection) -> str | None:
+    # Bytes bound characters from above (UTF-8 is at most 4 bytes each).
+    limit = MAX_KEY_LENGTH * 4
+    if conn.execute(
+        f"SELECT 1 FROM {TABLE} WHERE typeof(root_key) != 'text' "  # nosec B608
+        "OR typeof(relative_path) != 'text' OR typeof(category) != 'text' "
+        "OR typeof(updated_at) != 'text' "
+        f"OR length(CAST(root_key AS BLOB)) > {limit} "
+        f"OR length(CAST(relative_path AS BLOB)) > {limit} "
+        "OR length(CAST(updated_at AS BLOB)) > 64 "
+        "OR instr(CAST(category AS BLOB), x'00') > 0 "
+        f"OR length(category) < 1 OR length(category) > {MAX_CATEGORY_LENGTH} "
+        "OR category != TRIM(category) OR category GLOB ? LIMIT 1",
+        (_CONTROL_GLOB,),
+    ).fetchone():
+        return "Image categories contains invalid values"
+    for root, path in conn.execute(
+        f"SELECT CAST(root_key AS BLOB), CAST(relative_path AS BLOB) "  # nosec B608
+        f"FROM {TABLE} WHERE length(CAST(root_key AS BLOB)) > {MAX_KEY_LENGTH} "
+        f"OR length(CAST(relative_path AS BLOB)) > {MAX_KEY_LENGTH}"
+    ):
+        for raw in (root, path):
+            if len(bytes(raw).decode("utf-8", "replace")) > MAX_KEY_LENGTH:
+                return "Image categories contains invalid values"
+    return None
+
+
+def image_categories_schema_error(
+    conn: sqlite3.Connection,
+    *,
+    allow_missing: bool = True,
+    check_values: bool = True,
+) -> str | None:
+    """Reject an unsupported image categories schema (and, optionally, values).
+
+    Reads only. A missing table is acceptable for backups that predate it
+    unless ``allow_missing`` is false. ``check_values`` also refuses a stored
+    value the write path could never have written.
+    """
+    error = _structure_error(conn, allow_missing)
+    if error is not None or not check_values:
+        return error
+    exists = conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
+    ).fetchone()
+    return _values_error(conn) if exists else None
+
+
+def migrate_image_categories(conn: sqlite3.Connection) -> None:
+    """Install the table and its index atomically; structure is checked
+    before and after. Reads only ``sqlite_master`` and this table's PRAGMAs
+    and writes only its own two objects (no backfill)."""
+    conn.execute("SAVEPOINT image_categories_schema")
+    try:
+        error = _structure_error(conn, True, allow_missing_index=True)
+        if error:
+            raise sqlite3.DatabaseError(error)
+        conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
+        conn.execute(
+            _INDEX_DDL.replace("CREATE INDEX", "CREATE INDEX IF NOT EXISTS", 1)
+        )
+        error = image_categories_schema_error(
+            conn, allow_missing=False, check_values=False
+        )
+        if error:
+            raise sqlite3.DatabaseError(error)
+        conn.execute("RELEASE image_categories_schema")
+    except BaseException:
+        conn.execute("ROLLBACK TO image_categories_schema")
+        conn.execute("RELEASE image_categories_schema")
+        raise
+
+
+def copy_image_category(conn: sqlite3.Connection, removed_key, kept_key) -> int:
+    """Give the kept copy the removed copy's category when it has none.
+
+    Runs inside the caller's transaction; never deletes the removed copy's
+    row, so a give-back or a failed delete loses nothing. Returns 1 when a
+    row was inserted, else 0.
+    """
+    if tuple(removed_key) == tuple(kept_key):
+        return 0
+    return max(conn.execute(COPY_SQL, (*kept_key, *removed_key)).rowcount, 0)
+
+
+class ImageCategoryOperations:
+    """PromptDatabase mixin: set, clear and read images' own categories.
+
+    Never reads or writes ``prompts``, ``prompt_pairs`` or
+    ``generated_images``.
+    """
+
+    def image_category(self, root_key: str, relative_key: str) -> str | None:
+        """One file's own category, or None."""
+        row = (
+            self.model.get_connection()
+            .execute(CATEGORY_OF_SQL, (root_key, relative_key))
+            .fetchone()
+        )
+        return row[0] if row else None
+
+    def own_categories_of(self, root_keys: Iterable[str]) -> dict:
+        """``{(root key, relative key): category}`` under the given sources.
+
+        One primary-key range read per source.
+        """
+        found = {}
+        conn = self.model.get_connection()
+        for root_key in dict.fromkeys(root_keys):
+            for relative, category in conn.execute(CATEGORIES_OF_ROOT_SQL, (root_key,)):
+                found[(root_key, relative)] = category
+        return found
+
+    def set_image_categories(self, file_keys, category: str) -> dict:
+        """Set *category* on every file key, in one transaction.
+
+        ``file_keys``: ``(root key, relative key)`` pairs from any number of
+        sources. Returns ``{updated, unchanged}``; a file that already had
+        exactly this value is ``unchanged`` and keeps its ``updated_at``.
+        """
+        from .image_removal_schema import clear_marks_of_keys
+        from .models import utc_now_iso  # models imports this module's schema
+
+        keys = list(dict.fromkeys(tuple(key) for key in file_keys))
+        counts = {"updated": 0, "unchanged": 0}
+        now = utc_now_iso()
+        with self._pair_transaction() as conn:
+            for key in keys:
+                row = conn.execute(CATEGORY_OF_SQL, key).fetchone()
+                if row is not None and row[0] == category:
+                    counts["unchanged"] += 1
+                    continue
+                conn.execute(UPSERT_SQL, (*key, category, now))
+                counts["updated"] += 1
+            clear_marks_of_keys(conn, keys)  # #480: a categorized file is listed
+        return counts
+
+    def clear_image_categories(self, file_keys) -> int:
+        """Remove the own category of every file key (one transaction).
+
+        Returns the number of rows removed.
+        """
+        keys = list(dict.fromkeys(tuple(key) for key in file_keys))
+        removed = 0
+        with self._pair_transaction() as conn:
+            for key in keys:
+                removed += conn.execute(DELETE_SQL, key).rowcount
+        return removed

database/image_collections.py

@@ -0,0 +1,492 @@
+"""Collections (#331): named, hand-picked groups of image files.
+
+A member names its file exactly as a favorite does (``image_favorites.py``):
+the source folder's real, case-normalized path (``root_key``) and the path
+below it (``relative_path``), both server-side only. A member whose file is
+moved, renamed or deleted outside PromptManager is kept and simply not listed;
+nothing follows a file to a new place.
+
+Names are trimmed with each run of whitespace collapsed to one space, 1 to 100
+characters without control characters; ``name_key`` (NFC of the case-folded
+NFC name) is UNIQUE, so names are unique ignoring case and Unicode form.
+
+The two tables install like ``image_favorites`` and ``saved_views``: optional
+in old backups, required once a database is current, created empty with no
+backfill. Startup checks only the structure, so a value saved under older
+rules never stops the library from opening; restore checks the values too,
+but never recomputes ``name_key`` (case tables differ between Python
+versions).
+"""
+
+import re
+import sqlite3
+import unicodedata
+from typing import Optional
+
+from .image_favorites import MAX_KEY_LENGTH
+
+COLLECTIONS_TABLE = "image_collections"
+ITEMS_TABLE = "image_collection_items"
+FILE_INDEX = "idx_image_collection_items_file"
+TABLES = (COLLECTIONS_TABLE, ITEMS_TABLE)
+MAX_NAME_LENGTH = 100
+_TIMESTAMP_BYTES = 64
+# A name key's byte bound: case folding expands a character to at most
+# three, each at most 4 UTF-8 bytes.
+_MAX_KEY_BYTES = MAX_NAME_LENGTH * 3 * 4
+_WHITESPACE = re.compile(r"\s+")
+
+_TABLE_DDL = {
+    COLLECTIONS_TABLE: (
+        f"CREATE TABLE {COLLECTIONS_TABLE} (id INTEGER PRIMARY KEY AUTOINCREMENT, "
+        "name TEXT NOT NULL, name_key TEXT NOT NULL UNIQUE, "
+        "created_at TEXT NOT NULL)"
+    ),
+    ITEMS_TABLE: (
+        f"CREATE TABLE {ITEMS_TABLE} (collection_id INTEGER NOT NULL REFERENCES "
+        f"{COLLECTIONS_TABLE}(id) ON DELETE CASCADE, root_key TEXT NOT NULL, "
+        "relative_path TEXT NOT NULL, added_at TEXT NOT NULL, "
+        "PRIMARY KEY (collection_id, root_key, relative_path))"
+    ),
+}
+_INDEX_DDL = f"CREATE INDEX {FILE_INDEX} ON {ITEMS_TABLE}(root_key, relative_path)"
+# (name, declared type, not null, default, primary key position)
+_COLUMNS = {
+    COLLECTIONS_TABLE: (
+        ("id", "INTEGER", 0, None, 1),
+        ("name", "TEXT", 1, None, 0),
+        ("name_key", "TEXT", 1, None, 0),
+        ("created_at", "TEXT", 1, None, 0),
+    ),
+    ITEMS_TABLE: (
+        ("collection_id", "INTEGER", 1, None, 1),
+        ("root_key", "TEXT", 1, None, 2),
+        ("relative_path", "TEXT", 1, None, 3),
+        ("added_at", "TEXT", 1, None, 0),
+    ),
+}
+# (column, parent table, parent column, on delete)
+_FOREIGN_KEYS = {
+    COLLECTIONS_TABLE: set(),
+    ITEMS_TABLE: {("collection_id", COLLECTIONS_TABLE, "id", "CASCADE")},
+}
+
+
+class CollectionInvalid(ValueError):
+    """A name, id or target breaks the write rules (a 400)."""
+
+
+def _has_control(text: str) -> bool:
+    """Any Unicode control character (category Cc: C0, DEL and C1)."""
+    return any(unicodedata.category(char) == "Cc" for char in text)
+
+
+def normalize_collection_name(name: object) -> str:
+    """The stored spelling of *name*: trimmed, whitespace collapsed, checked.
+
+    Raises ``CollectionInvalid`` unless the result is 1 to 100 characters
+    without control characters.
+    """
+    if not isinstance(name, str):
+        raise CollectionInvalid("Name must be a string")
+    collapsed = _WHITESPACE.sub(" ", name).strip()
+    if not 1 <= len(collapsed) <= MAX_NAME_LENGTH:
+        raise CollectionInvalid(f"Name must be 1 to {MAX_NAME_LENGTH} characters")
+    if _has_control(collapsed):
+        raise CollectionInvalid("Name contains control characters")
+    return collapsed
+
+
+def collection_name_key(name: str) -> str:
+    """The uniqueness key of a name: NFC of the case-folded NFC spelling."""
+    return unicodedata.normalize("NFC", unicodedata.normalize("NFC", name).casefold())
+
+
+def _compact(sql: str) -> str:
+    return re.sub(r"\s+", "", sql).lower()
+
+
+def _expected_catalog() -> set:
+    objects = {("table", name, _compact(sql)) for name, sql in _TABLE_DDL.items()}
+    objects.add(("index", FILE_INDEX, _compact(_INDEX_DDL)))
+    objects |= {("index", f"sqlite_autoindex_{table}_1", "") for table in TABLES}
+    return objects
+
+
+def _catalog(conn: sqlite3.Connection) -> list:
+    # sqlite_sequence (AUTOINCREMENT) is shared with other tables and ignored.
+    return conn.execute(
+        "SELECT type,name,sql FROM sqlite_master "
+        "WHERE name IN (?,?,?) OR tbl_name IN (?,?)",
+        (*TABLES, FILE_INDEX, *TABLES),
+    ).fetchall()
+
+
+def _structure_error(conn: sqlite3.Connection) -> Optional[str]:
+    for table, expected in _COLUMNS.items():
+        columns = tuple(
+            (row[1], row[2].upper(), row[3], row[4], row[5])
+            for row in conn.execute(f"PRAGMA table_info({table})")
+        )
+        if columns != expected:
+            return f"Image collections table {table} has invalid columns"
+        foreign_keys = {
+            (row[3], row[2], row[4], row[6])
+            for row in conn.execute(f"PRAGMA foreign_key_list({table})")
+        }
+        if foreign_keys != _FOREIGN_KEYS[table]:
+            return f"Image collections table {table} has invalid foreign keys"
+    return None
+
+
+def _decoded_too_long(conn, sql: str, limit: int) -> bool:
+    """Whether a byte-bounded candidate row decodes to over *limit* characters."""
+    for row in conn.execute(sql):
+        for raw in row:
+            if len(bytes(raw).decode("utf-8", "replace")) > limit:
+                return True
+    return False
+
+
+def _value_error(conn: sqlite3.Connection) -> Optional[str]:
+    # Only this module's constants are interpolated into the SQL below. Bytes
+    # bound characters from above (UTF-8 is at most 4 bytes each); the few
+    # candidates over the character bound are measured decoded in Python.
+    if conn.execute(
+        f"SELECT 1 FROM {COLLECTIONS_TABLE} WHERE "  # nosec B608
+        "typeof(id) != 'integer' OR typeof(name) != 'text' "
+        "OR typeof(name_key) != 'text' OR typeof(created_at) != 'text' "
+        f"OR length(CAST(name AS BLOB)) > {MAX_NAME_LENGTH * 4} "
+        "OR length(CAST(name_key AS BLOB)) = 0 "
+        f"OR length(CAST(name_key AS BLOB)) > {_MAX_KEY_BYTES} "
+        f"OR length(CAST(created_at AS BLOB)) > {_TIMESTAMP_BYTES} LIMIT 1"
+    ).fetchone():
+        return "Image collections contains invalid values"
+    # The key is not recomputed: Unicode case tables differ between Python
+    # versions, and a key written by another version must stay restorable.
+    # Its UNIQUE constraint keeps names distinct; writes compute it.
+    for (name,) in conn.execute(f"SELECT name FROM {COLLECTIONS_TABLE}"):  # nosec B608
+        if (
+            not 1 <= len(name) <= MAX_NAME_LENGTH
+            or _has_control(name)
+            or name != name.strip()
+        ):
+            return "Image collections contains invalid names"
+    limit = MAX_KEY_LENGTH * 4
+    if conn.execute(
+        f"SELECT 1 FROM {ITEMS_TABLE} WHERE "  # nosec B608
+        "typeof(collection_id) != 'integer' OR typeof(root_key) != 'text' "
+        "OR typeof(relative_path) != 'text' OR typeof(added_at) != 'text' "
+        f"OR length(CAST(root_key AS BLOB)) > {limit} "
+        f"OR length(CAST(relative_path AS BLOB)) > {limit} "
+        f"OR length(CAST(added_at AS BLOB)) > {_TIMESTAMP_BYTES} LIMIT 1"
+    ).fetchone() or _decoded_too_long(
+        conn,
+        f"SELECT CAST(root_key AS BLOB), CAST(relative_path AS BLOB) "  # nosec B608
+        f"FROM {ITEMS_TABLE} WHERE length(CAST(root_key AS BLOB)) > {MAX_KEY_LENGTH} "
+        f"OR length(CAST(relative_path AS BLOB)) > {MAX_KEY_LENGTH}",
+        MAX_KEY_LENGTH,
+    ):
+        return "Image collections contains invalid members"
+    if conn.execute(f"PRAGMA foreign_key_check({ITEMS_TABLE})").fetchone():
+        return "Image collections contains members of a missing collection"
+    return None
+
+
+def image_collections_schema_error(
+    conn: sqlite3.Connection,
+    *,
+    allow_missing: bool = True,
+    check_values: bool = True,
+) -> Optional[str]:
+    """Reject partial or unsupported collections schemas and invalid rows.
+
+    Reads only: nothing is ever repaired. A database without either table is
+    acceptable for backups that predate them unless ``allow_missing`` is
+    false. ``check_values=False`` (live startup) checks the structure only.
+    """
+    objects = _catalog(conn)
+    if not objects:
+        return None if allow_missing else "Missing image collections schema"
+    present = {name for kind, name, _sql in objects if kind == "table"}
+    if present != set(TABLES):
+        return "Incomplete image collections schema"
+    actual = {(kind, name, _compact(sql or "")) for kind, name, sql in objects}
+    if actual != _expected_catalog():
+        return "Image collections has an invalid schema definition"
+    error = _structure_error(conn)
+    if error or not check_values:
+        return error
+    return _value_error(conn)
+
+
+def migrate_image_collections(conn: sqlite3.Connection) -> None:
+    """Install both tables atomically; validate before and after creating them."""
+    conn.execute("SAVEPOINT image_collections_schema")
+    try:
+        error = image_collections_schema_error(conn, check_values=False)
+        if error:
+            raise sqlite3.DatabaseError(error)
+        for sql in _TABLE_DDL.values():
+            conn.execute(sql.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
+        conn.execute(_INDEX_DDL.replace("INDEX", "INDEX IF NOT EXISTS", 1))
+        error = image_collections_schema_error(
+            conn, allow_missing=False, check_values=False
+        )
+        if error:
+            raise sqlite3.DatabaseError(error)
+        conn.execute("RELEASE image_collections_schema")
+    except BaseException:
+        conn.execute("ROLLBACK TO image_collections_schema")
+        conn.execute("RELEASE image_collections_schema")
+        raise
+
+
+class CollectionMissing(LookupError):
+    """No collection has the id (a 404 ``collection-not-found``)."""
+
+
+class CollectionNameTaken(ValueError):
+    """Another collection already has the name's key (a 409 ``name-taken``)."""
+
+    def __init__(self, collection_id: int):
+        super().__init__("Another collection has this name")
+        self.collection_id = collection_id
+
+
+class CollectionFilesMissing(LookupError):
+    """A new name with nothing to insert (a 404 ``files-not-found``)."""
+
+
+LIST_SQL = f"SELECT id, name FROM {COLLECTIONS_TABLE} ORDER BY name_key, id"
+GET_SQL = f"SELECT id, name FROM {COLLECTIONS_TABLE} WHERE id = ?"
+BY_KEY_SQL = f"SELECT id, name FROM {COLLECTIONS_TABLE} WHERE name_key = ?"
+MEMBERS_SQL = (
+    f"SELECT relative_path FROM {ITEMS_TABLE} "
+    "WHERE collection_id = ? AND root_key = ?"
+)
+ROOT_MEMBERS_SQL = (
+    f"SELECT collection_id, relative_path FROM {ITEMS_TABLE} WHERE root_key = ?"
+)
+COUNTS_SQL = (
+    f"SELECT c.id, (SELECT COUNT(*) FROM {ITEMS_TABLE} i "
+    f"WHERE i.collection_id = c.id) FROM {COLLECTIONS_TABLE} c"
+)
+FILE_SQL = (
+    f"SELECT collection_id FROM {ITEMS_TABLE} "
+    "WHERE root_key = ? AND relative_path = ? ORDER BY collection_id"
+)
+# Every read, for the EXPLAIN QUERY PLAN check: none scans the items table and
+# only the two whole-list reads (LIST_SQL, COUNTS_SQL) scan the collections.
+READ_SQL = (
+    LIST_SQL,
+    GET_SQL,
+    BY_KEY_SQL,
+    MEMBERS_SQL,
+    ROOT_MEMBERS_SQL,
+    COUNTS_SQL,
+    FILE_SQL,
+)
+_INSERT_ITEM_SQL = f"INSERT OR IGNORE INTO {ITEMS_TABLE} VALUES (?, ?, ?, ?)"
+_DELETE_ITEM_SQL = (
+    f"DELETE FROM {ITEMS_TABLE} "
+    "WHERE collection_id = ? AND root_key = ? AND relative_path = ?"
+)
+
+
+def _collection_id(value: object) -> int:
+    if type(value) is not int or not 1 <= value <= 999_999_999:
+        raise CollectionInvalid("Collection id must be a positive integer")
+    return value
+
+
+def _items(items: object) -> list:
+    """Distinct ``(root key, relative key)`` pairs, in order."""
+    if not isinstance(items, (list, tuple)):
+        raise CollectionInvalid("Items must be a list")
+    found = []
+    for item in items:
+        if (
+            not isinstance(item, (list, tuple))
+            or len(item) != 2
+            or not all(
+                isinstance(part, str) and 1 <= len(part) <= MAX_KEY_LENGTH
+                for part in item
+            )
+        ):
+            raise CollectionInvalid("Each item is a root key and a relative key")
+        found.append((item[0], item[1]))
+    return list(dict.fromkeys(found))
+
+
+def _row(row) -> dict:
+    return {"id": row[0], "name": row[1]}
+
+
+class ImageCollectionOperations:
+    """PromptDatabase mixin: collections and their members.
+
+    Each write runs in its own ``BEGIN IMMEDIATE`` transaction
+    (``_pair_transaction``), so two tabs creating one name are serialized.
+    """
+
+    def list_collections(self) -> list[dict]:
+        """Every collection ``{id, name}`` in name-key order, then id."""
+        rows = self.model.get_connection().execute(LIST_SQL)
+        return [_row(row) for row in rows]
+
+    def get_collection(self, collection_id: int) -> Optional[dict]:
+        """``{id, name}`` of one collection, or None."""
+        row = (
+            self.model.get_connection()
+            .execute(GET_SQL, (_collection_id(collection_id),))
+            .fetchone()
+        )
+        return _row(row) if row else None
+
+    def collection_exists(self, collection_id: int) -> bool:
+        return self.get_collection(collection_id) is not None
+
+    def collection_members(self, collection_id: int, root_key: str) -> set[str]:
+        """The relative keys stored for one collection under one source."""
+        rows = self.model.get_connection().execute(
+            MEMBERS_SQL, (_collection_id(collection_id), root_key)
+        )
+        return {path for (path,) in rows}
+
+    def root_collection_members(self, root_key: str) -> dict[int, set[str]]:
+        """``{collection id: relative keys}`` stored under one source."""
+        found: dict[int, set[str]] = {}
+        for collection_id, path in self.model.get_connection().execute(
+            ROOT_MEMBERS_SQL, (root_key,)
+        ):
+            found.setdefault(collection_id, set()).add(path)
+        return found
+
+    def collection_count_snapshot(self, root_keys: list) -> tuple[dict, dict]:
+        """``({root key: {id: relative keys}}, {id: stored rows})`` read in
+        one read transaction, so a write between the reads cannot skew them.
+        """
+        conn = self.model.get_connection()
+        if conn.in_transaction:
+            raise RuntimeError("Collection counts require their own transaction")
+        conn.execute("BEGIN")
+        try:
+            members = {}
+            for root_key in dict.fromkeys(root_keys):
+                found: dict[int, set[str]] = {}
+                for collection_id, path in conn.execute(ROOT_MEMBERS_SQL, (root_key,)):
+                    found.setdefault(collection_id, set()).add(path)
+                members[root_key] = found
+            stored = {cid: count for cid, count in conn.execute(COUNTS_SQL)}
+        finally:
+            conn.rollback()  # it only read
+        return members, stored
+
+    def collection_counts(self) -> dict[int, int]:
+        """``{collection id: stored rows}`` for every collection."""
+        rows = self.model.get_connection().execute(COUNTS_SQL)
+        return {collection_id: count for collection_id, count in rows}
+
+    def file_collections(self, root_key: str, relative_path: str) -> list[int]:
+        """Ids of the collections holding one file."""
+        rows = self.model.get_connection().execute(FILE_SQL, (root_key, relative_path))
+        return [collection_id for (collection_id,) in rows]
+
+    def add_to_collection(
+        self, *, items: object, collection_id: object = None, name: object = None
+    ) -> dict:
+        """Add files to an existing id or a name; create the name when needed.
+
+        A name whose key exists uses that collection; otherwise a collection
+        is created only when at least one item is inserted. Returns
+        ``{collection, created, added, already}``.
+        """
+        from .image_removal_schema import clear_marks_of_keys
+        from .models import utc_now_iso  # models imports this module's schema
+
+        if (collection_id is None) == (name is None):
+            raise CollectionInvalid("Give either a collection id or a name")
+        pairs = _items(items)
+        if name is not None:
+            spelled = normalize_collection_name(name)
+        else:
+            collection_id = _collection_id(collection_id)
+        now = utc_now_iso()
+        with self._pair_transaction() as conn:
+            created = False
+            if name is not None:
+                row = conn.execute(
+                    BY_KEY_SQL, (collection_name_key(spelled),)
+                ).fetchone()
+                if row is None:
+                    if not pairs:
+                        raise CollectionFilesMissing("Nothing to add")
+                    cursor = conn.execute(
+                        f"INSERT INTO {COLLECTIONS_TABLE} "  # nosec B608
+                        "(name, name_key, created_at) VALUES (?, ?, ?)",
+                        (spelled, collection_name_key(spelled), now),
+                    )
+                    row, created = (cursor.lastrowid, spelled), True
+            else:
+                row = conn.execute(GET_SQL, (collection_id,)).fetchone()
+                if row is None:
+                    raise CollectionMissing("No such collection")
+            collection = _row(row)
+            added = sum(
+                conn.execute(
+                    _INSERT_ITEM_SQL, (collection["id"], root, path, now)
+                ).rowcount
+                for root, path in pairs
+            )
+            clear_marks_of_keys(conn, pairs)  # #480: a collected file is listed
+        return {
+            "collection": collection,
+            "created": created,
+            "added": added,
+            "already": len(pairs) - added,
+        }
+
+    def remove_from_collection(self, collection_id: object, items: object) -> dict:
+        """Remove files from one collection; ``{collection, removed}``."""
+        collection_id = _collection_id(collection_id)
+        pairs = _items(items)
+        with self._pair_transaction() as conn:
+            row = conn.execute(GET_SQL, (collection_id,)).fetchone()
+            if row is None:
+                raise CollectionMissing("No such collection")
+            removed = sum(
+                conn.execute(_DELETE_ITEM_SQL, (collection_id, root, path)).rowcount
+                for root, path in pairs
+            )
+        return {"collection": _row(row), "removed": removed}
+
+    def rename_collection(self, collection_id: object, name: object) -> dict:
+        """The renamed ``{id, name}``; a case variant of its own name is fine."""
+        collection_id = _collection_id(collection_id)
+        spelled = normalize_collection_name(name)
+        key = collection_name_key(spelled)
+        with self._pair_transaction() as conn:
+            if conn.execute(GET_SQL, (collection_id,)).fetchone() is None:
+                raise CollectionMissing("No such collection")
+            taken = conn.execute(BY_KEY_SQL, (key,)).fetchone()
+            if taken is not None and taken[0] != collection_id:
+                raise CollectionNameTaken(taken[0])
+            conn.execute(
+                f"UPDATE {COLLECTIONS_TABLE} SET name = ?, name_key = ? "  # nosec B608
+                "WHERE id = ?",
+                (spelled, key, collection_id),
+            )
+        return {"id": collection_id, "name": spelled}
+
+    def delete_collection(self, collection_id: object) -> bool:
+        """Delete one collection and its members (never a file). True."""
+        collection_id = _collection_id(collection_id)
+        with self._pair_transaction() as conn:
+            cursor = conn.execute(
+                f"DELETE FROM {COLLECTIONS_TABLE} WHERE id = ?",  # nosec B608
+                (collection_id,),
+            )
+            if cursor.rowcount == 0:
+                raise CollectionMissing("No such collection")
+        return True

database/image_favorites.py

@@ -104,11 +104,14 @@ class ImageFavoriteOperations:
         """Mark one file as a favorite; repeating it is a no-op. Returns True."""
         from .models import utc_now_iso  # models imports this module's schema
 
+        from .image_removal_schema import clear_marks_of_keys
+
         with self.model.get_connection() as conn:
             conn.execute(
                 "INSERT OR IGNORE INTO image_favorites VALUES (?, ?, ?)",
                 (root_key, relative_path, utc_now_iso()),
             )
+            clear_marks_of_keys(conn, [(root_key, relative_path)])  # #480
         return True
 
     def clear_image_favorite(self, root_key: str, relative_path: str) -> bool:

database/image_removal.py

@@ -0,0 +1,348 @@
+"""Remove a file's records from the library, and its removal mark (#480).
+
+A *removal* describes one physical file the way the API resolved it from the
+walked listing (``py/api/image_delete.py``):
+
+- ``spellings``: every path string records may name it by
+  (``duplicate_spellings.Spellings.of``);
+- ``locator``: its private canonical locator;
+- ``favorites``: ``(root key, relative key)`` under every listed root holding
+  it, the keys of its favorite and its collection memberships;
+- ``category``: its own category's file key, or None.
+
+``remove_image_records`` deletes, in one ``BEGIN IMMEDIATE`` transaction:
+
+- its image records (``generated_images``) under every spelling, plus legacy
+  rows matched by ``image_path``; their saved-pair links go by cascade and the
+  workflow documents they held are released;
+- the image tags and auto-tag marks of its byte records. The annotation file
+  and byte rows go too, unless a Needs review item still cites them
+  (``media_review_items.byte_id`` is ``ON DELETE RESTRICT``): a decided or
+  revised item keeps its decision;
+- its pending, undecided Needs review items (settled as duplicate removal
+  settles them);
+- its own category, favorite and collection memberships.
+
+With ``mark`` it also writes the file's removal mark (library-only removal):
+the file stays on disk, hidden until its bytes change. Prompts, recorded runs
+and the media file itself are never touched here.
+"""
+
+import os
+
+from .annotation_schema import BYTES_TABLE, FILES_TABLE
+from .annotations import _BUMP_INDEX
+from .autotag_schema import AUTOTAG_TABLE
+from .duplicate_copies import (
+    DELETE_FAVORITE_SQL,
+    DELETE_LINK_SQL,
+    DELETE_MEMBERSHIPS_SQL,
+    _file_link_rows,
+)
+from .image_categories import DELETE_SQL as DELETE_CATEGORY_SQL
+from .image_removal_schema import (
+    META_TABLE,
+    TABLE,
+    bump_generation,
+    clear_marks,
+    has_marks_table,
+    mark_of,
+    write_mark as _write_mark,
+)
+from .library import display_title
+from .models import utc_now_iso
+from .review import _BUMP_REVIEW_INDEX, _PENDING_OF_LOCATOR, _SETTLE_ITEM
+from .review_changes import notify_review_change
+from .workflow_documents import documents_of_rows, release_documents
+
+BYTES_OF_LOCATOR_SQL = (
+    f"SELECT b.id FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b "  # nosec B608
+    "ON b.file_id=f.id WHERE f.locator=?"
+)
+TAG_COUNT_SQL = "SELECT COUNT(*) FROM media_image_tags WHERE byte_id=?"
+AUTOTAG_COUNT_SQL = (
+    f"SELECT COUNT(*) FROM {AUTOTAG_TABLE} WHERE byte_id=?"  # nosec B608
+)
+CLEAR_TAGS_SQL = "DELETE FROM media_image_tags WHERE byte_id=?"
+CLEAR_AUTOTAG_SQL = f"DELETE FROM {AUTOTAG_TABLE} WHERE byte_id=?"  # nosec B608
+BUMP_TAG_REVISION_SQL = (
+    f"UPDATE {BYTES_TABLE} SET tag_revision=tag_revision+1 WHERE id=?"  # nosec B608
+)
+REVIEW_REFS_SQL = (
+    f"SELECT COUNT(*) FROM {FILES_TABLE} f "  # nosec B608
+    f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
+    "JOIN media_review_items i ON i.byte_id=b.id WHERE f.locator=?"
+)
+DELETE_FILE_SQL = f"DELETE FROM {FILES_TABLE} WHERE locator=?"  # nosec B608
+FILE_EXISTS_SQL = f"SELECT 1 FROM {FILES_TABLE} WHERE locator=?"  # nosec B608
+FAVORITE_EXISTS_SQL = (
+    "SELECT COUNT(*) FROM image_favorites WHERE root_key=? AND relative_path=?"
+)
+MEMBERSHIP_COUNT_SQL = (
+    "SELECT COUNT(*) FROM image_collection_items WHERE root_key=? AND relative_path=?"
+)
+CATEGORY_EXISTS_SQL = (
+    "SELECT COUNT(*) FROM image_categories WHERE root_key=? AND relative_path=?"
+)
+PAIR_LINK_COUNT_SQL = "SELECT COUNT(*) FROM image_prompt_pairs WHERE image_id=?"
+PROMPT_LINKS_SQL = "SELECT id FROM generated_images WHERE prompt_id=?"
+PROMPT_TITLE_SQL = (
+    "SELECT p.text, s.title FROM prompts p "
+    "LEFT JOIN prompt_library_state s ON s.prompt_id=p.id WHERE p.id=?"
+)
+MARKS_SQL = f"SELECT locator, size, mtime_ns FROM {TABLE}"  # nosec B608
+REVISION_SQL = f"SELECT revision FROM {META_TABLE} WHERE singleton_id=1"  # nosec B608
+MAX_ORPHAN_TITLES = 3
+RECORD_KINDS = (
+    "prompt_links",
+    "pair_links",
+    "image_tags",
+    "auto_tagged",
+    "categories",
+    "collections",
+    "favorites",
+    "review_items",
+)
+
+
+def _link_ids(conn, removal):
+    """``{image id: prompt id}`` of the records naming the file, any spelling."""
+    found = {}
+    for spelled in dict.fromkeys(removal["spellings"]):
+        for image_id, prompt_id in _file_link_rows(conn, spelled):
+            found[image_id] = prompt_id
+    return found
+
+
+def _byte_ids(conn, locator):
+    return [row[0] for row in conn.execute(BYTES_OF_LOCATOR_SQL, (locator,))]
+
+
+def file_locator(path) -> str:
+    """The locator of an absolute path (``normcase(realpath)``)."""
+    return os.path.normcase(os.path.realpath(str(path)))
+
+
+def removed_from_library(conn, path) -> bool:
+    """Whether *path* is hidden by a removal mark whose bytes still match.
+
+    A mark whose file changed (other size or ``mtime_ns``) or is gone is
+    stale: it is deleted on *conn* (the caller's write transaction), and the
+    file is not hidden.
+    """
+    locator = file_locator(path)
+    mark = mark_of(conn, locator)
+    if mark is None:
+        return False
+    try:
+        info = os.lstat(path)
+    except OSError:
+        clear_marks(conn, [locator])
+        return False
+    if (info.st_size, info.st_mtime_ns) == tuple(mark):
+        return True
+    clear_marks(conn, [locator])
+    return False
+
+
+def mark_hides(marks: dict, locator: str, path) -> bool:
+    """Whether a removal mark in *marks* hides the file at *path* (blocking).
+
+    *marks* is ``{locator: (size, mtime_ns)}``. The mark of *locator* hides
+    the file while ``os.lstat(path)`` still has the size and ``mtime_ns`` it
+    was removed with; a file that cannot be looked at is not hidden.
+    """
+    mark = marks.get(locator)
+    if mark is None:
+        return False
+    try:
+        info = os.lstat(path)
+    except OSError:
+        return False
+    return (info.st_size, info.st_mtime_ns) == tuple(mark)
+
+
+def write_mark(conn, locator, size, mtime_ns):
+    _write_mark(conn, locator, size, mtime_ns, utc_now_iso())
+
+
+class ImageRemovalOperations:
+    """PromptDatabase mixin: remove a file's records; removal marks."""
+
+    def count_removal_records(self, removals) -> dict:
+        """Counts of what removing *removals* deletes, in one read snapshot.
+
+        ``{records: {kind: n}, orphaned_prompts: {count, titles}}``: prompts
+        whose every image record is among these files lose their last image
+        (they stay in the Library).
+        """
+        from .maintenance import _read_snapshot
+
+        def read(conn):
+            totals = dict.fromkeys(RECORD_KINDS, 0)
+            links = {}
+            for removal in removals:
+                found = _link_ids(conn, removal)
+                links.update(found)
+                for image_id in found:
+                    totals["pair_links"] += conn.execute(
+                        PAIR_LINK_COUNT_SQL, (image_id,)
+                    ).fetchone()[0]
+                tagged = autotagged = False
+                for byte_id in _byte_ids(conn, removal["locator"]):
+                    tagged |= bool(
+                        conn.execute(TAG_COUNT_SQL, (byte_id,)).fetchone()[0]
+                    )
+                    autotagged |= bool(
+                        conn.execute(AUTOTAG_COUNT_SQL, (byte_id,)).fetchone()[0]
+                    )
+                totals["image_tags"] += int(tagged)
+                totals["auto_tagged"] += int(autotagged)
+                totals["review_items"] += len(
+                    conn.execute(_PENDING_OF_LOCATOR, (removal["locator"],)).fetchall()
+                )
+                totals["favorites"] += int(
+                    any(
+                        conn.execute(FAVORITE_EXISTS_SQL, key).fetchone()[0]
+                        for key in removal["favorites"]
+                    )
+                )
+                totals["collections"] += sum(
+                    conn.execute(MEMBERSHIP_COUNT_SQL, key).fetchone()[0]
+                    for key in removal["favorites"]
+                )
+                if removal.get("category"):
+                    totals["categories"] += conn.execute(
+                        CATEGORY_EXISTS_SQL, removal["category"]
+                    ).fetchone()[0]
+            totals["prompt_links"] = len(links)
+            return {
+                "records": totals,
+                "orphaned_prompts": self._orphaned_prompts(conn, links),
+            }
+
+        return _read_snapshot(self.model.db_path, read)
+
+    @staticmethod
+    def _orphaned_prompts(conn, links):
+        removed = set(links)
+        orphans = []
+        for prompt_id in dict.fromkeys(links.values()):
+            remaining = [
+                row[0]
+                for row in conn.execute(PROMPT_LINKS_SQL, (prompt_id,))
+                if row[0] not in removed
+            ]
+            if not remaining:
+                orphans.append(prompt_id)
+        titles = []
+        for prompt_id in orphans[:MAX_ORPHAN_TITLES]:
+            row = conn.execute(PROMPT_TITLE_SQL, (prompt_id,)).fetchone()
+            if row is not None:
+                titles.append(display_title((row[1],), row[0]) or "")
+        return {"count": len(orphans), "titles": titles}
+
+    def remove_image_records(self, removal, *, mark=None) -> dict:
+        """Delete every record of one file (one transaction); see the module.
+
+        ``mark``: ``(size, mtime_ns)`` to write the file's removal mark, or
+        None (the file was moved to the trash). Returns the counts deleted.
+        """
+        counts = dict.fromkeys(RECORD_KINDS, 0)
+        locator = removal["locator"]
+        with self._pair_transaction() as conn:
+            links = _link_ids(conn, removal)
+            if links:
+                documents = documents_of_rows(conn, "id", list(links))
+                for image_id in links:
+                    counts["pair_links"] += conn.execute(
+                        PAIR_LINK_COUNT_SQL, (image_id,)
+                    ).fetchone()[0]
+                    conn.execute(DELETE_LINK_SQL, (image_id,))
+                release_documents(conn, documents)
+            counts["prompt_links"] = len(links)
+            counts.update(self._remove_annotations(conn, locator))
+            for key in removal["favorites"]:
+                counts["favorites"] += conn.execute(DELETE_FAVORITE_SQL, key).rowcount
+                counts["collections"] += conn.execute(
+                    DELETE_MEMBERSHIPS_SQL, key
+                ).rowcount
+            if removal.get("category"):
+                counts["categories"] += conn.execute(
+                    DELETE_CATEGORY_SQL, tuple(removal["category"])
+                ).rowcount
+            if mark is not None:
+                write_mark(conn, locator, *mark)
+        if counts["review_items"]:
+            notify_review_change(self.model.db_path)
+        return counts
+
+    def _remove_annotations(self, conn, locator) -> dict:
+        """Tags, auto-tag marks, pending review items and, when free, the rows."""
+        counts = {"image_tags": 0, "auto_tagged": 0, "review_items": 0}
+        bytes_ = _byte_ids(conn, locator)
+        for byte_id in bytes_:
+            tags = conn.execute(CLEAR_TAGS_SQL, (byte_id,)).rowcount
+            autotags = conn.execute(CLEAR_AUTOTAG_SQL, (byte_id,)).rowcount
+            if tags:
+                conn.execute(BUMP_TAG_REVISION_SQL, (byte_id,))
+            counts["image_tags"] |= bool(tags)
+            counts["auto_tagged"] |= bool(autotags)
+        for (item_id,) in conn.execute(_PENDING_OF_LOCATOR, (locator,)).fetchall():
+            counts["review_items"] += conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
+        if counts["review_items"]:
+            conn.execute(_BUMP_REVIEW_INDEX)
+        if conn.execute(FILE_EXISTS_SQL, (locator,)).fetchone():
+            if not conn.execute(REVIEW_REFS_SQL, (locator,)).fetchone()[0]:
+                conn.execute(DELETE_FILE_SQL, (locator,))
+            conn.execute(_BUMP_INDEX)
+        counts["image_tags"] = int(counts["image_tags"])
+        counts["auto_tagged"] = int(counts["auto_tagged"])
+        return counts
+
+    def removal_marks(self) -> dict:
+        """``{locator: (size, mtime_ns)}`` of every removal mark (one read)."""
+        conn = self.model.get_connection()
+        if not has_marks_table(conn):
+            return {}
+        return {row[0]: (row[1], row[2]) for row in conn.execute(MARKS_SQL)}
+
+    def removal_revision(self) -> int:
+        conn = self.model.get_connection()
+        if not has_marks_table(conn):
+            return 0
+        row = conn.execute(REVISION_SQL).fetchone()
+        return row[0] if row else 0
+
+    def clear_removal_marks(self, locators) -> int:
+        """Show these files again (their marks go); the count removed."""
+        locators = list(locators)
+        if not locators:
+            return 0
+        with self._pair_transaction() as conn:
+            removed = clear_marks(conn, locators)
+        # #498: again after the commit, so a reader that cached the marks
+        # between the in-transaction bump and the commit reloads them.
+        bump_generation()
+        return removed
+
+    def clear_hiding_marks(self, files) -> set:
+        """Delete the marks that still hide their file (#498, Return to library).
+
+        *files* are ``(locator, path)`` pairs. In one ``BEGIN IMMEDIATE``
+        transaction each mark is read again and deleted only while the file at
+        *path* still has its size and ``mtime_ns``. Returns the locators whose
+        marks went.
+        """
+        cleared = set()
+        files = list(files)
+        if not files:
+            return cleared
+        with self._pair_transaction() as conn:
+            for locator, path in files:
+                mark = mark_of(conn, locator)
+                if mark is not None and mark_hides({locator: mark}, locator, path):
+                    cleared.add(locator)
+            clear_marks(conn, sorted(cleared))
+        bump_generation()
+        return cleared

database/image_removal_schema.py

@@ -0,0 +1,213 @@
+"""Removal marks (#480): files removed from the library that stay on disk.
+
+``image_removals`` holds one row per hidden file: its private canonical
+locator (the ``media_annotation_files.locator`` form) and the ``size`` and
+``mtime_ns`` it had when it was removed. A file at that locator with other
+bytes is a new file and shows again. ``image_removals_meta`` is one row, the
+marks revision, bumped in the same transaction as every mark change so the
+workspace listing caches can tell marks changed.
+
+The group installs like ``image_favorites``: optional in old backups,
+required once a database is current. Older builds ignore it.
+"""
+
+import os
+import re
+import sqlite3
+
+from .annotation_schema import MAX_LOCATOR_LENGTH
+
+TABLE = "image_removals"
+META_TABLE = "image_removals_meta"
+TABLES = (TABLE, META_TABLE)
+_TABLE_DDL = {
+    TABLE: (
+        f"CREATE TABLE {TABLE} (locator TEXT PRIMARY KEY, "
+        "size INTEGER NOT NULL CHECK(size>=0), mtime_ns INTEGER NOT NULL, "
+        "removed_at TEXT NOT NULL) WITHOUT ROWID"
+    ),
+    META_TABLE: (
+        f"CREATE TABLE {META_TABLE} (singleton_id INTEGER PRIMARY KEY "
+        "CHECK(singleton_id=1), revision INTEGER NOT NULL DEFAULT 0 "
+        "CHECK(revision>=0))"
+    ),
+}
+_SEED_META = (
+    f"INSERT OR IGNORE INTO {META_TABLE} (singleton_id, revision) VALUES (1, 0)"
+)
+_COLUMNS = {
+    TABLE: (
+        ("locator", "TEXT", 1, 1),
+        ("size", "INTEGER", 1, 0),
+        ("mtime_ns", "INTEGER", 1, 0),
+        ("removed_at", "TEXT", 1, 0),
+    ),
+    META_TABLE: (
+        ("singleton_id", "INTEGER", 0, 1),
+        ("revision", "INTEGER", 1, 0),
+    ),
+}
+_AUTOINDEX = f"sqlite_autoindex_{TABLE}_1"
+
+
+def _compact(sql: str) -> str:
+    return re.sub(r"\s+", "", sql or "").lower()
+
+
+def _structure_error(conn: sqlite3.Connection, allow_missing: bool) -> str | None:
+    rows = conn.execute(
+        "SELECT type,name,tbl_name,sql FROM sqlite_master WHERE name IN (?,?) "
+        "OR tbl_name IN (?,?)",
+        TABLES + TABLES,
+    ).fetchall()
+    if not rows:
+        return None if allow_missing else "Missing image removal marks schema"
+    tables = {name: sql for kind, name, _tbl, sql in rows if kind == "table"}
+    others = {(kind, name) for kind, name, _tbl, _sql in rows if kind != "table"}
+    if set(tables) != set(TABLES) or others - {("index", _AUTOINDEX)}:
+        return "Image removal marks name is reserved by an unsupported schema object"
+    for table in TABLES:
+        if _compact(tables[table]) != _compact(_TABLE_DDL[table]):
+            return "Image removal marks have an invalid table definition"
+        columns = tuple(
+            (row[1], row[2].upper(), row[3], row[5])
+            for row in conn.execute(f"PRAGMA table_info({table})")
+        )
+        if columns != _COLUMNS[table]:
+            return "Image removal marks have invalid columns"
+    return None
+
+
+def _values_error(conn: sqlite3.Connection) -> str | None:
+    limit = MAX_LOCATOR_LENGTH * 4
+    if conn.execute(
+        f"SELECT 1 FROM {TABLE} WHERE typeof(locator) != 'text' "  # nosec B608
+        f"OR length(CAST(locator AS BLOB)) > {limit} OR locator = '' "
+        "OR typeof(size) != 'integer' OR typeof(mtime_ns) != 'integer' "
+        "OR typeof(removed_at) != 'text' OR length(CAST(removed_at AS BLOB)) > 64 "
+        "LIMIT 1"
+    ).fetchone():
+        return "Image removal marks contain invalid values"
+    meta = conn.execute(
+        f"SELECT singleton_id, typeof(revision) FROM {META_TABLE}"  # nosec B608
+    ).fetchall()
+    if meta != [(1, "integer")]:
+        return "Image removal marks have an invalid revision"
+    return None
+
+
+def image_removals_schema_error(
+    conn: sqlite3.Connection, *, allow_missing: bool = True
+) -> str | None:
+    """Reject unsupported schemas and invalid persisted values (reads only)."""
+    error = _structure_error(conn, allow_missing)
+    if error:
+        return error
+    exists = conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
+    ).fetchone()
+    return _values_error(conn) if exists else None
+
+
+def migrate_image_removals(conn: sqlite3.Connection) -> None:
+    """Install both tables atomically; validate before and after."""
+    conn.execute("SAVEPOINT image_removals_schema")
+    try:
+        error = image_removals_schema_error(conn)
+        if error:
+            raise sqlite3.DatabaseError(error)
+        for sql in _TABLE_DDL.values():
+            conn.execute(sql.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
+        conn.execute(_SEED_META)
+        error = image_removals_schema_error(conn, allow_missing=False)
+        if error:
+            raise sqlite3.DatabaseError(error)
+        conn.execute("RELEASE image_removals_schema")
+    except BaseException:
+        conn.execute("ROLLBACK TO image_removals_schema")
+        conn.execute("RELEASE image_removals_schema")
+        raise
+
+
+_UNMARK_SQL = f"DELETE FROM {TABLE} WHERE locator=?"  # nosec B608
+_MARK_SQL = (
+    f"INSERT OR REPLACE INTO {TABLE} "  # nosec B608
+    "(locator, size, mtime_ns, removed_at) VALUES (?, ?, ?, ?)"
+)
+_MARK_OF_SQL = f"SELECT size, mtime_ns FROM {TABLE} WHERE locator=?"  # nosec B608
+BUMP_MARKS_SQL = (
+    f"UPDATE {META_TABLE} SET revision=revision+1 WHERE singleton_id=1"  # nosec B608
+)
+
+
+# Bumped whenever this process writes or clears a mark (even in a transaction
+# that later rolls back): readers that cache the marks reload when it moves.
+_generation = 0
+
+
+def generation() -> int:
+    """The marks generation of this process (see ``_generation``)."""
+    return _generation
+
+
+def _bump() -> None:
+    global _generation
+    _generation += 1
+
+
+def bump_generation() -> None:
+    """Move the generation after a commit (a cached read made before it reloads)."""
+    _bump()
+
+
+def has_marks_table(conn: sqlite3.Connection) -> bool:
+    return (
+        conn.execute(
+            "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
+        ).fetchone()
+        is not None
+    )
+
+
+def locator_of_key(root_key: str, relative_key: str) -> str:
+    """The locator of a favorite-style file key (root key + relative key)."""
+    return os.path.normcase(os.path.join(root_key, *relative_key.split("/")))
+
+
+def mark_of(conn: sqlite3.Connection, locator: str):
+    """``(size, mtime_ns)`` of the file's removal mark, or None."""
+    if not has_marks_table(conn):
+        return None
+    row = conn.execute(_MARK_OF_SQL, (locator,)).fetchone()
+    return (row[0], row[1]) if row else None
+
+
+def write_mark(
+    conn: sqlite3.Connection, locator: str, size: int, mtime_ns: int, now: str
+):
+    conn.execute(_MARK_SQL, (locator, int(size), int(mtime_ns), now))
+    conn.execute(BUMP_MARKS_SQL)
+    _bump()
+
+
+def clear_marks(conn: sqlite3.Connection, locators) -> int:
+    """Delete the marks of *locators* on *conn* (no commit); the count removed.
+
+    Every record write for a file calls this (#480): a file with records is
+    in the library, so it is never hidden.
+    """
+    locators = list(dict.fromkeys(locators))
+    if not locators or not has_marks_table(conn):
+        return 0
+    removed = 0
+    for locator in locators:
+        removed += conn.execute(_UNMARK_SQL, (locator,)).rowcount
+    if removed:
+        conn.execute(BUMP_MARKS_SQL)
+        _bump()
+    return removed
+
+
+def clear_marks_of_keys(conn: sqlite3.Connection, file_keys) -> int:
+    """``clear_marks`` for favorite-style ``(root key, relative key)`` pairs."""
+    return clear_marks(conn, [locator_of_key(*key) for key in file_keys])

database/library.py

@@ -5,7 +5,9 @@ import sqlite3
 from collections.abc import Iterator
 from contextlib import contextmanager
 
+from .library_folders import folder_condition, list_library_folders
 from .library_tokens import library_edit_token
+from .prompt_archive import ARCHIVED, SHELVED
 
 _ELIGIBLE = """(EXISTS(SELECT 1 FROM prompt_pairs ep WHERE ep.positive_prompt_id=p.id)
  OR EXISTS(SELECT 1 FROM generated_images eg WHERE eg.prompt_id=p.id)
@@ -105,6 +107,12 @@ ARCHIVE_SCOPES = {
 }
 
 
+# Shelves (#443): the parents on the shelf, or the archived ones.
+PARENT_SCOPES = {"active": SHELVED, "archived": ARCHIVED}
+# A positive pair owner that is on the shelf, as a condition on alias pp.
+SHELVED_PAIR = SHELVED.replace("p.id", "pp.positive_prompt_id")
+
+
 def _escape(value: str) -> str:
     return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
 
@@ -121,7 +129,54 @@ def _archive(value: str) -> str:
     return ARCHIVE_SCOPES[value]
 
 
-def _filter(text: str, category: str | None, tags: list[str]) -> tuple[str, list]:
+# How the Tags filter matches (#444): every tag (AND), any of them (OR), or
+# parents without any tag.
+TAG_MODES = ("all", "any", "untagged")
+# A parent that no tag's Browse returns (#444 "Untagged prompts"): no tag of its
+# own and no active pair carrying one. The vocabulary's untagged count uses it too.
+UNTAGGED = (
+    "NOT EXISTS(SELECT 1 FROM prompt_tags ut WHERE ut.prompt_id=p.id) "
+    "AND NOT EXISTS(SELECT 1 FROM prompt_pairs up JOIN prompt_pair_tags upt "
+    "ON upt.pair_id=up.id WHERE up.positive_prompt_id=p.id AND up.archived=0)"
+)
+
+
+def _tag_mode(tag_mode: object, tags: list[str]) -> str:
+    if not isinstance(tag_mode, str) or tag_mode not in TAG_MODES:
+        raise ValueError("tag_mode must be one of " + ", ".join(TAG_MODES))
+    if tag_mode == "untagged" and tags:
+        raise ValueError("Untagged prompts cannot also filter by tags")
+    return tag_mode
+
+
+def _tag_conditions(tags: list[str], any_tag: bool) -> tuple[list, list, list]:
+    """Parent-arm and pair-arm tag conditions and the tags' parameters."""
+    parent_exists = (
+        "EXISTS(SELECT 1 FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id "
+        "WHERE pt.prompt_id=p.id AND t.name{test})"
+    )
+    pair_exists = (
+        "EXISTS(SELECT 1 FROM prompt_pair_tags pt JOIN tags t ON t.id=pt.tag_id "
+        "WHERE pt.pair_id=pp.id AND t.name{test})"
+    )
+    if any_tag and tags:
+        # One condition per arm: the card, or one active pair, carries any tag.
+        test = " IN (" + ",".join("?" for _ in tags) + ")"
+        return [parent_exists.format(test=test)], [pair_exists.format(test=test)], tags
+    return (
+        [parent_exists.format(test="=?") for _ in tags],
+        [pair_exists.format(test="=?") for _ in tags],
+        tags,
+    )
+
+
+def _filter(
+    text: str,
+    category: str | None,
+    tags: list[str],
+    tag_mode: str = "all",
+    folder_prefixes: list[str] | None = None,
+) -> tuple[str, list]:
     if not isinstance(text, str) or len(text) > 500:
         raise ValueError("Search text must be a string of at most 500 characters")
     if category is not None and (not isinstance(category, str) or len(category) > 200):
@@ -132,6 +187,8 @@ def _filter(text: str, category: str | None, tags: list[str]) -> tuple[str, list
         or any(not isinstance(tag, str) or not tag or len(tag) > 200 for tag in tags)
     ):
         raise ValueError("Invalid library tags")
+    mode = _tag_mode(tag_mode, tags)
+    untagged = mode == "untagged"
     parent, pair, parent_params, pair_params = [], ["pp.archived=0"], [], []
     if text:
         parent.append("(p.text LIKE ? ESCAPE '\\' OR ls.title LIKE ? ESCAPE '\\')")
@@ -146,17 +203,11 @@ def _filter(text: str, category: str | None, tags: list[str]) -> tuple[str, list
         pair.append("pp.category=?")
         parent_params.append(category)
         pair_params.append(category)
-    for tag in tags:
-        parent.append(
-            "EXISTS(SELECT 1 FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id "
-            "WHERE pt.prompt_id=p.id AND t.name=?)"
-        )
-        pair.append(
-            "EXISTS(SELECT 1 FROM prompt_pair_tags pt JOIN tags t ON t.id=pt.tag_id "
-            "WHERE pt.pair_id=pp.id AND t.name=?)"
-        )
-        parent_params.append(tag)
-        pair_params.append(tag)
+    parent_tags, pair_tags, tag_params = _tag_conditions(tags, mode == "any")
+    parent.extend(parent_tags)
+    pair.extend(pair_tags)
+    parent_params.extend(tag_params)
+    pair_params.extend(tag_params)
     condition = (
         "("
         + (" AND ".join(parent) or "1=1")
@@ -168,7 +219,17 @@ def _filter(text: str, category: str | None, tags: list[str]) -> tuple[str, list
         + " AND ".join(pair)
         + ")"
     )
-    return f"({_ELIGIBLE}) AND ({condition})", [*parent_params, *pair_params]
+    if untagged:
+        # A whole-card condition: inside the parent-or-pair OR an untagged pair
+        # would let a tagged parent through.
+        condition = f"({condition}) AND {UNTAGGED}"
+    params = [*parent_params, *pair_params]
+    if folder_prefixes:
+        # A whole-card condition around both arms: images link to the parent.
+        folder, folder_params = folder_condition(folder_prefixes)
+        condition = f"({condition}) AND {folder}"
+        params.extend(folder_params)
+    return f"({_ELIGIBLE}) AND ({condition})", params
 
 
 class LibraryOperations:
@@ -216,6 +277,8 @@ class LibraryOperations:
         rows = conn.execute(
             "SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes,p.created_at,"
             "p.run_count,p.last_used_at,"
+            + ARCHIVED
+            + " AS archived,"
             + (f"{sort_expression} AS _sort," if sort_expression else "")
             + "(SELECT COUNT(*) FROM prompt_pairs v WHERE v.positive_prompt_id=p.id "
             "AND v.archived=0) AS variant_count,"
@@ -239,6 +302,7 @@ class LibraryOperations:
             # Usage is shown, never compared: the token hashes owner fields only.
             row["run_count"] = row["run_count"] if type(row["run_count"]) is int else 0
             row["last_used_at"] = utc_iso(row["last_used_at"])
+            row["archived"] = bool(row["archived"])
             row["edit_token"] = library_edit_token(epoch, row)
         return result
 
@@ -251,14 +315,21 @@ class LibraryOperations:
         text: str = "",
         category: str | None = None,
         tags: list[str] | None = None,
+        tag_mode: str = "all",
+        folder_prefixes: list[str] | None = None,
         sort: str = "newest",
         limit: int = 40,
         after: dict | None = None,
+        shelf: str = "active",
     ) -> dict:
         _limit(limit)
         if sort not in _ORDERS:
             raise ValueError("Unknown library sort order")
-        where, params = _filter(text, category, [] if tags is None else tags)
+        if not isinstance(shelf, str) or shelf not in PARENT_SCOPES:
+            raise ValueError("shelf must be active or archived")
+        where, params = _filter(
+            text, category, [] if tags is None else tags, tag_mode, folder_prefixes
+        )
         if after is not None:
             if (
                 not isinstance(after, dict)
@@ -270,15 +341,18 @@ class LibraryOperations:
             ceiling = after["ceiling"]
         where += " AND p.id<=?"
         params.append(ceiling)
-        # One SELECT counts the filtered parents and their active variants.
-        total, variant_total = conn.execute(
-            "SELECT COUNT(*),COALESCE(SUM((SELECT COUNT(*) FROM prompt_pairs v "
-            "WHERE v.positive_prompt_id=p.id AND v.archived=0)),0) FROM "
-            + _FROM
-            + " WHERE "
-            + where,
+        # One SELECT counts the filtered parents of this scope, their active
+        # variants, and the filtered archived parents (Show archived, #443).
+        scope = PARENT_SCOPES[shelf]
+        total, variant_total, archived_total = conn.execute(
+            f"SELECT COALESCE(SUM({scope}),0),"
+            f"COALESCE(SUM(CASE WHEN {scope} THEN (SELECT COUNT(*) "
+            "FROM prompt_pairs v WHERE v.positive_prompt_id=p.id "
+            "AND v.archived=0) ELSE 0 END),0),"
+            f"COALESCE(SUM({ARCHIVED}),0) FROM " + _FROM + " WHERE " + where,
             params,
         ).fetchone()
+        where += f" AND {scope}"
         expression, direction = _ORDERS[sort]
         if after is not None:
             operator = ">" if direction == "ASC" else "<"
@@ -311,6 +385,9 @@ class LibraryOperations:
             "items": rows,
             "total": total,
             "variant_total": variant_total,
+            "archived_total": archived_total,
+            # Echoed so a client can tell this read applied its tag match.
+            "tag_mode": tag_mode,
             "has_more": has_more,
             "next_key": next_key,
         }
@@ -321,9 +398,12 @@ class LibraryOperations:
         text: str = "",
         category: str | None = None,
         tags: list[str] | None = None,
+        tag_mode: str = "all",
+        folder_prefixes: list[str] | None = None,
         sort: str = "newest",
         limit: int = 40,
         after: dict | None = None,
+        shelf: str = "active",
     ) -> dict:
         with self.library_snapshot(include_ceiling=True) as (conn, epoch, ceiling):
             return self._list_library(
@@ -333,9 +413,12 @@ class LibraryOperations:
                 text=text,
                 category=category,
                 tags=tags,
+                tag_mode=tag_mode,
+                folder_prefixes=folder_prefixes,
                 sort=sort,
                 limit=limit,
                 after=after,
+                shelf=shelf,
             )
 
     def _is_library_positive(self, conn: sqlite3.Connection, positive_id: int) -> bool:
@@ -463,9 +546,13 @@ class LibraryOperations:
             "categories": [row["category"] for row in rows],
         }
 
-    def get_library_options(self) -> dict:
+    def get_library_options(self, folder_roots=None) -> dict:
+        """Categories; with *folder_roots* (root spellings) also ``folders``."""
         with self.library_snapshot() as (conn, epoch, _ceiling):
-            return self._get_library_options(conn, epoch)
+            result = self._get_library_options(conn, epoch)
+            if folder_roots is not None:
+                result["folders"] = list_library_folders(conn, folder_roots, _ELIGIBLE)
+            return result
 
     def get_library_previews(self, positive_ids: list[int]) -> dict[int, list[dict]]:
         """Read at most two links per parent in one bounded page query."""
@@ -576,7 +663,9 @@ class LibraryOperations:
             owners = {}
             order = []
             for row in conn.execute(
-                "SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes FROM "
+                "SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes,"
+                + ARCHIVED
+                + " AS archived FROM "
                 + _FROM
                 + " WHERE "
                 + _ELIGIBLE
@@ -589,6 +678,7 @@ class LibraryOperations:
                     "tags": [],
                     "rating": row["rating"],
                     "notes": row["notes"],
+                    "archived": bool(row["archived"]),
                     "variants": [],
                 }
                 order.append(row["id"])
@@ -628,3 +718,64 @@ class LibraryOperations:
                 "kind": "prompt-manager-library",
                 "positives": [owners[key] for key in order],
             }
+
+    def library_counts(self) -> dict:
+        """System › Overview counts from one snapshot, writing nothing.
+
+        ``positive_prompts`` and ``negative_variants`` are the unfiltered
+        Library's ``total`` and ``variant_total``: eligible parents on the
+        shelf (not archived, #443) and their active pairs (every pair's
+        positive is eligible). ``images`` counts indexed
+        ``generated_images`` rows, not files on disk. An image has no rating
+        of its own (it shows its linked prompt's), so ``rated_images`` counts
+        the indexed images whose linked prompt is rated. ``review_pending`` is
+        the Needs review queue's pending total.
+        """
+        with self.library_snapshot() as (conn, _epoch, _ceiling):
+            row = conn.execute(
+                "SELECT (SELECT COUNT(*) FROM prompts p WHERE "
+                + _ELIGIBLE
+                + " AND "
+                + SHELVED
+                + "),"
+                "(SELECT COUNT(*) FROM prompt_pairs pp WHERE pp.archived=0 AND "
+                + SHELVED_PAIR
+                + "),"
+                "(SELECT COUNT(*) FROM generated_images),"
+                "(SELECT COUNT(*) FROM generated_images gi JOIN prompts r "
+                "ON r.id=gi.prompt_id WHERE r.rating IS NOT NULL),"
+                "(SELECT COUNT(*) FROM media_review_items WHERE state='pending')"
+            ).fetchone()
+        return {
+            "positive_prompts": row[0],
+            "negative_variants": row[1],
+            "images": row[2],
+            "rated_images": row[3],
+            "review_pending": row[4],
+        }
+
+    def usage_by_category(self) -> list[dict]:
+        """Runs and average rating of the Library's parents, per category.
+
+        Only eligible parents count, so a negative-only prompt's runs (which
+        repeat its pairs' runs) are never added twice. A blank category groups
+        with no category as ``None``. Unrated parents are left out of the
+        average, which is ``None`` when no parent is rated. Most runs first.
+        """
+        with self.library_snapshot() as (conn, _epoch, _ceiling):
+            rows = conn.execute(
+                "SELECT * FROM (SELECT NULLIF(TRIM(p.category),'') AS category,"
+                "COALESCE(SUM(p.run_count),0) AS runs,AVG(p.rating) AS rating "
+                "FROM prompts p WHERE " + _ELIGIBLE + " GROUP BY 1) "
+                "ORDER BY runs DESC,category IS NULL,category"
+            ).fetchall()
+        return [
+            {
+                "category": row["category"],
+                "runs": row["runs"],
+                "average_rating": (
+                    None if row["rating"] is None else round(row["rating"], 2)
+                ),
+            }
+            for row in rows
+        ]

database/library_archive.py

@@ -0,0 +1,473 @@
+"""Delete selected Library prompts; archive the paired ones (#443, D152).
+
+A selected Library parent is classified, in this order, as ``not_found``
+(missing, or not a Library parent), ``protected`` (tagged ``__protected__``;
+never changed), ``already_archived``, ``archive`` (the positive or negative of
+any saved pair, archived pair or not: archiving hides it from the shelf and
+removes nothing) or ``delete`` (in no pair: deleted exactly as the legacy
+delete does, through ``delete_prompt_rows``). Every command checks the
+storage epoch first and runs in one ``BEGIN IMMEDIATE`` transaction.
+
+The write executes what the user reviewed: it is given the reviewed archive
+and delete lists, and deletes only reviewed deletes the route backed up
+(``candidates``) whose fingerprint is unchanged since that backup.
+"""
+
+import hashlib
+import json
+import re
+import sqlite3
+from typing import Iterable
+
+from .library import _ELIGIBLE
+from .library_commands import _validate_bulk_ids, _validate_epoch
+from .models import utc_now_iso
+from .prompt_archive import ARCHIVED as _ARCHIVED
+from .prompt_archive import TABLE as ARCHIVE
+from .workflow_documents import documents_of_rows, release_documents
+
+try:
+    from ..utils.hashing import generate_prompt_hash
+except ImportError:
+    from utils.hashing import generate_prompt_hash
+
+# Ids bound per statement: the pair check binds each id twice, which stays
+# under SQLite's historical 999-variable limit (maintenance.py ID_BATCH).
+ID_BATCH = 400
+_PROTECTED = (
+    "EXISTS(SELECT 1 FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id "
+    "WHERE pt.prompt_id=p.id AND t.name='__protected__')"
+)
+_PAIRED = (
+    "EXISTS(SELECT 1 FROM prompt_pairs pp "
+    "WHERE pp.positive_prompt_id=p.id OR pp.negative_prompt_id=p.id)"
+)
+# #482: used as the negative of another positive's pair (archived or not):
+# never deleted, so that positive's variant stays (archive only).
+_NEGATIVE_OF_OTHER = (
+    "EXISTS(SELECT 1 FROM prompt_pairs op "
+    "WHERE op.negative_prompt_id=p.id AND op.positive_prompt_id!=p.id)"
+)
+# Classes in the order a prompt is tested; the first that holds wins.
+CLASSES = (
+    "not_found",
+    "protected",
+    "already_archived",
+    "not_archived",
+    "archive",
+    "delete",
+)
+# #492: the shelf a Delete… runs on. On the archived shelf an archived prompt is
+# classified by its pairs, and one restored since is ``not_archived``.
+SHELVES = ("active", "archived")
+
+
+def _check_shelf(shelf: str) -> bool:
+    """Whether ``shelf`` is the archived shelf (ValueError when unknown)."""
+    if shelf not in SHELVES:
+        raise ValueError("shelf must be active or archived")
+    return shelf == "archived"
+
+
+_DIGEST = re.compile(r"^[0-9a-f]{64}$")
+
+
+def _batches(ids: list) -> Iterable[list]:
+    for start in range(0, len(ids), ID_BATCH):
+        yield ids[start : start + ID_BATCH]
+
+
+def delete_prompt_rows(conn: sqlite3.Connection, prompt_ids: list[int]) -> int:
+    """Delete prompts and their image link rows; return the prompts deleted.
+
+    The legacy delete's statements (``delete_prompt``, ``bulk_delete_prompts``)
+    in the caller's transaction: image link rows first (``image_prompt_pairs``
+    cascades, review decisions keep a NULL image), then the prompts (tags and
+    Library state cascade), then the workflow documents no row references.
+    Image files are never touched. The caller checks pairs first.
+    """
+    documents = documents_of_rows(conn, "prompt_id", prompt_ids)
+    deleted = 0
+    for chunk in _batches(list(prompt_ids)):
+        marks = ",".join("?" for _ in chunk)
+        conn.execute(
+            f"DELETE FROM generated_images WHERE prompt_id IN ({marks})",  # nosec
+            chunk,
+        )
+        deleted += conn.execute(
+            f"DELETE FROM prompts WHERE id IN ({marks})", chunk  # nosec B608
+        ).rowcount
+    release_documents(conn, documents)
+    return deleted
+
+
+class LibraryArchiveOperations:
+    """PromptDatabase mixin: classify, archive or delete, and restore parents."""
+
+    def _classify_library_delete(
+        self, conn: sqlite3.Connection, prompt_ids: list[int], shelf: str = "active"
+    ) -> dict:
+        on_archived_shelf = _check_shelf(shelf)
+        found, deletable = {}, set()
+        for chunk in _batches(prompt_ids):
+            marks = ",".join("?" for _ in chunk)
+            for row in conn.execute(
+                f"SELECT p.id,{_PROTECTED},{_ARCHIVED},{_PAIRED},"  # nosec B608
+                f"{_NEGATIVE_OF_OTHER} "
+                f"FROM prompts p WHERE p.id IN ({marks}) AND {_ELIGIBLE}",
+                chunk,
+            ):
+                protected, archived, paired, negative_of_other = row[1:5]
+                if protected:
+                    found[row[0]] = "protected"
+                elif on_archived_shelf and not archived:
+                    found[row[0]] = "not_archived"
+                elif archived and not on_archived_shelf:
+                    found[row[0]] = "already_archived"
+                elif paired:
+                    found[row[0]] = "archive"
+                    if not negative_of_other:
+                        deletable.add(row[0])
+                else:
+                    found[row[0]] = "delete"
+        classes = {name: [] for name in CLASSES}
+        for pid in prompt_ids:
+            classes[found.get(pid, "not_found")].append(pid)
+        # #482: the archive ids Delete permanently may delete with their pairs.
+        classes["deletable"] = [pid for pid in classes["archive"] if pid in deletable]
+        return classes
+
+    @staticmethod
+    def _image_links(conn: sqlite3.Connection, prompt_ids: list[int]) -> int:
+        total = 0
+        for chunk in _batches(prompt_ids):
+            marks = ",".join("?" for _ in chunk)
+            total += conn.execute(
+                "SELECT COUNT(*) FROM generated_images "  # nosec B608
+                f"WHERE prompt_id IN ({marks})",
+                chunk,
+            ).fetchone()[0]
+        return total
+
+    @staticmethod
+    def _fingerprints(conn: sqlite3.Connection, prompt_ids: list[int]) -> dict:
+        """What a safety backup holds of each prompt a delete would remove.
+
+        The prompt row (usage included), its tag ids, its Library title and
+        its image link rows (count and newest id), as a JSON-safe list.
+        """
+        prints = {}
+        for chunk in _batches(prompt_ids):
+            marks = ",".join("?" for _ in chunk)
+            for row in conn.execute(
+                "SELECT p.id,p.text,p.hash,p.category,p.rating,p.notes,"  # nosec
+                "p.updated_at,p.run_count,p.last_used_at,"
+                "(SELECT group_concat(tag_id) FROM (SELECT tag_id FROM prompt_tags "
+                "WHERE prompt_id=p.id ORDER BY tag_id)),"
+                "(SELECT title FROM prompt_library_state WHERE prompt_id=p.id),"
+                "(SELECT COUNT(*) FROM generated_images WHERE prompt_id=p.id),"
+                "(SELECT MAX(id) FROM generated_images WHERE prompt_id=p.id) "
+                f"FROM prompts p WHERE p.id IN ({marks})",
+                chunk,
+            ):
+                prints[row[0]] = [
+                    value if isinstance(value, (int, str, type(None))) else repr(value)
+                    for value in tuple(row)[1:]
+                ]
+        return prints
+
+    @staticmethod
+    def _distinct_images(conn: sqlite3.Connection, prompt_ids: list[int]) -> int:
+        """Image rows of ``prompt_ids`` (each image once; #482 deletes them)."""
+        return LibraryArchiveOperations._image_links(conn, prompt_ids)
+
+    @staticmethod
+    def _pair_prints(conn: sqlite3.Connection, prompt_ids: list[int]) -> dict:
+        """#482: each prompt's own saved pairs, as the review saw them.
+
+        Per pair (by id): its negative, archived flag, metadata, usage, ordered
+        tag ids and its image-pair links (count and newest id).
+        """
+        prints = {pid: [] for pid in prompt_ids}
+        for chunk in _batches(prompt_ids):
+            marks = ",".join("?" for _ in chunk)
+            for row in conn.execute(
+                "SELECT pp.positive_prompt_id,pp.id,pp.negative_prompt_id,"  # nosec
+                "pp.archived,pp.title,pp.category,pp.rating,pp.notes,pp.run_count,"
+                "pp.last_used_at,pp.updated_at,"
+                "(SELECT group_concat(tag_id) FROM (SELECT tag_id FROM "
+                "prompt_pair_tags WHERE pair_id=pp.id ORDER BY tag_id)),"
+                "(SELECT COUNT(*) FROM image_prompt_pairs WHERE pair_id=pp.id),"
+                "(SELECT MAX(id) FROM image_prompt_pairs WHERE pair_id=pp.id) "
+                f"FROM prompt_pairs pp WHERE pp.positive_prompt_id IN ({marks}) "
+                "ORDER BY pp.id",
+                chunk,
+            ):
+                prints[row[0]].append(
+                    [
+                        v if isinstance(v, (int, str, type(None))) else repr(v)
+                        for v in tuple(row)[1:]
+                    ]
+                )
+        return prints
+
+    @classmethod
+    def _review_digests(cls, conn: sqlite3.Connection, prompt_ids: list[int]) -> dict:
+        """#482: one digest per deletable prompt of its row and its pairs."""
+        prompts = cls._fingerprints(conn, prompt_ids)
+        pairs = cls._pair_prints(conn, prompt_ids)
+        return {
+            pid: hashlib.sha256(
+                json.dumps(
+                    [prompts.get(pid), pairs[pid]], separators=(",", ":")
+                ).encode("utf-8")
+            ).hexdigest()
+            for pid in prompt_ids
+        }
+
+    @staticmethod
+    def _paired_counts(
+        conn: sqlite3.Connection, prompt_ids: list[int], selected: list[int]
+    ) -> dict:
+        """#482: what Delete permanently would remove with ``prompt_ids``."""
+        pairs = archived = variants = 0
+        negatives = set()
+        for chunk in _batches(prompt_ids):
+            marks = ",".join("?" for _ in chunk)
+            for archived_flag, negative, positive in conn.execute(
+                "SELECT archived,negative_prompt_id,positive_prompt_id "  # nosec
+                f"FROM prompt_pairs WHERE positive_prompt_id IN ({marks})",
+                chunk,
+            ):
+                pairs += 1
+                archived += 1 if archived_flag else 0
+                if negative is not None:
+                    variants += 1
+                    if negative != positive:
+                        negatives.add(negative)
+        return {
+            "pairs": pairs,
+            "archived_pairs": archived,
+            "variants": variants,
+            "kept_negatives": len(negatives - set(selected)),
+        }
+
+    @staticmethod
+    def _checked_ids(prompt_ids: list[int], expected_epoch: str) -> list[int]:
+        _validate_epoch(expected_epoch)
+        _validate_bulk_ids(prompt_ids)
+        return list(dict.fromkeys(prompt_ids))
+
+    def classify_library_delete(
+        self, prompt_ids: list[int], *, expected_epoch: str, shelf: str = "active"
+    ) -> dict:
+        """What a delete of ``prompt_ids`` would do now (reads only).
+
+        Returns the id lists per class, in request order, ``image_links``
+        (the image link rows the ``delete`` class would remove) and
+        ``fingerprints`` of the ``delete`` class (``_fingerprints``). On the
+        archived ``shelf`` (#492) archived prompts are classified by their pairs.
+        """
+        _check_shelf(shelf)
+        prompt_ids = self._checked_ids(prompt_ids, expected_epoch)
+        with self.library_snapshot() as (conn, _epoch, _ceiling):
+            epoch = self._library_epoch(conn, expected_epoch)
+            classes = self._classify_library_delete(conn, prompt_ids, shelf)
+            deletable = classes["deletable"]
+            return {
+                "library_epoch": epoch,
+                **classes,
+                "image_links": self._image_links(conn, classes["delete"]),
+                "fingerprints": self._fingerprints(conn, classes["delete"]),
+                # #482: Delete permanently, with the deletable prompts' pairs.
+                "review_digests": self._review_digests(conn, deletable),
+                "paired_counts": self._paired_counts(conn, deletable, prompt_ids),
+                "paired_image_links": self._distinct_images(conn, deletable),
+            }
+
+    @staticmethod
+    def _repair_hash(conn: sqlite3.Connection, prompt_id: int) -> None:
+        """Give an archived prompt its canonical hash, so ingestion finds it.
+
+        Scans, the monitor and node saves resolve prompts by hash; a stale or
+        missing one would make them insert a visible copy (Q3). When another
+        row already holds the canonical hash, lookups find that row and no new
+        copy is made, so nothing is written.
+        """
+        text, stored = conn.execute(
+            "SELECT text,hash FROM prompts WHERE id=?", (prompt_id,)
+        ).fetchone()
+        canonical = generate_prompt_hash(text) if isinstance(text, str) else None
+        if canonical is None or stored == canonical:
+            return
+        if conn.execute(
+            "SELECT 1 FROM prompts WHERE hash=? AND id!=?", (canonical, prompt_id)
+        ).fetchone():
+            return
+        conn.execute("UPDATE prompts SET hash=? WHERE id=?", (canonical, prompt_id))
+
+    def archive_or_delete_library_prompts(
+        self,
+        archive_ids: list[int],
+        delete_ids: list[int],
+        *,
+        expected_epoch: str,
+        candidates: dict | None,
+        delete_paired: dict | None = None,
+        shelf: str = "active",
+    ) -> dict:
+        """Archive and delete what the user reviewed, all or nothing.
+
+        ``archive_ids`` and ``delete_ids`` are the reviewed classes.
+        ``candidates`` maps each reviewed delete the route backed up to its
+        fingerprint then (None: every reviewed delete, unchecked; tests and
+        callers without a backup). A reviewed delete is deleted only when it
+        still classifies ``delete`` and its fingerprint is unchanged; one that
+        is now paired, and a reviewed archive now unpaired, are archived (the
+        restorable direction); anything else is kept and counted.
+
+        ``delete_paired`` (#482) maps reviewed archive ids the user chose to
+        Delete permanently to their review digest. Each is deleted with its
+        own saved pairs only while it is still deletable and its digest is
+        unchanged; otherwise it is archived and counted ``changed``.
+
+        On the archived ``shelf`` (#492) the ids are archived already: nothing
+        is archived or rewritten, every reviewed id not deleted (protected or
+        changed) counts ``changed``, and one restored since ``not_archived``.
+        """
+        on_archived_shelf = _check_shelf(shelf)
+        _validate_epoch(expected_epoch)
+        for ids in (archive_ids, delete_ids):
+            if not isinstance(ids, list) or any(
+                type(pid) is not int or not 1 <= pid <= 2**63 - 1 for pid in ids
+            ):
+                raise ValueError("prompt ids must be lists of positive integers")
+        reviewed_delete = set(delete_ids)
+        delete_paired = {} if delete_paired is None else delete_paired
+        if not isinstance(delete_paired, dict) or any(
+            type(pid) is not int
+            or not isinstance(digest, str)
+            or not _DIGEST.match(digest)
+            for pid, digest in delete_paired.items()
+        ):
+            raise ValueError("delete_paired must map prompt ids to review digests")
+        if not set(delete_paired) <= set(archive_ids):
+            raise ValueError("A permanent delete must be a reviewed archive id")
+        if not archive_ids and not delete_ids:
+            raise ValueError("Nothing to delete or archive")
+        if reviewed_delete & set(archive_ids):
+            raise ValueError("A prompt cannot be both archived and deleted")
+        prompt_ids = list(dict.fromkeys([*archive_ids, *delete_ids]))
+        with self._pair_transaction() as conn:
+            epoch = self._library_epoch(conn, expected_epoch)
+            classes = self._classify_library_delete(conn, prompt_ids, shelf)
+            now_prints = self._fingerprints(conn, classes["delete"])
+            deletes, archives, changed = [], [], 0
+            paired_now = set(classes["deletable"])
+            chosen = [pid for pid in classes["archive"] if pid in delete_paired]
+            digests = self._review_digests(
+                conn, [pid for pid in chosen if pid in paired_now]
+            )
+            paired_deletes = []
+            for pid in classes["archive"]:
+                if pid not in delete_paired:
+                    archives.append(pid)
+                elif digests.get(pid) == delete_paired[pid]:
+                    paired_deletes.append(pid)
+                else:
+                    archives.append(pid)  # changed since the review: restorable
+                    changed += 1
+            for pid in classes["delete"]:
+                if pid not in reviewed_delete:
+                    archives.append(pid)  # reviewed as archive, now unpaired
+                    changed += 1 if pid in delete_paired else 0
+                elif candidates is None or candidates.get(pid) == now_prints[pid]:
+                    deletes.append(pid)
+                else:
+                    changed += 1
+            changed += len(classes["protected"])
+            if on_archived_shelf:
+                # Already archived: the rest stay as they are, each one counted.
+                archives = []
+                changed = len(prompt_ids) - len(deletes) - len(paired_deletes)
+                changed -= len(classes["not_found"]) + len(classes["not_archived"])
+            image_links = self._image_links(conn, [*deletes, *paired_deletes])
+            now = utc_now_iso()
+            for pid in archives:
+                self._repair_hash(conn, pid)
+            conn.executemany(
+                f"INSERT INTO {ARCHIVE}(prompt_id, archived_at) VALUES (?, ?)",
+                [(pid, now) for pid in archives],
+            )
+            # A changed row is the newest edit the restore fingerprint reads.
+            conn.executemany(
+                "UPDATE prompts SET updated_at=? WHERE id=?",
+                [(now, pid) for pid in archives],
+            )
+            # #482: the image-pair links (RESTRICT) go first, then the pairs
+            # (pair tags CASCADE, review decisions SET NULL), then the prompts.
+            pairs_removed = 0
+            for chunk in _batches(paired_deletes):
+                marks = ",".join("?" for _ in chunk)
+                pairs_of = (
+                    "SELECT id FROM prompt_pairs "  # nosec B608
+                    f"WHERE positive_prompt_id IN ({marks})"
+                )
+                conn.execute(
+                    f"DELETE FROM image_prompt_pairs WHERE pair_id IN ({pairs_of})",
+                    chunk,
+                )
+                pairs_removed += conn.execute(
+                    "DELETE FROM prompt_pairs "  # nosec B608
+                    f"WHERE positive_prompt_id IN ({marks})",
+                    chunk,
+                ).rowcount
+            deleted = delete_prompt_rows(conn, [*deletes, *paired_deletes])
+            return {
+                "library_epoch": epoch,
+                "deleted": deleted,
+                "archived": len(archives),
+                "already_archived": len(classes["already_archived"]),
+                "not_archived": len(classes["not_archived"]),
+                "not_found": len(classes["not_found"]),
+                "changed": changed,
+                "image_links": image_links,
+                "pairs_removed": pairs_removed,
+            }
+
+    def restore_library_prompts(
+        self, prompt_ids: list[int], *, expected_epoch: str
+    ) -> dict:
+        """Put archived prompts back on the shelf, all or nothing.
+
+        Every requested id with an archive row is restored, whatever its
+        protection or Library parent state: restoring only un-hides it.
+        """
+        prompt_ids = self._checked_ids(prompt_ids, expected_epoch)
+        with self._pair_transaction() as conn:
+            epoch = self._library_epoch(conn, expected_epoch)
+            restored, existing = [], set()
+            for chunk in _batches(prompt_ids):
+                marks = ",".join("?" for _ in chunk)
+                for pid, archived in conn.execute(
+                    f"SELECT p.id,{_ARCHIVED} FROM prompts p "  # nosec B608
+                    f"WHERE p.id IN ({marks})",
+                    chunk,
+                ):
+                    existing.add(pid)
+                    if archived:
+                        restored.append(pid)
+            now = utc_now_iso()
+            conn.executemany(
+                f"DELETE FROM {ARCHIVE} WHERE prompt_id=?", [(pid,) for pid in restored]
+            )
+            conn.executemany(
+                "UPDATE prompts SET updated_at=? WHERE id=?",
+                [(now, pid) for pid in restored],
+            )
+            return {
+                "library_epoch": epoch,
+                "restored": len(restored),
+                "unchanged": len(existing) - len(restored),
+                "not_found": len(prompt_ids) - len(existing),
+            }

database/library_commands.py

@@ -21,6 +21,10 @@ _UUID = re.compile(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}")
 _TOKEN = re.compile(r"[0-9a-f]{64}")
 # Parent metadata stored on the prompt row; title lives in prompt_library_state.
 _PROMPT_COLUMNS = ("category", "rating", "notes")
+# Ids per IN (...) read: well under SQLite's oldest bound-parameter limit (999).
+_BULK_CHUNK = 500
+# The tags one prompt may carry: what the editor's save accepts (``validate_tags``).
+MAX_LIBRARY_TAGS = 20
 
 
 class LibraryCommandError(Exception):
@@ -85,6 +89,13 @@ def _validate_epoch(expected_epoch: str) -> None:
         raise ValueError("expected_epoch must be a canonical UUID")
 
 
+def _validate_bulk_ids(prompt_ids: list[int]) -> None:
+    if not isinstance(prompt_ids, list) or not prompt_ids:
+        raise ValueError("prompt_ids must be a non-empty list")
+    if any(type(pid) is not int or not 1 <= pid <= 2**63 - 1 for pid in prompt_ids):
+        raise ValueError("prompt_ids must be positive integers")
+
+
 def _validate_guards(positive_id: int, expected_epoch: str, if_match: str) -> None:
     if type(positive_id) is not int or not 1 <= positive_id <= 2**63 - 1:
         raise ValueError("Invalid positive prompt ID")
@@ -308,3 +319,134 @@ class LibraryCommandOperations:
                 positive_created=False,
                 pair_created=pair_created,
             )
+
+    def bulk_set_library_category(
+        self, prompt_ids: list[int], category: str, *, expected_epoch: str
+    ) -> dict:
+        """Set one category on many Library parents, all or nothing (#437).
+
+        Guarded by the storage epoch the page read. Only rows whose category
+        differs are written (their ``updated_at`` and edit token change) and
+        registered as Library parents, as the editor's PATCH does; ids
+        that are missing or are not Library parents count as not found.
+        """
+        _validate_epoch(expected_epoch)
+        _validate_bulk_ids(prompt_ids)
+        prompt_ids = list(dict.fromkeys(prompt_ids))
+        if not isinstance(category, str) or not category.strip():
+            raise ValueError("Category is required")
+        validate_category(category)
+        category = category.strip()
+        with self._pair_transaction() as conn:
+            epoch = self._library_epoch(conn, expected_epoch)
+            current = {}
+            for start in range(0, len(prompt_ids), _BULK_CHUNK):
+                chunk = prompt_ids[start : start + _BULK_CHUNK]
+                marks = ",".join("?" for _ in chunk)
+                current.update(
+                    conn.execute(
+                        f"SELECT p.id,p.category FROM prompts p "  # nosec B608
+                        f"WHERE p.id IN ({marks}) AND {_ELIGIBLE}",
+                        chunk,
+                    ).fetchall()
+                )
+            changed = [pid for pid, value in current.items() if value != category]
+            for pid in changed:
+                self._ensure_library_parent(conn, pid)
+            now = utc_now_iso()
+            conn.executemany(
+                "UPDATE prompts SET category=?,updated_at=? WHERE id=?",
+                [(category, now, pid) for pid in changed],
+            )
+            return {
+                "library_epoch": epoch,
+                "category": category,
+                "updated": len(changed),
+                "unchanged": len(current) - len(changed),
+                "not_found": len(prompt_ids) - len(current),
+            }
+
+    def bulk_add_library_tags(
+        self, prompt_ids: list[int], tags: list[str], *, expected_epoch: str
+    ) -> dict:
+        """Add tags to many Library parents, all or nothing (#442).
+
+        Names are normalized once (trim, the vocabulary name rules, casefold
+        repeats folded, at most 20). A parent is given only the names it lacks,
+        compared with ``casefold`` against its own tags; registry rows are
+        found and inserted by their exact name, and only for names some parent
+        gets. A parent whose tags would pass 20 is skipped (``over_limit``) so
+        the editor can still save it. Pair and image tags are not touched.
+        """
+        from .annotations import ImageTagLimitError, normalize_image_tags
+
+        _validate_epoch(expected_epoch)
+        _validate_bulk_ids(prompt_ids)
+        prompt_ids = list(dict.fromkeys(prompt_ids))
+        too_many = f"At most {MAX_LIBRARY_TAGS} tags can be added at once"
+        try:
+            tags = normalize_image_tags(tags)
+        except ImageTagLimitError:
+            # Its message names images; this is a prompt route.
+            raise ValueError(too_many) from None
+        if not tags:
+            raise ValueError("tags must name at least one tag")
+        if len(tags) > MAX_LIBRARY_TAGS:
+            raise ValueError(too_many)
+        with self._pair_transaction() as conn:
+            epoch = self._library_epoch(conn, expected_epoch)
+            held = self._library_parent_tags(conn, prompt_ids)
+            additions, over_limit = {}, 0
+            for pid, names in held.items():
+                present = {name.casefold() for name in names}
+                missing = [name for name in tags if name.casefold() not in present]
+                if not missing:
+                    continue
+                if len(names) + len(missing) > MAX_LIBRARY_TAGS:
+                    over_limit += 1
+                    continue
+                additions[pid] = missing
+            wanted = list(
+                dict.fromkeys(n for names in additions.values() for n in names)
+            )
+            tag_ids = self._ensure_tags(conn, wanted)
+            now = utc_now_iso()
+            for pid, names in additions.items():
+                conn.executemany(
+                    "INSERT OR IGNORE INTO prompt_tags (prompt_id, tag_id) "
+                    "VALUES (?, ?)",
+                    [(pid, tag_ids[name]) for name in names],
+                )
+                self._ensure_library_parent(conn, pid)
+                conn.execute("UPDATE prompts SET updated_at=? WHERE id=?", (now, pid))
+            return {
+                "library_epoch": epoch,
+                "tags": tags,
+                "updated": len(additions),
+                "unchanged": len(held) - len(additions) - over_limit,
+                "not_found": len(prompt_ids) - len(held),
+                "over_limit": over_limit,
+            }
+
+    def _library_parent_tags(
+        self, conn: sqlite3.Connection, prompt_ids: list[int]
+    ) -> dict[int, list[str]]:
+        """The Library parents among ``prompt_ids`` and their own tag names."""
+        held: dict[int, list[str]] = {}
+        for start in range(0, len(prompt_ids), _BULK_CHUNK):
+            chunk = prompt_ids[start : start + _BULK_CHUNK]
+            marks = ",".join("?" for _ in chunk)
+            for (pid,) in conn.execute(
+                f"SELECT p.id FROM prompts p WHERE p.id IN ({marks}) "  # nosec B608
+                f"AND {_ELIGIBLE}",
+                chunk,
+            ):
+                held[pid] = []
+            for pid, name in conn.execute(
+                "SELECT pt.prompt_id,t.name FROM prompt_tags pt "  # nosec B608
+                f"JOIN tags t ON t.id=pt.tag_id WHERE pt.prompt_id IN ({marks})",
+                chunk,
+            ):
+                if pid in held:
+                    held[pid].append(name)
+        return held

database/library_folders.py

@@ -0,0 +1,114 @@
+"""Library output-folder filter (#444): folder names, root prefixes and listing.
+
+A folder is a ``/``-joined path relative to an output root, as Images' folder
+filter and the legacy ``/prompt_manager/subfolders`` name them. A parent
+matches a folder when one of its linked images (``generated_images.prompt_id``,
+the link its previews and image count read) lies in that folder or below it,
+under any spelling of any output root.
+
+Matching compares ``generated_images.file_path``, which is always
+``normcase(normpath(abspath(path)))`` (``models.normalize_image_path``), with
+prefixes built by the same normalization here, so case folding is the
+platform's own on both sides (Windows folds non-ASCII too, unlike SQLite's
+``lower()``) and the comparison is an exact ``substr`` with no wildcards.
+"""
+
+import os
+import sqlite3
+
+_MAX_FOLDER = 1024
+_MAX_PREFIXES = 64
+
+
+def normalize_folder(value: object, pathmod=os.path) -> str:
+    """The folder as ``a/b``, outer slashes dropped; empty means no filter.
+
+    ``/`` always separates. ``\\`` separates only on Windows (``ntpath``): on
+    POSIX it is an ordinary character of a folder's name, and the listing
+    (``relative_folder``) names such a folder with it, so both agree.
+    ``.``/``..``/empty segments, control characters and (on Windows) a drive
+    are refused: the value only ever names a folder below a root.
+    """
+    if not isinstance(value, str) or len(value) > _MAX_FOLDER:
+        raise ValueError(f"folder must be text of at most {_MAX_FOLDER} characters")
+    windows = pathmod.sep == "\\"
+    folder = (value.replace("\\", "/") if windows else value).strip("/")
+    if not folder:
+        return ""
+    parts = folder.split("/")
+    if (
+        any(part in ("", ".", "..") for part in parts)
+        or (windows and ":" in parts[0])
+        or any(ord(char) < 32 or ord(char) == 127 for char in folder)
+    ):
+        raise ValueError("folder must be a folder below an output root")
+    return folder
+
+
+def _base(root: object, pathmod) -> str:
+    return pathmod.normcase(pathmod.normpath(str(root)))
+
+
+def folder_prefixes(roots, folder: str, pathmod=os.path) -> list[str]:
+    """One ``file_path`` prefix per root spelling for a normalized folder."""
+    prefixes = []
+    for root in roots:
+        joined = pathmod.join(_base(root, pathmod), *folder.split("/"))
+        prefix = pathmod.normcase(pathmod.normpath(joined)) + pathmod.sep
+        if prefix not in prefixes:
+            prefixes.append(prefix)
+    if not prefixes or len(prefixes) > _MAX_PREFIXES:
+        raise ValueError("No output folder to filter by")
+    return prefixes
+
+
+def folder_condition(prefixes: list[str]) -> tuple[str, list]:
+    """A condition on alias ``p``: one linked image under any prefix."""
+    tests = " OR ".join("substr(fg.file_path,1,?)=?" for _ in prefixes)
+    params = [value for prefix in prefixes for value in (len(prefix), prefix)]
+    return (
+        "EXISTS(SELECT 1 FROM generated_images fg WHERE fg.prompt_id=p.id "
+        f"AND ({tests}))",
+        params,
+    )
+
+
+def relative_folder(image_path: str, roots, pathmod=os.path) -> str | None:
+    """The folder of *image_path* below the first root spelling holding it.
+
+    The spelling of the stored path is kept (its case on Windows); ``None``
+    for an image directly in a root or outside every root.
+    """
+    directory = pathmod.normpath(pathmod.dirname(image_path))
+    folded = pathmod.normcase(directory)
+    for root in roots:
+        base = _base(root, pathmod)
+        start = base if base.endswith(pathmod.sep) else base + pathmod.sep
+        if not folded.startswith(start):
+            continue
+        # normcase keeps the length for ordinary names; else show the folded form.
+        rest = (directory if len(directory) == len(folded) else folded)[len(start) :]
+        parts = rest.split(pathmod.sep)
+        if any(part in ("", ".", "..") for part in parts):
+            return None
+        return "/".join(parts)
+    return None
+
+
+def list_library_folders(
+    conn: sqlite3.Connection, roots, eligible: str, pathmod=os.path
+) -> list[str]:
+    """Every folder (and ancestor) holding an image of an eligible parent."""
+    folders = set()
+    for (image_path,) in conn.execute(
+        "SELECT DISTINCT g.image_path FROM generated_images g "
+        "JOIN prompts p ON p.id=g.prompt_id WHERE " + eligible
+    ):
+        if not isinstance(image_path, str) or not image_path:
+            continue
+        folder = relative_folder(image_path, roots, pathmod)
+        if folder is None:
+            continue
+        parts = folder.split("/")
+        folders.update("/".join(parts[:index]) for index in range(1, len(parts) + 1))
+    return sorted(folders, key=lambda value: (value.casefold(), value))

database/library_move.py

@@ -0,0 +1,831 @@
+"""Move library (#249 S6b): copy the node-folder library, verify it, switch.
+
+``plan_move`` checks a ``utils.library_location`` location can move and
+sizes the copy; ``move_library`` does it, in this order:
+
+1. **Disk check** on the target's volume (the copy, the backups and
+   config.json, plus a margin). The source is never renamed across folders,
+   so another drive is simply a copy.
+2. **Copy** with SQLite's backup API from a private connection holding one
+   read transaction: the per-table counts, the workflow documents' digest
+   and ``PRAGMA data_version`` describe exactly the copied snapshot. The copy
+   is a temp file in the target folder, made self-contained
+   (``journal_mode=DELETE``), fsynced and owner-only.
+3. **Verify**: ``verify_database_file`` (integrity check, schema, every
+   workflow document decompressed against its hash, #315), the same row
+   count in every table and the same documents.
+4. **Copy files**: every pooled safety backup and pre-migration backup,
+   checked by size and SHA-256 after the copy (their times are kept, so the
+   pool prunes in the same order).
+5. **Publish** under the old database's write lock (``BEGIN IMMEDIATE``) and
+   the config.json lock: if any other connection wrote since the snapshot
+   (``data_version``), steps 2–4 run again (``attempts``, then
+   ``library-busy``). Else config.json is copied (owner-only, checked), a
+   ``pending`` marker is written, the temp files take their final names
+   (``os.replace`` inside the target folder), the marker turns ``moved``
+   and ``switch`` points the process at the new library while the old file
+   is still locked, so no write can land in it.
+6. **Retire**: the old file's WAL is checkpointed and the old ``prompts.db``
+   and ``config.json`` are renamed ``*.moved-to-<folder>`` (never deleted;
+   the old backups stay where they are). A file still open elsewhere is left
+   in place and reported (``old-in-use``): the marker already makes a restart
+   use the new library.
+
+Any failure before the marker says ``moved`` removes everything the move
+wrote in the target folder and leaves the old library in use, untouched.
+A crash at any point leaves either the old library in use (the marker is
+absent or ``pending``; the next move replaces its own leftovers) or the new
+one (``moved``). ``MoveFailed`` carries a code and a message without paths.
+"""
+
+import contextlib
+import hashlib
+import json
+import os
+import shutil
+import sqlite3
+import time
+import types
+import uuid
+from pathlib import Path
+from dataclasses import dataclass, field
+from typing import Callable, Dict, List, Optional, Tuple
+
+from .models import BUSY_TIMEOUT_SECONDS, PromptModel
+
+try:
+    from ..utils import library_location as ll
+    from ..utils.logging_config import get_logger
+    from ..utils.workspace_keys import KEY_FILE_NAME
+except ImportError:
+    from utils import library_location as ll
+    from utils.logging_config import get_logger
+    from utils.workspace_keys import KEY_FILE_NAME
+
+logger = get_logger("prompt_manager.database.library_move")
+
+TEMP_PREFIX = ".pm-move-"
+MOVED_INFIX = ".moved-to-"
+DISK_MARGIN_BYTES = 16 * 2**20
+DEFAULT_ATTEMPTS = 3
+_COPY_CHUNK = 4 * 2**20
+# Backups that live beside the database and move with it.
+BACKUP_INFIXES = (".backup_", ".pre-workflow-storage_")
+
+MESSAGES = {
+    "not-movable": (
+        "This library is not in the node folder, so there is nothing to move."
+    ),
+    "same-folder": "The new folder is the node folder itself.",
+    "target-invalid": "The new folder cannot hold the library.",
+    "target-has-library": (
+        "The new folder already holds a Prompt Manager library. Nothing was moved."
+    ),
+    "target-conflict": (
+        "The new folder already holds files with the same names. Nothing was moved."
+    ),
+    "insufficient-space": (
+        "There is not enough free space for the copy. Nothing was moved."
+    ),
+    "copy-failed": "The copy could not be written or did not match. Nothing was moved.",
+    "verify-failed": "The copy did not pass verification. Nothing was moved.",
+    "library-busy": (
+        "Prompt Manager kept writing to the library while it was copied. "
+        "Nothing was moved; try again when no workflow is running."
+    ),
+    "switch-failed": (
+        "Prompt Manager could not switch to the verified copy, so the copy was "
+        "removed. Nothing was moved."
+    ),
+}
+
+
+class MoveFailed(Exception):
+    """A move that did not happen (or did not finish), with a coded reason."""
+
+    def __init__(self, code: str, message: Optional[str] = None):
+        self.code = code
+        self.message = message or MESSAGES.get(code, "The library was not moved.")
+        super().__init__(self.message)
+
+
+@dataclass(frozen=True)
+class MovePlan:
+    location: "ll.LibraryLocation"
+    db_path: str
+    source_dir: str
+    target_dir: str
+    config_path: Optional[str]
+    backups: Tuple[str, ...]
+    db_bytes: int
+    files_bytes: int
+
+    @property
+    def required_bytes(self) -> int:
+        return self.db_bytes + self.files_bytes + DISK_MARGIN_BYTES
+
+    @property
+    def target_location(self) -> "ll.LibraryLocation":
+        """Where the library is once moved (an explicit config.json stays)."""
+        source = self.location.target_source or ll.SOURCE_DEFAULT
+        config = (
+            self.location.config_path
+            if self.location.config_explicit
+            else os.path.join(self.target_dir, ll.CONFIG_NAME)
+        )
+        return ll.LibraryLocation(
+            source,
+            os.path.join(self.target_dir, ll.DATABASE_NAME),
+            config,
+            self.target_dir,
+            self.target_dir,
+            None,
+            source,
+            self.location.config_explicit,
+        )
+
+
+@dataclass
+class MoveResult:
+    location: "ll.LibraryLocation"
+    counts: Dict[str, int]
+    files: List[str]
+    renamed: List[str] = field(default_factory=list)
+    retire_problem: Optional[str] = None
+
+
+def _size(path: str) -> int:
+    try:
+        return os.path.getsize(path)
+    except OSError:
+        return 0
+
+
+def _free_bytes(path: str) -> int:
+    return shutil.disk_usage(path).free
+
+
+def _existing_ancestor(path: str) -> str:
+    current = os.path.abspath(path)
+    while not os.path.isdir(current):
+        parent = os.path.dirname(current)
+        if parent == current:
+            break
+        current = parent
+    return current
+
+
+def backup_names(db_path: str) -> Tuple[str, ...]:
+    """The pooled and pre-migration backups beside ``db_path`` (names)."""
+    folder = os.path.dirname(os.path.abspath(db_path))
+    base = os.path.basename(db_path)
+    prefixes = tuple(base + infix for infix in BACKUP_INFIXES)
+    try:
+        names = os.listdir(folder)
+    except OSError:
+        return ()
+    return tuple(
+        sorted(
+            name
+            for name in names
+            if name.startswith(prefixes) and os.path.isfile(os.path.join(folder, name))
+        )
+    )
+
+
+def plan_move(location) -> MovePlan:
+    """Check ``location`` can move to its target; raises MoveFailed."""
+    if not location.movable:
+        raise MoveFailed("not-movable")
+    target = os.path.abspath(location.target_dir)
+    source_dir = os.path.dirname(os.path.abspath(location.db_path))
+    if ll.same_folder(target, source_dir) or os.path.normcase(
+        os.path.realpath(target)
+    ) == os.path.normcase(os.path.realpath(source_dir)):
+        raise MoveFailed("same-folder")
+    if os.path.exists(target) and not os.path.isdir(target):
+        raise MoveFailed("target-invalid")
+    backups = backup_names(location.db_path)
+    # config.json moves with a node-folder library; one named by
+    # PROMPT_MANAGER_CONFIG_PATH is the user's and stays where it is.
+    config = None if location.config_explicit else location.config_path
+    config = config if config and os.path.isfile(config) else None
+    if ll.read_marker(target).get("state") != ll.MARKER_PENDING:
+        if os.path.exists(os.path.join(target, ll.DATABASE_NAME)):
+            raise MoveFailed("target-has-library")
+        names = [ll.CONFIG_NAME] if config else []
+        if any(os.path.exists(os.path.join(target, n)) for n in names + list(backups)):
+            raise MoveFailed("target-conflict")
+    db_bytes = _size(location.db_path) + _size(location.db_path + "-wal")
+    files_bytes = sum(_size(os.path.join(source_dir, n)) for n in backups)
+    files_bytes += _size(config) if config else 0
+    return MovePlan(
+        location,
+        os.path.abspath(location.db_path),
+        source_dir,
+        target,
+        config,
+        backups,
+        db_bytes,
+        files_bytes,
+    )
+
+
+def space_problem(plan: MovePlan) -> Optional[Tuple[int, int]]:
+    """``(required, free)`` when the target's volume is short, else None."""
+    free = _free_bytes(_existing_ancestor(plan.target_dir))
+    if free < plan.required_bytes:
+        return plan.required_bytes, free
+    return None
+
+
+# ── copy and verify ───────────────────────────────────────────────────
+
+
+def _tables(conn) -> List[str]:
+    return [
+        row[0]
+        for row in conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' "
+            "AND name NOT LIKE 'sqlite_%' ORDER BY name"
+        )
+    ]
+
+
+def _describe(conn) -> Tuple[Dict[str, int], str]:
+    """Row counts per table and a digest of the workflow documents."""
+    counts = {}
+    for table in _tables(conn):
+        # A name read from sqlite_master, quoted as an identifier.
+        quoted = '"' + table.replace('"', '""') + '"'
+        sql = f"SELECT COUNT(*) FROM {quoted}"  # nosec B608
+        counts[table] = conn.execute(sql).fetchone()[0]
+    digest = hashlib.sha256()
+    if "workflow_documents" in counts:
+        for row in conn.execute(
+            "SELECT id, sha256, byte_length FROM workflow_documents ORDER BY id"
+        ):
+            digest.update(repr((row[0], bytes(row[1]), row[2])).encode())
+    return counts, digest.hexdigest()
+
+
+def _fsync_file(path: str) -> None:
+    with open(path, "rb+") as handle:
+        os.fsync(handle.fileno())
+
+
+def _fsync_dir(path: str) -> None:
+    if os.name != "posix":
+        return
+    fd = os.open(path, os.O_RDONLY)
+    try:
+        os.fsync(fd)
+    finally:
+        os.close(fd)
+
+
+def _private(path: str) -> None:
+    if os.name == "posix":
+        os.chmod(path, 0o600)
+
+
+def _copy_snapshot(source: sqlite3.Connection, temp: str):
+    """Back up ``source``'s current snapshot to ``temp``; return what it holds."""
+    source.execute("BEGIN")
+    try:
+        version = source.execute("PRAGMA data_version").fetchone()[0]
+        counts, documents = _describe(source)
+        destination = sqlite3.connect(temp)
+        try:
+            source.backup(destination)
+        finally:
+            destination.close()
+    finally:
+        source.execute("COMMIT")
+    with contextlib.closing(sqlite3.connect(temp)) as conn:
+        conn.execute("PRAGMA journal_mode=DELETE").fetchone()
+    _fsync_file(temp)
+    _private(temp)
+    return version, counts, documents
+
+
+def _verify_copy(temp: str, counts, documents) -> None:
+    ok, reason = PromptModel.verify_database_file(temp)
+    if not ok:
+        logger.error(f"Library move: the copy did not verify: {reason}")
+        detail = reason if reason.startswith("Workflow document") else None
+        raise MoveFailed(
+            "verify-failed",
+            f"{MESSAGES['verify-failed']} {detail}" if detail else None,
+        )
+    uri = Path(os.path.abspath(temp)).as_uri() + "?mode=ro"
+    try:
+        with contextlib.closing(sqlite3.connect(uri, uri=True)) as conn:
+            copied = _describe(conn)
+    except sqlite3.Error as error:
+        logger.error(f"Library move: the copy could not be read: {error}")
+        raise MoveFailed("verify-failed") from error
+    if copied != (counts, documents):
+        logger.error("Library move: the copy's rows or documents differ")
+        raise MoveFailed("verify-failed")
+
+
+def _hash_copy(source: str, destination: str) -> Tuple[int, str]:
+    """Copy ``source`` to ``destination`` (fsynced, owner-only, times kept)."""
+    digest = hashlib.sha256()
+    size = 0
+    with open(source, "rb") as reader, open(destination, "wb") as writer:
+        while True:
+            chunk = reader.read(_COPY_CHUNK)
+            if not chunk:
+                break
+            digest.update(chunk)
+            size += len(chunk)
+            writer.write(chunk)
+        writer.flush()
+        os.fsync(writer.fileno())
+    stat = os.stat(source)
+    os.utime(destination, ns=(stat.st_atime_ns, stat.st_mtime_ns))
+    _private(destination)
+    return size, digest.hexdigest()
+
+
+def _file_digest(path: str) -> Tuple[int, str]:
+    digest = hashlib.sha256()
+    size = 0
+    with open(path, "rb") as reader:
+        while True:
+            chunk = reader.read(_COPY_CHUNK)
+            if not chunk:
+                return size, digest.hexdigest()
+            digest.update(chunk)
+            size += len(chunk)
+
+
+def _check_copies(copies) -> None:
+    for source, temp, expected in copies:
+        if _file_digest(temp) != expected or _file_digest(source) != expected:
+            logger.error(
+                f"Library move: {os.path.basename(source)} did not copy unchanged"
+            )
+            raise MoveFailed("copy-failed")
+
+
+def write_marker(
+    directory: str, state: str, source_db: str, source_config: Optional[str] = None
+) -> str:
+    """Atomically write the move marker; return its path.
+
+    ``from`` and ``config_from`` name the old files, so a restart after a
+    crash between the switch and the rename can finish the rename.
+    """
+    path = os.path.join(directory, ll.MARKER_NAME)
+    temp = os.path.join(directory, f"{TEMP_PREFIX}{uuid.uuid4().hex}.marker")
+    payload = json.dumps(
+        {
+            "version": 1,
+            "state": state,
+            "from": source_db,
+            "config_from": source_config,
+            "updated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        },
+        indent=2,
+    ).encode("utf-8")
+    fd = os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+    try:
+        with os.fdopen(fd, "wb") as handle:
+            handle.write(payload)
+            handle.flush()
+            os.fsync(handle.fileno())
+        _private(temp)
+        os.replace(temp, path)
+    except BaseException:
+        _remove(temp)
+        raise
+    _fsync_dir(directory)
+    return path
+
+
+def _remove(*paths) -> None:
+    for path in paths:
+        try:
+            if path and os.path.lexists(path):
+                os.remove(path)
+        except OSError:
+            logger.warning(f"Library move: could not remove {os.path.basename(path)}")
+
+
+def sweep_move_temps(directory: str) -> List[str]:
+    """Remove the temp files an interrupted move left in ``directory``."""
+    removed = []
+    try:
+        names = os.listdir(directory)
+    except OSError:
+        return removed
+    for name in names:
+        if name.startswith(TEMP_PREFIX):
+            _remove(os.path.join(directory, name))
+            removed.append(name)
+    return removed
+
+
+# ── the move ─────────────────────────────────────────────────────────
+
+
+def _noop(*_args, **_kwargs):
+    return None
+
+
+def switch_process(old_db: str, location, detached: list) -> None:
+    """Point this process at ``location``: resolver, every model, observers.
+
+    Adds the old file's connections, detached and still open, to
+    ``detached`` as soon as they are detached, so the mover retires them
+    (``PromptModel.retire_detached``) whether or not the rest succeeds.
+    """
+    from .execution_history import rekey_history_restore
+    from .review_changes import rekey_review_changes
+
+    ll.switch_to(location)
+    detached.extend(PromptModel.repoint_instances(old_db, location.db_path))
+    rekey_review_changes(old_db, location.db_path)
+    rekey_history_restore(old_db, location.db_path)
+
+
+def unswitch_process(old_db: str, new_db: str) -> list:
+    """Undo ``switch_process`` (any part of it); returns detached connections."""
+    from .execution_history import rekey_history_restore
+    from .review_changes import rekey_review_changes
+
+    ll.clear_override()
+    detached = PromptModel.repoint_instances(new_db, old_db)
+    rekey_review_changes(new_db, old_db)
+    rekey_history_restore(new_db, old_db)
+    return detached
+
+
+class _Move:
+    def __init__(self, plan, *, switch, config_lock, progress, checkpoint):
+        self.plan = plan
+        self.switch = switch
+        self.config_lock = config_lock
+        self.progress = progress
+        self.checkpoint = checkpoint
+        self.created_dir = False
+        self.temps: List[str] = []
+        self.published: List[str] = []
+        self.marker: Optional[str] = None
+        self.moved = False
+        # Retired connections still in use; awaited once the locks are free.
+        self.retiring: list = []
+
+    def temp(self, suffix: str) -> str:
+        path = os.path.join(
+            self.plan.target_dir, f"{TEMP_PREFIX}{uuid.uuid4().hex}{suffix}"
+        )
+        self.temps.append(path)
+        return path
+
+    def prepare_target(self) -> None:
+        target = self.plan.target_dir
+        self.created_dir = not os.path.isdir(target)
+        ll.ensure_private_dir(target)
+        sweep_move_temps(target)
+        if ll.read_marker(target).get("state") == ll.MARKER_PENDING:
+            # An earlier move stopped before it switched: its files are ours.
+            names = (ll.DATABASE_NAME, ll.CONFIG_NAME, KEY_FILE_NAME)
+            for name in names + self.plan.backups:
+                _remove(os.path.join(target, name))
+            for suffix in ("-wal", "-shm", "-journal"):
+                _remove(os.path.join(target, ll.DATABASE_NAME + suffix))
+
+    def copy_round(self, source):
+        """Steps 2–4 once: the verified temp copies and the snapshot version."""
+        self.progress("Copying the database")
+        temp_db = self.temp(".db")
+        version, counts, documents = _copy_snapshot(source, temp_db)
+        self.checkpoint("copied")
+        self.progress("Verifying the copy")
+        _verify_copy(temp_db, counts, documents)
+        self.checkpoint("verified")
+        copies = []
+        for name in self.plan.backups:
+            src = os.path.join(self.plan.source_dir, name)
+            temp = self.temp(".bak")
+            try:
+                expected = _hash_copy(src, temp)
+            except OSError as error:
+                logger.error(f"Library move: copying {name} failed: {error}")
+                raise MoveFailed("copy-failed") from error
+            copies.append((src, temp, expected, name))
+        if copies:
+            self.progress(f"Copied {len(copies)} backup file(s)")
+        self.checkpoint("files")
+        _check_copies([(s, t, e) for s, t, e, _ in copies])
+        return version, counts, temp_db, copies
+
+    def discard_round(self, temp_db, copies) -> None:
+        _remove(temp_db, *[temp for _s, temp, _e, _n in copies])
+
+    def publish(self, temp_db, copies) -> List[str]:
+        """Under the locks: config.json, marker, final names, marker moved."""
+        target = self.plan.target_dir
+        files = []
+        if self.plan.config_path:
+            temp = self.temp(".config")
+            try:
+                expected = _hash_copy(self.plan.config_path, temp)
+            except OSError as error:
+                raise MoveFailed("copy-failed") from error
+            _check_copies([(self.plan.config_path, temp, expected)])
+            copies = copies + [(self.plan.config_path, temp, expected, ll.CONFIG_NAME)]
+        copies = copies + self._key_copy()
+        self.marker = write_marker(
+            target, ll.MARKER_PENDING, self.plan.db_path, self.plan.config_path
+        )
+        self.checkpoint("pending")
+        for _src, temp, _expected, name in copies:
+            final = os.path.join(target, name)
+            os.replace(temp, final)
+            self.published.append(final)
+            files.append(name)
+        final_db = os.path.join(target, ll.DATABASE_NAME)
+        os.replace(temp_db, final_db)
+        self.published.append(final_db)
+        files.append(ll.DATABASE_NAME)
+        _fsync_dir(target)
+        self.checkpoint("published")
+        write_marker(target, ll.MARKER_MOVED, self.plan.db_path, self.plan.config_path)
+        self.moved = True
+        self.checkpoint("marked")
+        return files
+
+    def _key_copy(self) -> list:
+        """The workspace key's verified copy (#430), or none: it is re-creatable.
+
+        Without it the moved library starts with a new key, so pages left open
+        across the restart refresh once; that never fails the move.
+        """
+        source = os.path.join(self.plan.source_dir, KEY_FILE_NAME)
+        if not os.path.isfile(source):
+            return []
+        temp = self.temp(".key")
+        try:
+            expected = _hash_copy(source, temp)
+            _check_copies([(source, temp, expected)])
+        except (OSError, MoveFailed) as error:
+            _remove(temp)
+            logger.warning(
+                f"Library move: the workspace key was not copied ({error}); "
+                "open workspace pages refresh once after the restart"
+            )
+            return []
+        return [(source, temp, expected, KEY_FILE_NAME)]
+
+    def cleanup(self) -> None:
+        if self.moved:
+            return
+        _remove(*self.temps, *self.published, self.marker)
+        if self.created_dir:
+            try:
+                os.rmdir(self.plan.target_dir)
+            except OSError:
+                pass
+
+
+def _unique(path: str) -> str:
+    candidate, number = path, 1
+    while os.path.lexists(candidate):
+        candidate = f"{path}_{number}"
+        number += 1
+    return candidate
+
+
+def retire_old_files(plan: MovePlan) -> Tuple[List[str], Optional[str]]:
+    """Rename the old database and config.json; ``(renamed, problem code)``.
+
+    Every connection to the old file must be closed. The WAL is checkpointed
+    first; when it cannot be emptied (another process still has the file
+    open) nothing is renamed (``old-in-use``).
+    """
+    old = plan.db_path
+    folder = os.path.basename(os.path.normpath(plan.target_dir))
+    try:
+        with contextlib.closing(sqlite3.connect(old, timeout=1)) as conn:
+            busy = conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()[0]
+    except sqlite3.Error as error:
+        logger.warning(
+            f"Library move: the old database could not be checkpointed: {error}"
+        )
+        return [], "old-in-use"
+    if busy or _size(old + "-wal") > 0:
+        return [], "old-in-use"
+    renamed = []
+    try:
+        for path in (old, plan.config_path):
+            if not path or not os.path.isfile(path):
+                continue
+            new_name = _unique(path + MOVED_INFIX + folder)
+            os.rename(path, new_name)
+            renamed.append(os.path.basename(new_name))
+        for suffix in ("-wal", "-shm"):
+            if os.path.isfile(old + suffix) and _size(old + suffix) == 0:
+                os.remove(old + suffix)
+    except OSError as error:
+        logger.warning(f"Library move: the old files could not be renamed: {error}")
+        return renamed, "old-locked"
+    return renamed, None
+
+
+def finish_interrupted_move(location) -> List[str]:
+    """Rename old files a crash left between the switch and the rename.
+
+    The marker of the folder in use says ``moved`` and names the old files;
+    an old ``prompts.db`` (and config.json) still there under its own name is
+    renamed ``*.moved-to-<folder>`` now, never deleted. Returns the new
+    names (nothing when there is nothing to finish or the file is in use).
+    """
+    if location.source not in (ll.SOURCE_DEFAULT, ll.SOURCE_ENVIRONMENT):
+        return []
+    marker = ll.read_marker(location.data_dir)
+    old = marker.get("from")
+    if marker.get("state") != ll.MARKER_MOVED or not isinstance(old, str):
+        return []
+    if not os.path.isfile(old) or os.path.basename(old) != ll.DATABASE_NAME:
+        return []
+    if ll.same_folder(os.path.dirname(old), location.data_dir):
+        return []
+    config = marker.get("config_from")
+    if not (
+        isinstance(config, str)
+        and os.path.basename(config) == ll.CONFIG_NAME
+        and os.path.isfile(config)
+        and os.path.normcase(os.path.abspath(config))
+        != os.path.normcase(os.path.abspath(location.config_path))
+    ):
+        config = None
+    pending = types.SimpleNamespace(
+        db_path=old, config_path=config, target_dir=location.data_dir
+    )
+    renamed, problem = retire_old_files(pending)
+    if problem:
+        logger.warning(f"Library move: the old files are still in use ({problem})")
+    elif renamed:
+        logger.info(f"Library move finished: renamed {', '.join(renamed)}")
+    return renamed
+
+
+def move_library(
+    plan: MovePlan,
+    *,
+    switch: Callable = switch_process,
+    unswitch: Callable = unswitch_process,
+    after_switch: Callable = _noop,
+    config_lock=None,
+    progress: Callable[[str], None] = _noop,
+    checkpoint: Callable[[str], None] = _noop,
+    attempts: int = DEFAULT_ATTEMPTS,
+) -> MoveResult:
+    """Move the library as planned (blocking; see the module docstring).
+
+    ``switch(old_db, location, detached)`` points the process at the copy
+    and adds the old connections it detached to ``detached``; if it raises,
+    ``unswitch(old_db, new_db)`` undoes it while both files are still
+    locked, every detached connection is closed, the copy is removed and
+    the move fails (``switch-failed``). ``after_switch()`` runs
+    once the switch is final; it is best effort (logged, never fails the move).
+    """
+    short = space_problem(plan)
+    if short is not None:
+        raise MoveFailed("insufficient-space")
+    lock = config_lock if config_lock is not None else contextlib.nullcontext()
+    move = _Move(
+        plan,
+        switch=switch,
+        config_lock=lock,
+        progress=progress,
+        checkpoint=checkpoint,
+    )
+    move.unswitch = unswitch
+    location = plan.target_location
+    try:
+        move.prepare_target()
+        files, counts = _copy_and_switch(move, location, attempts)
+    except Exception:
+        move.cleanup()
+        raise
+    try:
+        after_switch()
+    except Exception:
+        logger.exception("Library move: a follow-up after the switch failed")
+    from .execution_history import notify_history_restore
+
+    notify_history_restore(location.db_path)
+    progress("Renaming the old files")
+    renamed, problem = retire_old_files(plan)
+    checkpoint("retired")
+    return MoveResult(location, counts, files, renamed, problem)
+
+
+def _switch_guarded(move: _Move, location) -> list:
+    """Switch while the copy is locked too; undo it all if any part fails.
+
+    The copy's write lock keeps every repointed thread from writing to it
+    until the switch is whole, so undoing a partial switch loses nothing.
+    Returns the old file's detached connections (still open). On failure
+    every detached connection is retired; those still in use are added to
+    ``move.retiring`` for the caller to await once the locks are released.
+    """
+    plan = move.plan
+    guard = sqlite3.connect(
+        location.db_path, timeout=BUSY_TIMEOUT_SECONDS, isolation_level=None
+    )
+    try:
+        guard.execute("BEGIN IMMEDIATE")
+        detached: list = []
+        try:
+            move.switch(plan.db_path, location, detached)
+        except Exception as error:
+            logger.exception("Library move: switching failed; undoing it")
+            move.moved = False  # the copy and its marker are removed
+            # Retire, never wait, here: a connection still in use may be
+            # waiting on the very locks this thread holds.
+            try:
+                undone = move.unswitch(plan.db_path, location.db_path)
+            except Exception:
+                logger.exception("Library move: undoing the switch failed")
+            else:
+                _retire(move, undone)
+            # The old file's connections a partial switch detached: the
+            # models reopen it on next use, so close them now, not at GC.
+            _retire(move, detached)
+            raise MoveFailed("switch-failed") from error
+        move.checkpoint("switched")
+        return detached
+    finally:
+        if guard.in_transaction:
+            guard.execute("ROLLBACK")
+        guard.close()
+
+
+def _copy_and_switch(move: _Move, location, attempts: int):
+    """Copy, then switch under both locks; retired connections are awaited.
+
+    A connection that was inside a statement when it was retired is closed
+    by its own thread once the statement ends, which may need the locks
+    released first; the old files are renamed (and a failed copy removed)
+    only after that, since Windows refuses to rename or delete an open file.
+    """
+    try:
+        return _copy_and_switch_locked(move, location, attempts)
+    finally:
+        try:
+            PromptModel.await_detached(move.retiring, logger)
+        except Exception:
+            logger.exception("Library move: waiting for old connections failed")
+
+
+def _retire(move: _Move, connections: list) -> None:
+    """Retire ``connections``, adding those still in use to ``move.retiring``.
+
+    Never raises: a switch that completed must not be reported as failed
+    because one of the old connections could not be retired (logged; the
+    rename then reports the old files as still in use).
+    """
+    try:
+        move.retiring.extend(PromptModel.retire_detached(connections, logger))
+    except Exception:
+        logger.exception("Library move: retiring the old connections failed")
+
+
+def _copy_and_switch_locked(move: _Move, location, attempts: int):
+    plan = move.plan
+    for attempt in range(1, attempts + 1):
+        source = sqlite3.connect(
+            plan.db_path, timeout=BUSY_TIMEOUT_SECONDS, isolation_level=None
+        )
+        try:
+            source.execute(f"PRAGMA busy_timeout = {BUSY_TIMEOUT_SECONDS * 1000}")
+            version, counts, temp_db, copies = move.copy_round(source)
+            source.execute("BEGIN IMMEDIATE")
+            try:
+                if source.execute("PRAGMA data_version").fetchone()[0] != version:
+                    move.progress("The library changed during the copy; copying again")
+                    move.discard_round(temp_db, copies)
+                    continue
+                move.checkpoint("locked")
+                with move.config_lock:
+                    files = move.publish(temp_db, copies)
+                    detached = _switch_guarded(move, location)
+                # Still holding the old file's lock: a write that was waiting
+                # on it is interrupted and cannot commit, so it fails (and
+                # runs again on the new library) instead of landing in the
+                # old file. Its own thread closes the connection.
+                _retire(move, detached)
+            finally:
+                if source.in_transaction:
+                    source.execute("ROLLBACK")
+            return files, counts
+        finally:
+            source.close()
+    raise MoveFailed("library-busy")

database/maintenance.py

@@ -0,0 +1,877 @@
+"""Reviewed maintenance plans: read-only previews and their exact apply (#249 S6).
+
+Four operations, each the native owner of a legacy ``/maintenance`` one and
+built on the same queries, so a preview counts what the legacy cleanup would
+change:
+
+- ``duplicate_prompts``: prompts whose text is equal ignoring case and outer
+  spaces (``DUPLICATE_PROMPTS_SQL``, legacy ``cleanup_duplicates``); each
+  group keeps its oldest prompt, which takes over the others' images and
+  metadata. A group with a member in a saved pair is kept whole.
+- ``optimize``: VACUUM (legacy ``vacuum``).
+- ``unused_components``: prompts nothing references (``ORPHANED_PROMPTS_SQL``,
+  legacy ``prune_orphaned_prompts``, with the #239 protections).
+- ``missing_links``: image rows whose file is gone (legacy
+  ``cleanup_orphaned_images``), counted only under a source that can be read
+  now, as the health check's ``missing_files`` judges them
+  (``health.link_states``). It deletes database rows only, never a file.
+  It also forgets the removal marks (#480) that hide nothing any more: the
+  file is gone or changed, under a readable source (#501, ``_stale_marks``).
+
+``preview_maintenance`` reads one snapshot on its own ``mode=ro`` connection
+(``health.readonly_uri``) and closes it in ``finally``; files are looked at
+after the snapshot ended. It returns a ``MaintenanceDraft``: the public plan
+fields, the targets an apply needs and the fingerprint of exactly what was
+reviewed (ids, text hashes, metadata, tags, Library state and pair refs of
+every prompt involved, or each link's row and source). The fingerprint never
+leaves the server.
+
+``apply_maintenance`` recomputes that fingerprint on the write connection
+inside ``BEGIN IMMEDIATE`` and raises ``PlanChanged`` before writing anything
+when it differs; otherwise it applies the reviewed targets in batches of
+``ID_BATCH`` in that one transaction. VACUUM cannot run in a transaction, so
+``optimize`` binds to its database only (the caller checks that).
+"""
+
+import hashlib
+import json
+import os
+import shutil
+import sqlite3
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from .execution_contexts import LIVE_HINT_CTES
+from .health import BATCH_ROWS, link_states, readable_roots, readonly_uri
+from .image_removal_schema import TABLE as MARKS_TABLE
+from .image_removal_schema import clear_marks, has_marks_table
+from .pairs import PairDependencyError
+from .workflow_documents import documents_of_rows, release_documents
+
+try:
+    from ..utils.hashing import register_prompt_functions
+except ImportError:
+    from utils.hashing import register_prompt_functions
+
+MAINTENANCE_OPERATIONS = (
+    "duplicate_prompts",
+    "optimize",
+    "unused_components",
+    "missing_links",
+    # Applied as its own batched job (database/workflow_storage.py).
+    "workflow_storage",
+)
+MAX_DETAILS = 50  # detail rows a plan lists
+MAX_RESULT_ROWS = 200  # changed records an apply describes (the job lists 200)
+EXCERPT_CHARS = 80
+# Ids bound per statement: the pair-reference read binds each id twice, which
+# stays under SQLite's historical 999-variable limit.
+ID_BATCH = 400
+
+# Why a prompt no image links to is kept (#239): predicates on ``p``, in the
+# order the plan names them. ORPHANED_PROMPTS_SQL is built from them, so the
+# reasons a plan lists are exactly the rules the prune applies.
+PROTECTIONS = (
+    (
+        "EXISTS (SELECT 1 FROM prompt_pairs pp "
+        "WHERE pp.positive_prompt_id = p.id OR pp.negative_prompt_id = p.id)",
+        "in saved pairs",
+        "in a saved pair",
+    ),
+    (
+        "EXISTS (SELECT 1 FROM prompt_tags pt JOIN tags t ON pt.tag_id = t.id "
+        "WHERE pt.prompt_id = p.id AND t.name = '__protected__')",
+        "tagged __protected__",
+        "tagged __protected__",
+    ),
+    (
+        "EXISTS (SELECT 1 FROM prompt_library_state ls WHERE ls.prompt_id = p.id)",
+        "with Library state (a title or an explicit parent)",
+        "with Library state (a title or an explicit parent)",
+    ),
+    (
+        # #443: an archived prompt is restorable, never pruned.
+        "EXISTS (SELECT 1 FROM prompt_archive pa WHERE pa.prompt_id = p.id)",
+        "archived in the Library",
+        "archived in the Library",
+    ),
+    (
+        "p.id IN (SELECT prompt_id FROM live_hint_prompts)",
+        "used by recorded runs",
+        "used by a recorded run",
+    ),
+)
+
+# Exempt references: images, pairs, the __protected__ tag, Library parent
+# state, the Library archive, and live recorded-run hints (computed once,
+# see LIVE_HINT_CTES).
+ORPHANED_PROMPTS_SQL = (
+    "WITH "
+    + LIVE_HINT_CTES
+    + " SELECT p.id FROM prompts p"
+    + " LEFT JOIN generated_images gi ON p.id = gi.prompt_id"
+    + " WHERE gi.prompt_id IS NULL"
+    + "".join(f" AND NOT ({predicate})" for predicate, _many, _one in PROTECTIONS)
+)
+# The prompts no image links to, with which protections keep each one.
+_UNLINKED_PROTECTIONS_SQL = (
+    "WITH "
+    + LIVE_HINT_CTES
+    + " SELECT "
+    + ", ".join(
+        f"COALESCE(SUM({predicate}), 0)" for predicate, _many, _one in PROTECTIONS
+    )
+    + ", COALESCE(SUM("
+    + " OR ".join(f"({predicate})" for predicate, _many, _one in PROTECTIONS)
+    + "), 0) FROM prompts p WHERE NOT EXISTS "
+    "(SELECT 1 FROM generated_images gi WHERE gi.prompt_id = p.id)"
+)
+
+# Duplicate prompt groups, as the legacy cleanup finds them (case-insensitive
+# text; GROUP_CONCAT order is sorted by created_at afterwards).
+DUPLICATE_PROMPTS_SQL = """
+    SELECT LOWER(TRIM(text)) as normalized_text, COUNT(*) as count,
+           GROUP_CONCAT(id) as ids,
+           GROUP_CONCAT(created_at) as created_dates
+    FROM prompts
+    GROUP BY LOWER(TRIM(text))
+    HAVING COUNT(*) > 1
+"""
+
+_LINK_ROWS_SQL = (
+    "SELECT id, image_path, filename, file_path FROM generated_images ORDER BY id"
+)
+
+# What a prompt's fingerprint covers, per batch of ids.
+_PROMPT_STATE_SQL = (
+    "SELECT id, hash, text, category, rating, notes, created_at, updated_at "
+    "FROM prompts WHERE id IN ({marks}) ORDER BY id",
+    "SELECT prompt_id, tag_id FROM prompt_tags WHERE prompt_id IN ({marks}) "
+    "ORDER BY prompt_id, tag_id",
+    "SELECT * FROM prompt_library_state WHERE prompt_id IN ({marks}) "
+    "ORDER BY prompt_id",
+    "SELECT id, positive_prompt_id, negative_prompt_id FROM prompt_pairs "
+    "WHERE positive_prompt_id IN ({marks}) OR negative_prompt_id IN ({marks}) "
+    "ORDER BY id",
+)
+
+NO_FILES_DELETED = "No image file is deleted."
+LINKS_ONLY = "Database links only; no image file is deleted."
+UNUSED_DETAIL = "No image, saved pair, Library state or recorded run uses it"
+
+
+# Why a duplicate group is kept whole (``_duplicate_conflict``).
+CONFLICT_REASONS = {
+    "collision": ("group links one image twice", "groups link one image twice"),
+    "title": (
+        "group has its own Library title",
+        "groups have their own Library titles",
+    ),
+}
+CONFLICT_DETAILS = {
+    "collision": "Kept: two copies link the same image",
+    "title": "Kept: a copy has its own Library title",
+}
+OPTIMIZE_LOCK_NOTE = (
+    "Runs alone: the database is locked while it runs, and on a large library "
+    "this takes a while. Prompt runs counted and images captured meanwhile "
+    "wait up to 30 seconds each; anything that waits longer is not recorded."
+)
+
+
+class PlanChanged(Exception):
+    """What a plan was reviewed on changed before it was applied."""
+
+
+class MergeConflict(Exception):
+    """A strict duplicate merge would drop an image link or a Library title."""
+
+
+@dataclass
+class MaintenanceDraft:
+    """A preview: the public plan fields plus what only the server keeps."""
+
+    operation: str
+    fingerprint: str
+    targets: list
+    scope: dict
+    protected: dict
+    media: dict
+    details: list = field(default_factory=list)
+    # missing_links only: locators of removal marks whose file is gone (#501).
+    marks: list = field(default_factory=list)
+
+    def public(self):
+        """The plan fields a response may carry (no fingerprint, no targets)."""
+        return {
+            "operation": self.operation,
+            "scope": dict(self.scope),
+            "protected": {
+                "count": self.protected["count"],
+                "reasons": list(self.protected["reasons"]),
+            },
+            "media": dict(self.media),
+            "details": [dict(row) for row in self.details],
+        }
+
+
+def _plural(count, one, many=None):
+    return f"{count:,} {one if count == 1 else (many or one + 's')}"
+
+
+def _excerpt(text):
+    text = " ".join(str(text or "").split())
+    if len(text) <= EXCERPT_CHARS:
+        return text
+    return text[: EXCERPT_CHARS - 1].rstrip() + "…"
+
+
+def _megabytes(size):
+    return f"{size / 2**20:,.1f} MB"
+
+
+class _Fingerprint:
+    """A streaming sha256 over JSON-encoded parts."""
+
+    def __init__(self, operation):
+        self._hash = hashlib.sha256()
+        self.add(operation)
+
+    def add(self, part):
+        encoded = json.dumps(part, separators=(",", ":"), default=str)
+        self._hash.update(encoded.encode("utf-8") + b"\n")
+
+    def hexdigest(self):
+        return "sha256:" + self._hash.hexdigest()
+
+
+def _batches(ids):
+    ids = list(ids)
+    for start in range(0, len(ids), ID_BATCH):
+        yield ids[start : start + ID_BATCH]
+
+
+def _marks(batch):
+    return ",".join("?" for _ in batch)
+
+
+def _add_prompt_state(fingerprint, conn, ids):
+    """Fold every reviewed fact about the prompts ``ids`` into ``fingerprint``."""
+    for batch in _batches(sorted(ids)):
+        marks = _marks(batch)
+        for sql in _PROMPT_STATE_SQL:
+            params = batch * sql.count("{marks}")
+            for row in conn.execute(sql.format(marks=marks), params):
+                fingerprint.add(list(row))
+
+
+def _texts(conn, ids, limit=MAX_DETAILS):
+    """``{id: text}`` of the first ``limit`` prompts, for detail rows."""
+    ids = list(ids)[:limit]
+    if not ids:
+        return {}
+    rows = conn.execute(
+        f"SELECT id, text FROM prompts WHERE id IN ({_marks(ids)})", ids
+    )
+    return {row[0]: row[1] for row in rows}
+
+
+def _orphaned_ids(conn):
+    return sorted(row[0] for row in conn.execute(ORPHANED_PROMPTS_SQL))
+
+
+def _delete_ids(conn, table, ids):
+    """Delete ``table`` rows by id in batches; returns the rows removed.
+
+    The workflow documents only those rows referenced go with them (a
+    prompt's images go by cascade), in the same transaction.
+    """
+    if table not in ("prompts", "generated_images"):
+        raise ValueError(f"Unsupported table {table!r}")
+    removed = 0
+    for batch in _batches(ids):
+        documents = documents_of_rows(
+            conn, "id" if table == "generated_images" else "prompt_id", batch
+        )
+        cursor = conn.execute(
+            f"DELETE FROM {table} WHERE id IN ({_marks(batch)})",  # nosec B608
+            batch,
+        )
+        removed += cursor.rowcount
+        release_documents(conn, documents)
+    return removed
+
+
+def _mark_rows(conn, only=None):
+    """``[(locator, size, mtime_ns)]`` of the removal marks (all, or those
+    among *only*), sorted by locator."""
+    if not has_marks_table(conn):
+        return []
+    sql = f"SELECT locator, size, mtime_ns FROM {MARKS_TABLE}"  # nosec B608
+    if only is None:
+        return sorted(tuple(row) for row in conn.execute(sql))
+    found = []
+    for batch in _batches(sorted(only)):
+        found.extend(
+            tuple(row)
+            for row in conn.execute(
+                f"{sql} WHERE locator IN ({_marks(batch)})", batch  # nosec B608
+            )
+        )
+    return sorted(found)
+
+
+def _lists_something(folder):
+    """Whether *folder* holds at least one entry (an unplugged drive's mount
+    point, or an unbound volume, is an empty folder that reads fine)."""
+    try:
+        with os.scandir(folder) as entries:
+            return next(entries, None) is not None
+    except OSError:
+        return False
+
+
+def _name_listed(path):
+    """Whether *path*'s folder lists its name, compared under ``normcase``.
+
+    A locator is ``normcase``d; on a case-sensitive NTFS folder ``lstat`` of
+    the lower-cased spelling fails for a file that is there.
+    """
+    folder, name = os.path.split(path)
+    wanted = os.path.normcase(name)
+    try:
+        with os.scandir(folder) as entries:
+            return any(os.path.normcase(entry.name) == wanted for entry in entries)
+    except OSError:
+        return True  # cannot tell: keep the mark
+
+
+def _stale_mark(locator, size, mtime_ns):
+    """Whether the file of a mark is gone or has other bytes (blocking)."""
+    try:
+        info = os.lstat(locator)
+    except (FileNotFoundError, NotADirectoryError):
+        return os.path.isdir(os.path.dirname(locator)) and not _name_listed(locator)
+    except OSError:
+        return False
+    return (info.st_size, info.st_mtime_ns) != (size, mtime_ns)
+
+
+def _stale_marks(marks, roots):
+    """``[(locator, owner)]`` of the marks that hide nothing any more (#501).
+
+    *marks* are ``(locator, size, mtime_ns)`` rows. A mark is stale when its
+    file was changed, or is gone from a folder that still exists. Only marks
+    under a source that can be read now and lists something are judged (an
+    unplugged drive's marks, or its empty mount point's, stay); any doubt
+    keeps the mark. *owner* is the index of the longest source holding it.
+    Locators are ``normcase(realpath)`` paths, so the sources are too.
+    """
+    readable = readable_roots(roots)
+    prefixes = [
+        os.path.normcase(os.path.realpath(root)).rstrip("\\/") + os.sep
+        for root in roots
+    ]
+    judged = {}
+    stale = []
+    for locator, size, mtime_ns in marks:
+        owners = [i for i, prefix in enumerate(prefixes) if locator.startswith(prefix)]
+        if not owners:
+            continue  # under no source now (a source removed from the settings)
+        owner = max(owners, key=lambda i: len(prefixes[i]))
+        if owner not in judged:
+            judged[owner] = readable[owner] and _lists_something(roots[owner])
+        if judged[owner] and _stale_mark(locator, size, mtime_ns):
+            stale.append((locator, owner))
+    return stale
+
+
+def _read_snapshot(db_path, read):
+    """``read(conn)`` on one ``mode=ro`` snapshot; the connection always closes."""
+    conn = sqlite3.connect(readonly_uri(db_path), uri=True)
+    try:
+        conn.row_factory = sqlite3.Row
+        register_prompt_functions(conn)  # LIVE_HINT_CTES calls them (#460)
+        conn.execute("BEGIN")  # one read snapshot for the whole plan
+        return read(conn)
+    finally:
+        conn.close()
+
+
+def _relative(image_path, root):
+    try:
+        return Path(os.path.relpath(image_path, root)).as_posix()
+    except ValueError:  # another drive (Windows)
+        return os.path.basename(image_path)
+
+
+def _source_name(root, index):
+    return os.path.basename(str(root).rstrip("\\/")) or f"Source {index + 1}"
+
+
+class MaintenanceOperations:
+    """Mixin for PromptDatabase: maintenance previews and their apply."""
+
+    # ── shared with the legacy cleanup ───────────────────────────────
+
+    def _duplicate_groups(self, conn):
+        """``(groups, paired)``: duplicate id groups, oldest first.
+
+        ``paired`` are the groups with a member in a saved pair, which the
+        cleanup keeps whole.
+        """
+        groups, paired = [], []
+        for duplicate in conn.execute(DUPLICATE_PROMPTS_SQL).fetchall():
+            ids = duplicate["ids"].split(",")
+            created_dates = duplicate["created_dates"].split(",")
+            # Sort IDs by created_at date to keep the oldest
+            id_date_pairs = list(zip(ids, created_dates))
+            id_date_pairs.sort(key=lambda x: x[1])
+            sorted_ids = [int(pair[0]) for pair in id_date_pairs]
+            try:
+                self._ensure_prompts_unpaired(conn, sorted_ids)
+            except PairDependencyError:
+                paired.append(sorted_ids)
+                continue
+            groups.append(sorted_ids)
+        return groups, paired
+
+    @staticmethod
+    def _duplicate_conflict(conn, sorted_ids):
+        """Why merging the group would lose something, or None.
+
+        ``collision``: two members link the same file, so moving the images
+        to the oldest prompt would hit ``UNIQUE(prompt_id, file_path)``.
+        ``title``: a member that would be deleted has its own Library title.
+        """
+        marks = _marks(sorted_ids)
+        collision = conn.execute(
+            "SELECT 1 FROM generated_images WHERE prompt_id IN "  # nosec B608
+            f"({marks}) AND file_path IS NOT NULL GROUP BY file_path "
+            "HAVING COUNT(DISTINCT prompt_id) > 1 LIMIT 1",
+            sorted_ids,
+        ).fetchone()
+        if collision:
+            return "collision"
+        duplicates = sorted_ids[1:]
+        titled = conn.execute(
+            "SELECT 1 FROM prompt_library_state WHERE prompt_id IN "  # nosec B608
+            f"({_marks(duplicates)}) AND trim(coalesce(title, '')) != '' LIMIT 1",
+            duplicates,
+        ).fetchone()
+        return "title" if titled else None
+
+    def _merge_duplicate_group(self, conn, sorted_ids, strict=False):
+        """Keep the oldest prompt of a group; returns (removed, images moved).
+
+        ``strict`` (the reviewed apply): a merge that would drop an image
+        link or a Library title raises MergeConflict, and every helper error
+        propagates, so the transaction rolls back. The legacy cleanup keeps
+        its old, lenient behaviour.
+        """
+        primary_id, duplicate_ids = sorted_ids[0], sorted_ids[1:]
+        if strict:
+            conflict = self._duplicate_conflict(conn, sorted_ids)
+            if conflict:
+                raise MergeConflict(conflict)
+        self.logger.debug(
+            f"Merging duplicates: keeping {primary_id}, removing {duplicate_ids}"
+        )
+        primary = conn.execute(
+            "SELECT * FROM prompts WHERE id = ?", (primary_id,)
+        ).fetchone()
+        if not primary:
+            return 0, 0
+        merged_metadata = self._merge_duplicate_metadata(
+            conn, primary_id, duplicate_ids, strict=strict
+        )
+        moved = self._transfer_images_to_primary(
+            conn, primary_id, duplicate_ids, strict=strict
+        )
+        if merged_metadata:
+            self._update_primary_with_merged_metadata(
+                conn, primary_id, merged_metadata, strict=strict
+            )
+        removed = 0
+        # Images the lenient merge could not move go with their prompt.
+        documents = documents_of_rows(conn, "prompt_id", duplicate_ids)
+        for duplicate_id in duplicate_ids:
+            conn.execute("DELETE FROM prompts WHERE id = ?", (duplicate_id,))
+            removed += 1
+            self.logger.debug(f"Removed duplicate prompt {duplicate_id}")
+        release_documents(conn, documents)
+        return removed, moved
+
+    @staticmethod
+    def _missing_link_ids(rows, roots):
+        """Ids of the ``rows`` whose file is gone under a readable source."""
+        readable = readable_roots(roots)
+        return [
+            row[0]
+            for row, _owner, state in link_states(rows, roots, readable)
+            if state == "missing"
+        ]
+
+    # ── previews ──────────────────────────────────────────────────────
+
+    def preview_maintenance(self, operation, roots=()) -> MaintenanceDraft:
+        """The reviewed plan of ``operation``; writes nothing (blocking).
+
+        ``roots`` are the sources missing links are judged under (the health
+        check's: the effective sources plus configured ones that are gone).
+        """
+        if operation not in MAINTENANCE_OPERATIONS:
+            raise ValueError(f"Unknown maintenance operation {operation!r}")
+        roots = [str(root) for root in roots]
+        if operation == "optimize":
+            return self._plan_optimize()
+        if operation == "workflow_storage":
+            return self._plan_workflow_storage()
+        if operation == "missing_links":
+            rows, marks = _read_snapshot(
+                self.model.db_path,
+                lambda conn: (self._link_rows(conn), _mark_rows(conn)),
+            )
+            return self._plan_missing_links(rows, roots, _stale_marks(marks, roots))
+        reader = (
+            self._plan_duplicate_prompts
+            if operation == "duplicate_prompts"
+            else self._plan_unused_components
+        )
+        return _read_snapshot(self.model.db_path, reader)
+
+    @staticmethod
+    def _link_rows(conn):
+        rows = []
+        cursor = conn.execute(_LINK_ROWS_SQL)
+        while batch := cursor.fetchmany(BATCH_ROWS):
+            rows.extend(tuple(row) for row in batch)
+        return rows
+
+    def _duplicates_fingerprint(self, conn):
+        """``(groups, paired, conflicted, fingerprint)`` of the duplicate merge.
+
+        ``conflicted`` holds ``(ids, reason)`` of the groups a merge would
+        lose an image link or a Library title in (``_duplicate_conflict``);
+        the plan protects them.
+        """
+        candidates, paired = self._duplicate_groups(conn)
+        groups, conflicted = [], []
+        for ids in candidates:
+            reason = self._duplicate_conflict(conn, ids)
+            if reason:
+                conflicted.append((ids, reason))
+            else:
+                groups.append(ids)
+        fingerprint = _Fingerprint("duplicate_prompts")
+        fingerprint.add({"groups": groups, "paired": paired, "conflicted": conflicted})
+        _add_prompt_state(fingerprint, conn, [i for group in groups for i in group])
+        return groups, paired, conflicted, fingerprint.hexdigest()
+
+    def _plan_duplicate_prompts(self, conn):
+        groups, paired, conflicted, fingerprint = self._duplicates_fingerprint(conn)
+        removed = sum(len(group) - 1 for group in groups)
+        held = [ids for ids, _reason in conflicted]
+        texts = _texts(conn, [group[0] for group in (groups + paired + held)])
+        details = (
+            [
+                {
+                    "label": _excerpt(texts.get(group[0])),
+                    "detail": f"Keeps the oldest of {len(group)} copies",
+                }
+                for group in groups
+            ]
+            + [
+                {
+                    "label": _excerpt(texts.get(group[0])),
+                    "detail": "Kept: belongs to a saved pair",
+                }
+                for group in paired
+            ]
+            + [
+                {
+                    "label": _excerpt(texts.get(ids[0])),
+                    "detail": CONFLICT_DETAILS[reason],
+                }
+                for ids, reason in conflicted
+            ]
+        )
+        if groups:
+            label = (
+                f"{_plural(removed, 'duplicate prompt')} in "
+                f"{_plural(len(groups), 'group')}"
+            )
+        else:
+            label = "No duplicate prompt text"
+        reasons = []
+        if len(paired) == 1:
+            reasons.append("1 group belongs to a saved pair")
+        elif paired:
+            reasons.append(f"{len(paired):,} groups belong to saved pairs")
+        for reason, (one, many) in CONFLICT_REASONS.items():
+            count = sum(1 for _ids, why in conflicted if why == reason)
+            if count:
+                reasons.append(f"{count:,} {one if count == 1 else many}")
+        return MaintenanceDraft(
+            operation="duplicate_prompts",
+            fingerprint=fingerprint,
+            targets=groups,
+            scope={"count": removed, "label": label},
+            protected={
+                "count": sum(len(group) for group in paired + held),
+                "reasons": reasons,
+            },
+            media={"deletes_files": False, "label": NO_FILES_DELETED},
+            details=details[:MAX_DETAILS],
+        )
+
+    def _unused_fingerprint(self, conn):
+        ids = _orphaned_ids(conn)
+        fingerprint = _Fingerprint("unused_components")
+        fingerprint.add(ids)
+        _add_prompt_state(fingerprint, conn, ids)
+        return ids, fingerprint.hexdigest()
+
+    def _plan_unused_components(self, conn):
+        ids, fingerprint = self._unused_fingerprint(conn)
+        *by_reason, kept = conn.execute(_UNLINKED_PROTECTIONS_SQL).fetchone()
+        reasons = [
+            f"{count:,} {many if count != 1 else one}"
+            for count, (_predicate, many, one) in zip(by_reason, PROTECTIONS)
+            if count
+        ]
+        texts = _texts(conn, ids)
+        count = len(ids)
+        return MaintenanceDraft(
+            operation="unused_components",
+            fingerprint=fingerprint,
+            targets=ids,
+            scope={
+                "count": count,
+                "label": (
+                    _plural(count, "unused prompt component")
+                    if count
+                    else "No unused prompt components"
+                ),
+            },
+            protected={"count": kept, "reasons": reasons},
+            media={"deletes_files": False, "label": NO_FILES_DELETED},
+            details=[
+                {"label": _excerpt(texts.get(prompt_id)), "detail": UNUSED_DETAIL}
+                for prompt_id in ids[:MAX_DETAILS]
+            ],
+        )
+
+    @staticmethod
+    def _links_fingerprint(candidates, stale=()):
+        fingerprint = _Fingerprint("missing_links")
+        for row, owner in candidates:
+            fingerprint.add([row[0], row[1], row[3], owner])
+        for locator, owner in stale:
+            fingerprint.add(["mark", locator, owner])
+        return fingerprint.hexdigest()
+
+    def _plan_missing_links(self, rows, roots, stale=()):
+        readable = readable_roots(roots)
+        candidates, unavailable, outside = [], 0, 0
+        for row, owner, state in link_states(rows, roots, readable):
+            if state == "missing":
+                candidates.append((row, owner))
+            elif state == "unavailable":
+                unavailable += 1
+            elif state == "outside":
+                outside += 1
+        reasons = []
+        if unavailable:
+            reasons.append(
+                _plural(
+                    unavailable,
+                    "link under an unavailable source",
+                    "links under unavailable sources",
+                )
+            )
+        if outside:
+            reasons.append(
+                _plural(
+                    outside,
+                    "link outside the image sources",
+                    "links outside the image sources",
+                )
+            )
+        count = len(candidates)
+        parts = []
+        if count:
+            parts.append(
+                _plural(count, "link to a missing file", "links to missing files")
+            )
+        if stale:
+            parts.append(
+                _plural(
+                    len(stale),
+                    "removed image whose file is gone or changed",
+                    "removed images whose files are gone or changed",
+                )
+            )
+        details = [
+            {
+                "label": _relative(row[1], roots[owner]),
+                "detail": f"{_source_name(roots[owner], owner)} · the file is missing",
+            }
+            for row, owner in candidates
+        ] + [
+            {
+                "label": _relative(locator, os.path.realpath(roots[owner])),
+                "detail": (
+                    f"{_source_name(roots[owner], owner)} · removed image, "
+                    "the file is gone or changed"
+                ),
+            }
+            for locator, owner in stale
+        ]
+        return MaintenanceDraft(
+            operation="missing_links",
+            fingerprint=self._links_fingerprint(candidates, stale),
+            targets=[row[0] for row, _owner in candidates],
+            scope={
+                "count": count + len(stale),
+                "label": "; ".join(parts) or "No links to missing files",
+            },
+            protected={"count": unavailable + outside, "reasons": reasons},
+            media={"deletes_files": False, "label": LINKS_ONLY},
+            details=details[:MAX_DETAILS],
+            marks=[locator for locator, _owner in stale],
+        )
+
+    def _plan_optimize(self):
+        def read(conn):
+            return [
+                conn.execute(f"PRAGMA {name}").fetchone()[0]
+                for name in ("page_size", "page_count", "freelist_count")
+            ]
+
+        page_size, page_count, free_pages = _read_snapshot(self.model.db_path, read)
+        db_path = self.model.db_path
+        wal = db_path + "-wal"
+        size = page_size * page_count
+        wal_size = os.path.getsize(wal) if os.path.exists(wal) else 0
+        free = shutil.disk_usage(os.path.dirname(os.path.abspath(db_path))).free
+        # VACUUM writes the rebuilt database through the WAL beside it.
+        return MaintenanceDraft(
+            operation="optimize",
+            fingerprint=_Fingerprint("optimize").hexdigest(),
+            targets=[],
+            scope={"count": 1, "label": "The whole database"},
+            protected={"count": 0, "reasons": []},
+            media={"deletes_files": False, "label": NO_FILES_DELETED},
+            details=[
+                {
+                    "label": "Database file",
+                    "detail": f"{_megabytes(size)}, plus {_megabytes(wal_size)} "
+                    "not yet checkpointed",
+                },
+                {
+                    "label": "Free pages",
+                    "detail": f"About {_megabytes(free_pages * page_size)} "
+                    "can be reclaimed",
+                },
+                {
+                    "label": "Lock",
+                    "detail": OPTIMIZE_LOCK_NOTE,
+                },
+                {
+                    "label": "Free space needed",
+                    "detail": f"About {_megabytes(size)} beside the database and "
+                    f"{_megabytes(size)} in SQLite's temporary folder; "
+                    f"{_megabytes(free)} available beside the database",
+                },
+            ],
+        )
+
+    # ── apply ─────────────────────────────────────────────────────────
+
+    def apply_maintenance(
+        self, operation, fingerprint, targets, roots=(), marks=()
+    ) -> dict:
+        """Apply a reviewed plan exactly, or raise PlanChanged (blocking).
+
+        Inside ``BEGIN IMMEDIATE`` the fingerprint is computed again on the
+        write connection; when it differs nothing is written. ``targets`` and
+        ``roots`` are the plan's. Returns ``{changed, rows}``: how many
+        records changed and up to ``MAX_RESULT_ROWS`` descriptions of them
+        (``{prompt_id[, kept_id]}`` or ``{image_path}``). ``marks`` are a
+        missing-links plan's stale removal marks; that apply also returns
+        ``marks``, how many it forgot.
+        """
+        if operation not in MAINTENANCE_OPERATIONS or operation == "workflow_storage":
+            raise ValueError(f"Unknown maintenance operation {operation!r}")
+        if operation == "optimize":
+            # VACUUM cannot run in a transaction; the plan binds the database.
+            if fingerprint != _Fingerprint("optimize").hexdigest():
+                raise PlanChanged("optimize")
+            self.model.vacuum_database()
+            return {"changed": 1, "rows": []}
+        appliers = {
+            "duplicate_prompts": self._apply_duplicate_prompts,
+            "unused_components": self._apply_unused_components,
+        }
+        with self._pair_transaction() as conn:
+            if operation == "missing_links":
+                return self._apply_missing_links(
+                    conn, fingerprint, targets, roots, list(marks)
+                )
+            return appliers[operation](conn, fingerprint, targets, roots)
+
+    def _apply_duplicate_prompts(self, conn, fingerprint, targets, _roots):
+        groups, _paired, _conflicted, current = self._duplicates_fingerprint(conn)
+        if current != fingerprint or groups != targets:
+            raise PlanChanged("duplicate_prompts")
+        changed, rows = 0, []
+        for sorted_ids in groups:
+            removed, _moved = self._merge_duplicate_group(conn, sorted_ids, strict=True)
+            changed += removed
+            rows.extend(
+                {"prompt_id": duplicate_id, "kept_id": sorted_ids[0]}
+                for duplicate_id in sorted_ids[1:]
+            )
+        return {"changed": changed, "rows": rows[:MAX_RESULT_ROWS]}
+
+    def _apply_unused_components(self, conn, fingerprint, targets, _roots):
+        ids, current = self._unused_fingerprint(conn)
+        if current != fingerprint or ids != targets:
+            raise PlanChanged("unused_components")
+        texts = _texts(conn, ids, limit=MAX_RESULT_ROWS)
+        changed = _delete_ids(conn, "prompts", ids)
+        rows = [
+            {"prompt_id": prompt_id, "text": _excerpt(texts.get(prompt_id))}
+            for prompt_id in ids[:MAX_RESULT_ROWS]
+        ]
+        return {"changed": changed, "rows": rows}
+
+    def _apply_missing_links(self, conn, fingerprint, targets, roots, marks):
+        rows = []
+        for batch in _batches(sorted(targets)):
+            rows.extend(
+                tuple(row)
+                for row in conn.execute(
+                    "SELECT id, image_path, filename, file_path FROM generated_images "
+                    f"WHERE id IN ({_marks(batch)}) ORDER BY id",  # nosec B608
+                    batch,
+                )
+            )
+        readable = readable_roots(roots)
+        candidates = []
+        for row, owner, state in link_states(rows, roots, readable):
+            if state != "missing":  # back, or its source went away
+                raise PlanChanged("missing_links")
+            candidates.append((row, owner))
+        stale = _stale_marks(_mark_rows(conn, marks) if marks else [], roots)
+        if [locator for locator, _owner in stale] != marks:  # a mark went or is valid
+            raise PlanChanged("missing_links")
+        if self._links_fingerprint(candidates, stale) != fingerprint:
+            raise PlanChanged("missing_links")
+        changed = _delete_ids(conn, "generated_images", [row[0] for row in rows])
+        return {
+            "changed": changed,
+            "marks": clear_marks(conn, marks),
+            "rows": [{"image_path": row[1]} for row in rows[:MAX_RESULT_ROWS]],
+        }

database/managed_connection.py

@@ -0,0 +1,353 @@
+"""A SQLite connection that is never closed under the thread using it.
+
+``PromptModel`` gives each thread its own connection, but Move library and
+``close_all`` must retire every thread's connection from one thread. Closing
+a ``sqlite3.Connection`` while its owner is inside a statement (Python runs
+``sqlite3_step`` without the GIL) frees memory the owner then reads: on
+Windows the process dies with "access violation", elsewhere the owner sees
+garbage such as a ``MemoryError``.
+
+``ManagedConnection`` counts the calls its owner is inside. ``retire()`` is
+safe from any thread and never raises: an idle connection is closed at once;
+one in use is closed by its owner as the call ends. A retired connection
+refuses new work (cursors, statements, commits) with the error a closed
+connection raises.
+
+What retiring a connection in use does and does not do:
+
+- It calls ``sqlite3_interrupt`` (thread-safe). That ends a running
+  statement at its next step, but it does not wake a statement that is
+  waiting for another connection's lock: that one waits until the lock is
+  released (or the busy timeout ends it), and only then is it interrupted.
+  An interrupt that arrives between two statements, for example between the
+  implicit ``BEGIN`` and the DML or between ``executemany`` rows, is lost.
+- Writes are kept out of a file the process has switched away from because
+  a retired connection never commits: legacy-mode DML runs inside a
+  transaction that only ``commit()`` (refused) or ``close()`` (rolls back)
+  ends. A statement that commits by itself is not covered: DDL or a PRAGMA
+  write outside a transaction, or an explicit ``COMMIT`` passed to
+  ``execute``/``executescript`` (``executescript`` also commits first, in C,
+  on Python before 3.12). Managed connections must never be used that way
+  for library writes.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import threading
+import time
+import weakref
+from typing import Iterable, Iterator, List, Optional
+
+import sqlite3
+
+CLOSED_MESSAGE = "Cannot operate on a closed database."
+# Rows fetched per step when a cursor is iterated: the connection is marked
+# in use once per chunk instead of once per row (iteration costs about what
+# a plain cursor costs). A loop that stops early leaves up to this many rows
+# fetched; ``fetchone``/``fetchmany`` after a partial loop skip them.
+ITER_CHUNK = 128
+
+
+class ManagedCursor(sqlite3.Cursor):
+    """A cursor whose calls into SQLite count as its connection being in use."""
+
+    def execute(self, sql, parameters=(), /):
+        conn = self.connection
+        conn._enter()
+        try:
+            return super().execute(sql, parameters)
+        finally:
+            conn._leave()
+
+    def executemany(self, sql, seq_of_parameters, /):
+        conn = self.connection
+        conn._enter()
+        try:
+            return super().executemany(sql, seq_of_parameters)
+        finally:
+            conn._leave()
+
+    def executescript(self, sql_script, /):
+        conn = self.connection
+        conn._enter()
+        try:
+            return super().executescript(sql_script)
+        finally:
+            conn._leave()
+
+    def fetchone(self):
+        conn = self.connection
+        conn._enter()
+        try:
+            return super().fetchone()
+        finally:
+            conn._leave()
+
+    def fetchmany(self, *args, **kwargs):
+        conn = self.connection
+        conn._enter()
+        try:
+            return super().fetchmany(*args, **kwargs)
+        finally:
+            conn._leave()
+
+    def fetchall(self):
+        conn = self.connection
+        conn._enter()
+        try:
+            return super().fetchall()
+        finally:
+            conn._leave()
+
+    def __iter__(self) -> Iterator:
+        return self._chunks()
+
+    def _chunks(self) -> Iterator:
+        while True:
+            rows = self.fetchmany(ITER_CHUNK)
+            if not rows:
+                return
+            yield from rows
+
+    def __next__(self):
+        conn = self.connection
+        conn._enter()
+        try:
+            return super().__next__()
+        finally:
+            conn._leave()
+
+    def close(self):
+        conn = self.connection
+        try:
+            conn._enter()
+        except sqlite3.ProgrammingError:
+            return super().close()  # the connection is closed or closing
+        try:
+            return super().close()
+        finally:
+            conn._leave()
+
+
+class ManagedConnection(sqlite3.Connection):
+    """A connection only its owner closes while a call into it is running."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # Guards _depth, _retired, _closing and _cursors. Re-entrant so that
+        # code running in the closing thread while it holds the lock (a
+        # cursor() during the cursor-set copy) is refused instead of hanging.
+        self._use_lock = threading.RLock()
+        self._depth = 0
+        self._retired = False
+        self._closing = False
+        self._released = threading.Event()
+        # Python before 3.12 keeps the file open after close() while a cursor
+        # still holds a statement; these are closed first (see _close_now).
+        self._cursors: "weakref.WeakSet[sqlite3.Cursor]" = weakref.WeakSet()
+        self.close_logger: Optional[logging.Logger] = None
+
+    # ── in use ───────────────────────────────────────────────────────
+
+    def _enter(self) -> None:
+        with self._use_lock:
+            if self._retired:
+                raise sqlite3.ProgrammingError(CLOSED_MESSAGE)
+            self._depth += 1
+
+    def _leave(self) -> None:
+        with self._use_lock:
+            self._depth -= 1
+            close_now = self._retired and self._depth == 0
+        if close_now:
+            self._close_now()
+
+    @contextlib.contextmanager
+    def using(self):
+        """Mark the connection in use for a call this class cannot wrap.
+
+        For example as the *target* of another connection's ``backup()``.
+        """
+        self._enter()
+        try:
+            yield self
+        finally:
+            self._leave()
+
+    def cursor(self, factory=None):
+        with self._use_lock:
+            if self._retired:
+                raise sqlite3.ProgrammingError(CLOSED_MESSAGE)
+            cursor = super().cursor(factory or ManagedCursor)
+            self._cursors.add(cursor)
+        return cursor
+
+    def execute(self, sql, parameters=(), /):
+        return self.cursor().execute(sql, parameters)
+
+    def executemany(self, sql, seq_of_parameters, /):
+        return self.cursor().executemany(sql, seq_of_parameters)
+
+    def executescript(self, sql_script, /):
+        return self.cursor().executescript(sql_script)
+
+    def commit(self):
+        self._enter()
+        try:
+            return super().commit()
+        finally:
+            self._leave()
+
+    def rollback(self):
+        self._enter()  # a closed connection raises, as sqlite3's does
+        try:
+            return super().rollback()
+        finally:
+            self._leave()
+
+    def backup(self, target, **kwargs):
+        self._enter()
+        try:
+            return super().backup(target, **kwargs)
+        finally:
+            self._leave()
+
+    def create_function(self, *args, **kwargs):
+        self._enter()
+        try:
+            return super().create_function(*args, **kwargs)
+        finally:
+            self._leave()
+
+    @property
+    def in_transaction(self) -> bool:
+        self._enter()
+        try:
+            return super().in_transaction
+        finally:
+            self._leave()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        # The C implementation calls its own commit/rollback, not these.
+        if exc_type is None:
+            try:
+                self.commit()
+            except BaseException:
+                with contextlib.suppress(sqlite3.Error):
+                    self.rollback()
+                raise
+        else:
+            self.rollback()
+        return False
+
+    # ── closing ──────────────────────────────────────────────────────
+
+    def close(self) -> None:
+        """Close now if idle, else as soon as the running call ends.
+
+        Unlike ``sqlite3.Connection.close``, a call from another thread while
+        the owner is inside a call returns before the file is closed. A caller
+        that must know the file is closed (before renaming or deleting it)
+        waits for ``wait_closed`` (``PromptModel.close_detached`` does).
+        """
+        self.retire()
+
+    def retire(self) -> bool:
+        """Refuse new work and close; True when the connection is closed already.
+
+        Safe from any thread, and never raises: failures are logged.
+        """
+        try:
+            with self._use_lock:
+                if not self._retired:
+                    self._retired = True
+                    if self._depth:
+                        # Ends the running statement (see the module docstring
+                        # for what it does not end). The owner cannot start
+                        # closing the connection while this lock is held.
+                        self.interrupt()
+                        return False
+                elif self._depth:
+                    return False
+                else:
+                    return self._released.wait(0)
+        except Exception:
+            self._log_exception("Error retiring database connection")
+            return self._released.wait(0)
+        self._close_now()
+        return True
+
+    def wait_closed(self, timeout: float) -> bool:
+        return self._released.wait(timeout)
+
+    def _close_now(self) -> None:
+        # Only ever called with no call into the connection running, and a
+        # retired connection refuses new calls and cursors: its cursors are
+        # idle, and the set is copied under the lock that guards it.
+        with self._use_lock:
+            if self._closing:
+                return  # another thread closes it and sets _released
+            self._closing = True
+            cursors = list(self._cursors)
+        try:
+            for cursor in cursors:
+                with contextlib.suppress(sqlite3.Error):
+                    sqlite3.Cursor.close(cursor)
+            super().close()
+        except Exception:
+            self._log_exception("Error closing database connection")
+        finally:
+            self._released.set()
+
+    def _log_exception(self, message: str) -> None:
+        if self.close_logger is not None:
+            self.close_logger.exception(message)
+
+
+def retire_connections(
+    connections: Iterable[sqlite3.Connection], logger=None
+) -> List[ManagedConnection]:
+    """Retire every connection; return those whose owner has yet to close them.
+
+    Safe while holding a lock the owners wait on: nothing here waits for them.
+    One connection failing to retire is logged and never skips the rest.
+    """
+    pending = []
+    for conn in connections:
+        try:
+            if isinstance(conn, ManagedConnection):
+                if logger is not None:
+                    conn.close_logger = logger
+                if not conn.retire():
+                    pending.append(conn)
+            else:
+                conn.close()
+        except Exception:
+            if logger is not None:
+                logger.exception("Error closing database connection")
+    return pending
+
+
+def await_closed(
+    connections: Iterable[ManagedConnection], timeout: float, logger=None
+) -> bool:
+    """Wait until the owners closed ``connections`` (retired); False on timeout.
+
+    A connection still open after ``timeout`` (its owner is blocked, for
+    example on another process's lock after a lost interrupt) is logged and
+    left to close when its call ends; a Move library then reports the old
+    files as still in use instead of renaming them.
+    """
+    deadline = time.monotonic() + timeout
+    still_open = 0
+    for conn in connections:
+        if not conn.wait_closed(max(0.0, deadline - time.monotonic())):
+            still_open += 1
+    if still_open and logger is not None:
+        logger.warning(
+            f"{still_open} database connection(s) still in use after "
+            f"{timeout:g}s; they close when their statement ends"
+        )
+    return not still_open

database/models.py

@@ -9,20 +9,53 @@ import math
 import sqlite3
 import os
 import threading
-import tempfile
 import weakref
 from contextlib import closing
 from pathlib import Path
 from typing import Optional, Set, Tuple
 
+from .managed_connection import (
+    ManagedConnection,
+    await_closed,
+    retire_connections,
+)
+from . import restore_stage
 from .library_schema import library_schema_error, migrate_library_state
 from .image_favorites import image_favorites_schema_error, migrate_image_favorites
+from .image_collections import (
+    image_collections_schema_error,
+    migrate_image_collections,
+)
 from .annotation_schema import annotation_schema_error, migrate_annotation_state
+from .autotag_schema import (
+    autotag_schema_error,
+    autotag_without_annotations_error,
+    migrate_autotag_state,
+)
+from .review_schema import (
+    migrate_review_state,
+    review_schema_error,
+    review_without_annotations_error,
+)
 from .saved_views import migrate_saved_views, saved_views_schema_error
+
+try:
+    from ..utils.hashing import register_prompt_functions
+except ImportError:
+    from utils.hashing import register_prompt_functions
+from .image_categories import image_categories_schema_error, migrate_image_categories
+from .image_removal_schema import image_removals_schema_error, migrate_image_removals
+from .prompt_archive import migrate_prompt_archive, prompt_archive_schema_error
+from .workflow_documents import (
+    DOCUMENT_ID_COLUMNS,
+    migrate_workflow_documents,
+    workflow_documents_schema_error,
+)
+
+
 from .execution_history_schema import (
     history_schema_error,
     migrate_execution_history,
-    rotate_history_epoch,
 )
 
 # Import logging system
@@ -45,6 +78,19 @@ MAX_PAIR_PROVENANCE_LENGTH = 4_000_000
 MAX_PAIR_PROVENANCE_DEPTH = 64
 
 
+def _refuse_rebuild_with_documents(columns) -> None:
+    """The legacy generated_images rebuilds copy a fixed column list.
+
+    A file that needs one (``prompt_id`` TEXT, no ``file_path``) but already
+    has the workflow document ids would lose them, so it is refused.
+    """
+    if any(column in columns for column in DOCUMENT_ID_COLUMNS):
+        raise sqlite3.DatabaseError(
+            "This database needs an old image table rebuild but already has "
+            "workflow document references; the rebuild would drop them"
+        )
+
+
 def pair_provenance_error(encoded: object) -> Optional[str]:
     """Validate the same bounded JSON object for live writes and restores."""
     if not isinstance(encoded, str):
@@ -114,12 +160,15 @@ def configure_connection(conn: sqlite3.Connection) -> None:
     (common on Windows) let other writers' polling busy handlers starve past
     BUSY_TIMEOUT_SECONDS and fail with "database is locked" (#242). Without
     WAL (for example an in-memory database) SQLite's FULL default is kept.
+    It also registers the prompt-key SQL functions (register_prompt_functions).
     """
     journal_mode = conn.execute("PRAGMA journal_mode = WAL").fetchone()[0]
     conn.execute("PRAGMA foreign_keys = ON")
     conn.execute(f"PRAGMA busy_timeout = {BUSY_TIMEOUT_SECONDS * 1000}")
     if str(journal_mode).lower() == "wal":
         conn.execute("PRAGMA synchronous = NORMAL")
+    # History reads and the live-hint rule call Python's prompt key (#460)
+    register_prompt_functions(conn)
 
 
 class PromptModel:
@@ -303,7 +352,28 @@ class PromptModel:
         migrate_library_state(conn)
         migrate_image_favorites(conn)
         migrate_saved_views(conn)
+        migrate_image_collections(conn)
+        # A review group without its annotation group is refused before the
+        # annotation install could make it look valid (catalog only; the rows
+        # are validated once, in migrate_review_state).
+        review_error = review_without_annotations_error(conn)
+        if review_error:
+            raise sqlite3.DatabaseError(review_error)
+        # The auto-tag marks cite annotation bytes the same way (#379).
+        autotag_error = autotag_without_annotations_error(conn)
+        if autotag_error:
+            raise sqlite3.DatabaseError(autotag_error)
         migrate_annotation_state(conn)
+        migrate_review_state(conn)
+        migrate_autotag_state(conn)
+        # Image categories (#332): independent of every table above.
+        migrate_image_categories(conn)
+        # The Library parent archive (#443): independent of every table above.
+        migrate_prompt_archive(conn)
+        # Removal marks (#480): independent of every table above.
+        migrate_image_removals(conn)
+        # Last: the older generated_images rebuilds run only before file_path.
+        migrate_workflow_documents(conn)
 
     def _migrate_prompt_pairs(self, conn: sqlite3.Connection) -> None:
         conn.execute("SAVEPOINT prompt_pair_schema")
@@ -516,7 +586,10 @@ class PromptModel:
             "WHERE pp.positive_prompt_id IS NOT gi.prompt_id LIMIT 1"
         ).fetchone():
             return "Image pair association has a different positive prompt owner"
-        for row in conn.execute("SELECT provenance FROM image_prompt_pairs"):
+        # fetchone, not iteration: a managed connection's cursor prefetches
+        # ITER_CHUNK rows when iterated, and each provenance may be ~4 MB.
+        cursor = conn.execute("SELECT provenance FROM image_prompt_pairs")
+        while (row := cursor.fetchone()) is not None:
             error = pair_provenance_error(row[0])
             if error:
                 return error
@@ -575,19 +648,42 @@ class PromptModel:
         if conn is not None and self._local.generation == self._generation:
             return conn
 
-        # check_same_thread=False only so close_all can clean up other
-        # threads' connections; each connection is still used by one thread.
-        conn = sqlite3.connect(
-            self.db_path, check_same_thread=False, timeout=BUSY_TIMEOUT_SECONDS
-        )
-        conn.row_factory = sqlite3.Row
-        configure_connection(conn)
+        while True:
+            # The path and generation are read together, and the new
+            # connection is kept only if no Move library repointed the model
+            # meanwhile: otherwise it would write to the old file.
+            with self._conn_lock:
+                path, generation = self.db_path, self._generation
+            # check_same_thread=False only so close_all and Move library can
+            # retire other threads' connections; each connection is still
+            # used by one thread, and ManagedConnection never closes it under
+            # a running statement (see managed_connection).
+            conn = sqlite3.connect(
+                path,
+                check_same_thread=False,
+                timeout=BUSY_TIMEOUT_SECONDS,
+                factory=ManagedConnection,
+            )
+            try:
+                conn.row_factory = sqlite3.Row
+                configure_connection(conn)
+            except BaseException:
+                conn.close()
+                raise
+            with self._conn_lock:
+                if generation == self._generation:
+                    self._connections.add(conn)
+                    break
+            conn.close()
         self._local.conn = conn
-        self._local.generation = self._generation
-        with self._conn_lock:
-            self._connections.add(conn)
+        self._local.generation = generation
         return conn
 
+    @property
+    def generation(self) -> int:
+        """Bumped whenever every connection is closed or the model repointed."""
+        return self._generation
+
     @classmethod
     def close_all_instances(cls) -> None:
         """Close every connection of every live model in this process."""
@@ -596,22 +692,76 @@ class PromptModel:
         for model in models:
             model.close_all()
 
+    @classmethod
+    def repoint_instances(cls, old_path: str, new_path: str) -> list:
+        """Point every model of ``old_path`` at ``new_path`` (Move library).
+
+        Every thread opens the new file on its next ``get_connection``. The
+        old connections are detached, not closed, and returned: the caller
+        retires them (``retire_detached``) while it still holds the old
+        file's write lock. A write waiting on that lock is then interrupted
+        and can never commit, so it fails (and, through
+        ``PromptDatabase.survive_library_switch``, runs again on the new file).
+        """
+        old_key = os.path.normcase(os.path.realpath(old_path))
+        with cls._instances_lock:
+            models = list(cls._instances)
+        with cls._init_lock:
+            cls._initialized_paths.add(os.path.normcase(os.path.realpath(new_path)))
+        detached = []
+        for model in models:
+            if model._schema_key() == old_key:
+                detached.extend(model._detach_connections(new_path))
+        return detached
+
+    @staticmethod
+    def retire_detached(connections, logger=None) -> list:
+        """Retire detached connections without waiting for their owners.
+
+        Idle ones close now. One whose owner thread is inside a statement is
+        never closed from here (that frees memory the owner is using: an
+        access violation on Windows); it is interrupted, refuses to commit and
+        is closed by its owner as the statement ends. Returns those still
+        open, for ``await_detached`` once no lock the owners wait on is held.
+        """
+        return retire_connections(connections, logger)
+
+    @staticmethod
+    def await_detached(connections, logger=None) -> bool:
+        """Wait until retired connections are closed; False if some are not."""
+        return await_closed(connections, BUSY_TIMEOUT_SECONDS, logger)
+
+    @staticmethod
+    def close_detached(connections, logger=None) -> bool:
+        """Retire connections ``repoint_instances`` detached and await them.
+
+        Do not call it while holding a lock their owners may wait on; retire
+        them there (``retire_detached``) and await them after releasing it.
+        """
+        pending = retire_connections(connections, logger)
+        return await_closed(pending, BUSY_TIMEOUT_SECONDS, logger)
+
     def close_all(self) -> None:
         """Close every thread's connection; they reopen lazily on next use.
 
         Needed before deleting the database file (Windows keeps open files
         locked) and by tests that ran queries through executor threads.
         """
+        self.close_detached(self._detach_connections(), self.logger)
+
+    def _detach_connections(self, new_path: Optional[str] = None) -> list:
+        """Forget every connection (optionally pointing the model elsewhere).
+
+        Returns them for the caller to close; threads reopen on next use.
+        """
         with self._conn_lock:
+            if new_path is not None:
+                self.db_path = new_path
             connections = list(self._connections)
             self._connections.clear()
             self._generation += 1
         self._local.conn = None
-        for conn in connections:
-            try:
-                conn.close()
-            except sqlite3.Error as e:
-                self.logger.warning(f"Error closing database connection: {e}")
+        return connections
 
     def close(self) -> None:
         """Close the calling thread's connection, if it has one.
@@ -625,10 +775,7 @@ class PromptModel:
         self._local.conn = None
         with self._conn_lock:
             self._connections.discard(conn)
-        try:
-            conn.close()
-        except sqlite3.Error as e:
-            self.logger.warning(f"Error closing database connection: {e}")
+        self.close_detached([conn], self.logger)
 
     def _migrate_workflow_name_removal(self, conn: sqlite3.Connection) -> None:
         """
@@ -699,6 +846,7 @@ class PromptModel:
             columns = {column[1]: column[2] for column in cursor.fetchall()}
             if columns.get("prompt_id") != "TEXT":
                 return
+            _refuse_rebuild_with_documents(columns)
 
             self.logger.info("Migrating foreign key types: prompt_id TEXT -> INTEGER")
             with self._table_rebuild(conn):
@@ -864,6 +1012,7 @@ class PromptModel:
             }
             if "file_path" in columns:
                 return
+            _refuse_rebuild_with_documents(columns)
 
             self.logger.info("Migrating database: image uniqueness by full path")
             with self._table_rebuild(conn):
@@ -929,6 +1078,10 @@ class PromptModel:
 
         Reclaims unused space and defragments the database file,
         improving query performance and reducing file size.
+
+        Raises:
+            Exception: whatever stopped VACUUM (logged first); the database is
+            then unchanged.
         """
         try:
             with closing(sqlite3.connect(self.db_path)) as conn:
@@ -936,6 +1089,7 @@ class PromptModel:
                 conn.commit()
         except Exception as e:
             self.logger.error(f"Error vacuuming database: {e}")
+            raise
 
     def get_database_info(self) -> dict:
         """
@@ -1012,7 +1166,7 @@ class PromptModel:
 
     @staticmethod
     def verify_database_file(
-        path: str, *, require_library_state: bool = False
+        path: str, *, require_library_state: bool = False, documents: bool = True
     ) -> Tuple[bool, str]:
         """
         Check that a file is a healthy prompts database before restoring it.
@@ -1023,9 +1177,13 @@ class PromptModel:
         Args:
             path: File to inspect
             require_library_state: Also require the tables every current
-                database has (Library state, image favorites, saved views
-                and image annotations), as a migrated restore stage must;
-                old backups may lack them.
+                database has (Library state, image favorites, image
+                categories, saved views, collections, image annotations and
+                the review queue), as a migrated
+                restore stage must; old backups may lack them.
+            documents: Read every workflow document and refuse a damaged
+                one. A restore checks its upload without it and repairs
+                damaged documents on the stage (``database/restore_stage.py``).
 
         Returns:
             (ok, reason): reason is "ok" when the file is acceptable, otherwise
@@ -1085,16 +1243,51 @@ class PromptModel:
             )
             if favorites_error:
                 return False, favorites_error
+            categories_error = image_categories_schema_error(
+                conn, allow_missing=not require_library_state
+            )
+            if categories_error:
+                return False, categories_error
+            archive_error = prompt_archive_schema_error(
+                conn, allow_missing=not require_library_state
+            )
+            if archive_error:
+                return False, archive_error
+            removals_error = image_removals_schema_error(
+                conn, allow_missing=not require_library_state
+            )
+            if removals_error:
+                return False, removals_error
             views_error = saved_views_schema_error(
                 conn, allow_missing=not require_library_state
             )
             if views_error:
                 return False, views_error
+            collections_error = image_collections_schema_error(
+                conn, allow_missing=not require_library_state
+            )
+            if collections_error:
+                return False, collections_error
             annotation_error = annotation_schema_error(
                 conn, allow_missing=not require_library_state
             )
             if annotation_error:
                 return False, annotation_error
+            review_error = review_schema_error(
+                conn, allow_missing=not require_library_state
+            )
+            if review_error:
+                return False, review_error
+            autotag_error = autotag_schema_error(
+                conn, allow_missing=not require_library_state
+            )
+            if autotag_error:
+                return False, autotag_error
+            documents_error = workflow_documents_schema_error(
+                conn, allow_missing=not require_library_state, documents=documents
+            )
+            if documents_error:
+                return False, documents_error
             return True, "ok"
         except sqlite3.Error as e:
             return False, f"Invalid SQLite database: {e}"
@@ -1117,21 +1310,51 @@ class PromptModel:
             Path(src_path).resolve().as_uri() + "?mode=ro", uri=True
         )
         try:
-            source.backup(self.get_connection())
+            target = self.get_connection()
+            # The target is written under backup's step too: mark it in use
+            # so a concurrent close_all retires it instead of closing it.
+            with target.using():
+                source.backup(target)
         finally:
             source.close()
 
+    def stage_restore(self, src_path: str):
+        """Stage ``src_path`` beside this database for a restore (blocking).
+
+        Returns a ``StagedRestore`` (``database/restore_stage.py``): a
+        migrated, repaired and verified copy in the library's folder, its
+        digest and what it holds. Neither the live database nor the source
+        is touched. Raises ValueError when either does not verify.
+        """
+        return restore_stage.stage_restore(PromptModel, self.db_path, src_path)
+
+    def publish_staged(self, staged) -> str:
+        """Replace the live database with ``staged``; return the safety backup.
+
+        See ``restore_stage.publish_staged``: the staged bytes are rechecked,
+        the current database is kept as ``<db_path>.backup_<stamp>`` and the
+        WAL is truncated afterwards.
+        """
+        backup_path = restore_stage.publish_staged(self, staged)
+        self.logger.info("Database restored from a staged copy")
+        from .execution_history import notify_history_restore
+
+        notify_history_restore(self.db_path)
+        return backup_path
+
     def restore_from_file(self, src_path: str) -> str:
         """
         Replace the live database with the file at src_path.
 
-        A disposable SQLite backup of the source is migrated and verified first.
-        Only then is the current database backed up next to itself as
-        ``<db_path>.backup_<YYYYmmdd_HHMMSS>`` (WAL content included), and the
-        staged pages replace the live pages through SQLite's backup API.
-        Migration failures never replace live content or modify the source.
-        An online replacement failure leaves the live database intact and
-        propagates; the completed safety backup remains available.
+        ``stage_restore`` then ``publish_staged``: a disposable copy of the
+        source is migrated, repaired and verified in the library's folder
+        first. Only then is the current database backed up next to itself as
+        ``<db_path>.backup_<YYYYmmdd_HHMMSS_ffffff>`` (WAL content included),
+        and the staged pages replace the live pages through SQLite's backup
+        API. Staging failures never replace live content or modify the
+        source. An online replacement failure leaves the live database intact
+        and propagates; the completed safety backup remains available. The
+        stage is removed either way.
 
         Args:
             src_path: SQLite file to restore
@@ -1141,55 +1364,14 @@ class PromptModel:
             there was no database to back up.
 
         Raises:
-            RuntimeError: If the previous database could not be backed up
+            ValueError: If the source or its migrated copy does not verify
+            SafetyBackupFailed: If the previous database could not be backed up
             sqlite3.OperationalError: If SQLite refuses online replacement
         """
-        ok, reason = self.verify_database_file(src_path)
-        if not ok:
-            raise ValueError(reason)
-        with tempfile.TemporaryDirectory(prefix="prompt-manager-restore-") as directory:
-            staged_path = str(Path(directory) / "staged.db")
-            source = sqlite3.connect(
-                Path(src_path).resolve().as_uri() + "?mode=ro", uri=True
-            )
-            try:
-                with closing(sqlite3.connect(staged_path)) as destination:
-                    source.backup(destination)
-            finally:
-                source.close()
-            staged = None
-            try:
-                staged = PromptModel(staged_path)
-                rotate_history_epoch(staged.get_connection())
-                ok, reason = self.verify_database_file(
-                    staged_path, require_library_state=True
-                )
-                if not ok:
-                    raise ValueError(reason)
-                staged.close_all()
-                backup_path = ""
-                if os.path.exists(self.db_path):
-                    timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
-                    backup_path = f"{self.db_path}.backup_{timestamp}"
-                    if not self.backup_database(backup_path):
-                        raise RuntimeError("Could not back up the current database")
-                    self.logger.info(f"Current database backed up to: {backup_path}")
-                try:
-                    self._replace_contents_from(staged_path)
-                except sqlite3.OperationalError as exc:
-                    exc._prompt_manager_restore_backup_created = bool(backup_path)
-                    raise
-                with PromptModel._init_lock:
-                    PromptModel._initialized_paths.add(self._schema_key())
-                self.logger.info(f"Database restored from {src_path}")
-                from .execution_history import notify_history_restore
-
-                notify_history_restore(self.db_path)
-                return backup_path
-            finally:
-                if staged is not None:
-                    staged.close_all()
-                with PromptModel._init_lock:
-                    PromptModel._initialized_paths.discard(
-                        os.path.normcase(os.path.realpath(staged_path))
-                    )
+        staged = self.stage_restore(src_path)
+        try:
+            backup_path = self.publish_staged(staged)
+        finally:
+            staged.discard()
+        self.logger.info(f"Database restored from {src_path}")
+        return backup_path

database/pair_ingestion.py

@@ -1,136 +1,308 @@
-"""Atomic metadata enrichment for a single known generation context."""
+"""Atomic metadata enrichment for a single known generation context.
+
+Also the helpers both ingestion store paths share (#247): the file's link
+rows and the one-prompt-per-file rule (``file_link_rows``,
+``link_conflict``) and the marks a prompt created by ingestion carries
+(``Marks``, ``mark_created``).
+"""
 
-import json
 import os
-from typing import Any, Dict
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Tuple
 
 from .models import normalize_image_path
 from .operations import _coerce_file_info
+from .workflow_documents import (
+    TEXT_COLUMNS,
+    DocumentReader,
+    release_documents,
+    resolve_image_row,
+    stored_values,
+)
 
 try:
     from ..utils.hashing import generate_prompt_hash
+    from ..utils.json_strict import dumps_strict
+    from ..utils.prompt_pair_resolver import RESOLVER_VERSION
 except ImportError:
     from utils.hashing import generate_prompt_hash
+    from utils.json_strict import dumps_strict
+    from utils.prompt_pair_resolver import RESOLVER_VERSION
+
+
+def blank_positive(resolution: Dict[str, Any]) -> bool:
+    """A resolved pair whose stored positive text is empty (#342).
+
+    A PromptManager node whose editor is empty encodes only its prepend and
+    append text, which is never stored as a prompt: such a result names no
+    prompt to link.
+    """
+    if resolution.get("state") != "resolved":
+        return False
+    positive = resolution.get("positive") or {}
+    text = positive.get("storage_text", positive.get("text"))
+    return not (isinstance(text, str) and text.strip())
+
+
+@dataclass(frozen=True)
+class Marks:
+    """What a prompt ingestion creates carries, so it can be found and removed.
+
+    ``note`` is formatted with ``name``, the file's name.
+    """
+
+    category: str
+    tags: Tuple[str, ...]
+    note: str
+
+
+SCAN_MARKS = Marks("scanned", ("auto-scanned",), "Auto-scanned from {name}")
+RUN_MARKS = Marks("scanned", ("auto-scanned",), "Saved from a workflow run: {name}")
+
+
+def mark_created(conn: Any, db: Any, prompt_id: int, marks: Marks, name: str) -> None:
+    """Apply ``marks`` to a prompt row the caller just created on ``conn``."""
+    conn.execute(
+        "UPDATE prompts SET category = ?, notes = ? WHERE id = ?",
+        (marks.category, marks.note.format(name=name), prompt_id),
+    )
+    db._sync_prompt_tags(conn, prompt_id, list(marks.tags))
+
+
+def file_link_rows(conn: Any, image_path: str) -> List[Any]:
+    """The file's link rows, each with its prompt's hash.
+
+    Matched under the path and its resolved spelling (an output folder
+    reached through a symlink or junction), each normalized and raw;
+    deduplicated by row id.
+    """
+    rows: Dict[Any, Any] = {}
+    for spelling in dict.fromkeys((image_path, _resolved(image_path))):
+        for row in conn.execute(
+            "SELECT gi.*,p.hash AS positive_hash FROM generated_images gi "
+            "JOIN prompts p ON p.id=gi.prompt_id "
+            "WHERE gi.file_path=? OR gi.image_path=?",
+            (normalize_image_path(spelling), spelling),
+        ).fetchall():
+            rows.setdefault(row["id"], row)
+    return list(rows.values())
+
+
+def _resolved(path: str) -> str:
+    try:
+        return os.path.realpath(path)
+    except (OSError, ValueError):
+        return path
+
+
+def link_conflict(rows: List[Any], prompt_hash: str) -> bool:
+    """One prompt per file: more than one row, or a row of another prompt."""
+    return len(rows) > 1 or bool(rows and rows[0]["positive_hash"] != prompt_hash)
 
 
 def ingest_resolved_pair(
-    db: Any, image_path: str, metadata: Dict[str, Any], resolution: Dict[str, Any]
+    db: Any,
+    image_path: str,
+    metadata: Dict[str, Any],
+    resolution: Dict[str, Any],
+    *,
+    marks: Optional[Marks] = None,
 ) -> Dict[str, Any]:
+    """Store a resolved pair and link the file to it in one transaction.
+
+    With ``marks`` (ingestion from files), each pair component this call
+    creates carries them; without (the PromptManager node path) components
+    are created bare, as always.
+    """
     state = resolution.get("state", "unresolved")
+    if state == "resolved" and blank_positive(resolution):
+        state = "unresolved"
     if state != "resolved":
         return {"status": state, "pairs_added": 0, "links_added": 0}
+    with db._pair_transaction() as conn:
+        return _ingest_resolved_pair_in(
+            conn, db, image_path, metadata, resolution, marks=marks
+        )
+
+
+def _conflict() -> Dict[str, Any]:
+    return {"status": "conflict", "pairs_added": 0, "links_added": 0}
+
+
+def _ingest_resolved_pair_in(
+    conn: Any,
+    db: Any,
+    image_path: str,
+    metadata: Dict[str, Any],
+    resolution: Dict[str, Any],
+    *,
+    provenance: Dict[str, Any] | None = None,
+    marks: Optional[Marks] = None,
+) -> Dict[str, Any]:
+    """``ingest_resolved_pair`` inside the caller's open ``_pair_transaction``.
+
+    The compatible-only rules are the public ingester's: an image with more
+    than one exact row, another positive, or a prior pair with another
+    negative is a ``conflict`` and nothing is written. The caller owns the
+    transaction, so its own later failure rolls the link back too.
+    ``provenance`` replaces the link's default resolver provenance (Needs
+    review records the user's selection there). ``marks`` as in
+    ``ingest_resolved_pair``.
+    """
     positive, negative = resolution["positive"], resolution["negative"]
     positive_text = positive.get("storage_text", positive["text"])
     negative_text = negative.get("storage_text", negative["text"])
     normalized = normalize_image_path(image_path)
-    provenance = {"resolver_version": 1, "positive": positive, "negative": negative}
-    encoded = json.dumps(
-        provenance, sort_keys=True, separators=(",", ":"), allow_nan=False
-    )
-    with db._pair_transaction() as conn:
-        images = conn.execute(
-            "SELECT gi.*,p.hash AS positive_hash FROM generated_images gi "
-            "JOIN prompts p ON p.id=gi.prompt_id "
-            "WHERE gi.file_path=? OR gi.image_path=?",
-            (normalized, image_path),
+    if provenance is None:
+        provenance = {
+            "resolver_version": RESOLVER_VERSION,
+            "positive": positive,
+            "negative": negative,
+        }
+    encoded = dumps_strict(provenance, sort_keys=True, separators=(",", ":"))
+    images = file_link_rows(conn, image_path)
+    if link_conflict(images, generate_prompt_hash(positive_text)):
+        return _conflict()
+    existing = images[0] if images else None
+    if existing:
+        prior = conn.execute(
+            "SELECT n.hash AS negative_hash FROM image_prompt_pairs ip "
+            "JOIN prompt_pairs pp ON pp.id=ip.pair_id "
+            "LEFT JOIN prompts n ON n.id=pp.negative_prompt_id WHERE ip.image_id=?",
+            (existing["id"],),
         ).fetchall()
-        if len(images) > 1 or (
-            images and images[0]["positive_hash"] != generate_prompt_hash(positive_text)
-        ):
-            return {"status": "conflict", "pairs_added": 0, "links_added": 0}
-        existing = images[0] if images else None
-        if existing:
-            prior = conn.execute(
-                "SELECT n.hash AS negative_hash FROM image_prompt_pairs ip "
-                "JOIN prompt_pairs pp ON pp.id=ip.pair_id "
-                "LEFT JOIN prompts n ON n.id=pp.negative_prompt_id WHERE ip.image_id=?",
-                (existing["id"],),
-            ).fetchall()
-            expected = (
-                generate_prompt_hash(negative_text) if negative_text.strip() else None
-            )
-            if any(row["negative_hash"] != expected for row in prior):
-                return {"status": "conflict", "pairs_added": 0, "links_added": 0}
-        prior_pair = conn.execute(
-            "SELECT pp.id FROM prompt_pairs pp "
-            "JOIN prompts p ON p.id=pp.positive_prompt_id "
-            "LEFT JOIN prompts n ON n.id=pp.negative_prompt_id "
-            "WHERE p.hash=? AND n.hash IS ?",
-            (
-                generate_prompt_hash(positive_text),
-                generate_prompt_hash(negative_text) if negative_text.strip() else None,
-            ),
-        ).fetchone()
+        expected = (
+            generate_prompt_hash(negative_text) if negative_text.strip() else None
+        )
+        if any(row["negative_hash"] != expected for row in prior):
+            return _conflict()
+    prior_pair = conn.execute(
+        "SELECT pp.id FROM prompt_pairs pp "
+        "JOIN prompts p ON p.id=pp.positive_prompt_id "
+        "LEFT JOIN prompts n ON n.id=pp.negative_prompt_id "
+        "WHERE p.hash=? AND n.hash IS ?",
+        (
+            generate_prompt_hash(positive_text),
+            generate_prompt_hash(negative_text) if negative_text.strip() else None,
+        ),
+    ).fetchone()
+    if marks is None:
         pair_id = db._save_pair(conn, positive_text, negative_text)
-        pair = conn.execute(
-            "SELECT * FROM prompt_pairs WHERE id=?", (pair_id,)
-        ).fetchone()
-        pairs_added = int(prior_pair is None)
-        file_size, width, height, image_format = _coerce_file_info(metadata)
-        snapshots = {
-            column: json.dumps(metadata[key], sort_keys=True, allow_nan=False)
-            for column, key in (
-                ("workflow_data", "workflow"),
-                ("prompt_metadata", "prompt"),
-                ("parameters", "parameters"),
-            )
-            if key in metadata
-        }
-        info = {
-            "file_size": file_size,
-            "width": width,
-            "height": height,
-            "format": image_format,
-        }
-        snapshots.update(
-            {key: value for key, value in info.items() if value is not None}
+    else:
+        pair_id = _save_marked_pair(
+            conn, db, positive_text, negative_text, marks, os.path.basename(image_path)
         )
-        changed = False
-        if existing:
-            image_id = existing["id"]
-            updates = {
-                key: value for key, value in snapshots.items() if existing[key] != value
-            }
-            if updates:
-                conn.execute(
-                    "UPDATE generated_images SET "
-                    + ",".join(f"{key}=?" for key in updates)
-                    + " WHERE id=?",
-                    [*updates.values(), image_id],
-                )
-                changed = True
-        else:
-            columns = ["prompt_id", "image_path", "filename", "file_path", *snapshots]
-            values = [
-                pair["positive_prompt_id"],
-                image_path,
-                os.path.basename(image_path),
-                normalized,
-                *snapshots.values(),
+    pair = conn.execute("SELECT * FROM prompt_pairs WHERE id=?", (pair_id,)).fetchone()
+    pairs_added = int(prior_pair is None)
+    file_size, width, height, image_format = _coerce_file_info(metadata)
+    # ComfyUI's NaN/Infinity are stored as null: the snapshot is strict JSON.
+    snapshots = {
+        column: dumps_strict(metadata[key], sort_keys=True)
+        for column, key in (
+            ("workflow_data", "workflow"),
+            ("prompt_metadata", "prompt"),
+            ("parameters", "parameters"),
+        )
+        if key in metadata
+    }
+    info = {
+        "file_size": file_size,
+        "width": width,
+        "height": height,
+        "format": image_format,
+    }
+    snapshots.update({key: value for key, value in info.items() if value is not None})
+    changed = False
+    if existing:
+        image_id = existing["id"]
+        # Compared as stored text, whichever form the row is in.
+        current = resolve_image_row(DocumentReader(conn), existing)
+        updates = {
+            key: value for key, value in snapshots.items() if current[key] != value
+        }
+        if updates:
+            replaced = [
+                existing[reference]
+                for column, reference in TEXT_COLUMNS
+                if column in updates
             ]
-            image_id = conn.execute(
-                "INSERT INTO generated_images("
-                + ",".join(columns)
-                + ") VALUES("
-                + ",".join("?" for _ in columns)
-                + ")",
-                values,
-            ).lastrowid
-        linked = conn.execute(
-            "SELECT 1 FROM image_prompt_pairs WHERE pair_id=? "
-            "AND image_id=? AND provenance=?",
-            (pair_id, image_id, encoded),
-        ).fetchone()
-        db._link_image_to_pair(conn, pair_id, image_id, provenance)
-        status = (
-            "added"
-            if existing is None
-            else "enriched" if changed or not linked else "unchanged"
-        )
-        return {
-            "status": status,
-            "pairs_added": pairs_added,
-            "links_added": int(not linked),
-            "pair_id": pair_id,
-            "image_id": image_id,
-        }
+            updates = _document_form(conn, updates)
+            conn.execute(
+                "UPDATE generated_images SET "
+                + ",".join(f"{key}=?" for key in updates)
+                + " WHERE id=?",
+                [*updates.values(), image_id],
+            )
+            release_documents(conn, replaced)
+            changed = True
+    else:
+        stored = _document_form(conn, snapshots)
+        columns = ["prompt_id", "image_path", "filename", "file_path", *stored]
+        values = [
+            pair["positive_prompt_id"],
+            image_path,
+            os.path.basename(image_path),
+            normalized,
+            *stored.values(),
+        ]
+        image_id = conn.execute(
+            "INSERT INTO generated_images("
+            + ",".join(columns)
+            + ") VALUES("
+            + ",".join("?" for _ in columns)
+            + ")",
+            values,
+        ).lastrowid
+    linked = conn.execute(
+        "SELECT 1 FROM image_prompt_pairs WHERE pair_id=? "
+        "AND image_id=? AND provenance=?",
+        (pair_id, image_id, encoded),
+    ).fetchone()
+    db._link_image_to_pair(conn, pair_id, image_id, provenance)
+    status = (
+        "added"
+        if existing is None
+        else "enriched" if changed or not linked else "unchanged"
+    )
+    return {
+        "status": status,
+        "pairs_added": pairs_added,
+        "links_added": int(not linked),
+        "pair_id": pair_id,
+        "image_id": image_id,
+    }
+
+
+def _save_marked_pair(
+    conn: Any,
+    db: Any,
+    positive_text: str,
+    negative_text: str,
+    marks: Marks,
+    name: str,
+) -> int:
+    """``_save_pair`` that marks each component it creates."""
+    texts = [positive_text] + ([negative_text] if negative_text.strip() else [])
+    ids = []
+    for text in texts:
+        prompt_id, created = db._pair_component_outcome(conn, text)
+        if created:
+            mark_created(conn, db, prompt_id, marks, name)
+        ids.append(prompt_id)
+    negative_id = ids[1] if len(ids) > 1 else None
+    return db._save_pair_components(conn, ids[0], negative_id)[0]
+
+
+def _document_form(conn: Any, values: Dict[str, Any]) -> Dict[str, Any]:
+    """``values`` with the workflow and prompt graph texts stored once.
+
+    Each of those columns becomes NULL beside its document id (new form);
+    every other column is unchanged.
+    """
+    texts = {
+        column: values[column]
+        for column, _reference in TEXT_COLUMNS
+        if column in values
+    }
+    rest = {key: value for key, value in values.items() if key not in texts}
+    return {**rest, **stored_values(conn, texts)}

database/pairs.py

@@ -112,8 +112,14 @@ class PairOperations:
         tags: Optional[List[str]] = None,
         rating: Optional[int] = None,
         notes: Optional[str] = None,
+        fill_new_positive: bool = False,
     ) -> int:
-        """Save an ordered pair atomically; repeat saves preserve its metadata."""
+        """Save an ordered pair atomically; repeat saves preserve its metadata.
+
+        ``fill_new_positive`` also gives the positive prompt the category and
+        tags, only if this save inserted its row (#546). An existing prompt
+        never changes, so a cleared category stays cleared.
+        """
         with self._pair_transaction() as conn:
             return self._save_pair(
                 conn,
@@ -124,6 +130,7 @@ class PairOperations:
                 tags=tags,
                 rating=rating,
                 notes=notes,
+                fill_new_positive=fill_new_positive,
             )
 
     def _save_pair(
@@ -137,6 +144,7 @@ class PairOperations:
         tags: Optional[List[str]] = None,
         rating: Optional[int] = None,
         notes: Optional[str] = None,
+        fill_new_positive: bool = False,
     ) -> int:
         """Save an ordered pair atomically; repeat saves preserve its metadata."""
         if not isinstance(positive_text, str) or not positive_text.strip():
@@ -144,8 +152,13 @@ class PairOperations:
         if not isinstance(negative_text, str):
             raise ValueError("Negative prompt text must be a string; use '' for empty")
         # Validate metadata before any component insertion.
-        _validate_metadata(title, category, tags, rating, notes)
-        positive = self._pair_component(conn, positive_text)
+        tags = _validate_metadata(title, category, tags, rating, notes)
+        positive, created = self._pair_component_outcome(conn, positive_text)
+        if created and fill_new_positive:
+            conn.execute(
+                "UPDATE prompts SET category = ? WHERE id = ?", (category, positive)
+            )
+            self._sync_prompt_tags(conn, positive, tags or [])
         negative = (
             self._pair_component(conn, negative_text) if negative_text.strip() else None
         )

database/prompt_archive.py

@@ -0,0 +1,107 @@
+"""Library parent archive (#443, D152): prompts hidden from the shelf, restorable.
+
+A row means the Library parent is archived; no row means it is on the shelf.
+Nothing else about the prompt changes: its text, metadata, tags, run count,
+image links, pairs and recorded runs stay. Restore deletes the row. Deleting
+the prompt (any path) drops the row by cascade.
+
+A table of its own, not a column: ``prompt_library_state`` is validated by its
+exact definition, so a new column there would make older builds refuse the
+upgraded database, while older builds ignore an unknown table (they show the
+archived prompts again and keep the rows). Optional in old backups, required
+once a database is current.
+"""
+
+import re
+import sqlite3
+
+TABLE = "prompt_archive"
+MAX_STAMP_BYTES = 64
+# What ``utc_now_iso`` writes starts with a date; anything else was not written here.
+_STAMP_GLOB = "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T*"
+_DDL = (
+    f"CREATE TABLE {TABLE} (prompt_id INTEGER PRIMARY KEY, "
+    "archived_at TEXT NOT NULL, "
+    "FOREIGN KEY(prompt_id) REFERENCES prompts(id) ON DELETE CASCADE)"
+)
+_COLUMNS = (("prompt_id", "INTEGER", 0, 1), ("archived_at", "TEXT", 1, 0))
+_FOREIGN_KEYS = {("prompt_id", "prompts", "id", "CASCADE")}
+# A predicate on ``p`` (the prompts alias): the parent is not archived.
+SHELVED = f"NOT EXISTS(SELECT 1 FROM {TABLE} pa WHERE pa.prompt_id=p.id)"
+ARCHIVED = f"EXISTS(SELECT 1 FROM {TABLE} pa WHERE pa.prompt_id=p.id)"
+
+
+def _compact(sql: str) -> str:
+    return re.sub(r"\s+", "", sql or "").lower()
+
+
+def _structure_error(conn: sqlite3.Connection, allow_missing: bool) -> str | None:
+    objects = conn.execute(
+        "SELECT type,sql FROM sqlite_master WHERE name=? "
+        "OR (type IN ('index','trigger') AND tbl_name=?)",
+        (TABLE, TABLE),
+    ).fetchall()
+    if not objects:
+        return None if allow_missing else "Missing prompt archive schema"
+    tables = [sql for kind, sql in objects if kind == "table"]
+    if len(tables) != 1 or len(objects) != 1:
+        return "Prompt archive name is reserved by an unsupported schema object"
+    if _compact(tables[0]) != _compact(_DDL):
+        return "Prompt archive has an invalid table definition"
+    columns = tuple(
+        (row[1], row[2].upper(), row[3], row[5])
+        for row in conn.execute(f"PRAGMA table_info({TABLE})")
+    )
+    if columns != _COLUMNS:
+        return "Prompt archive has invalid columns"
+    foreign_keys = {
+        (row[3], row[2], row[4], row[6])
+        for row in conn.execute(f"PRAGMA foreign_key_list({TABLE})")
+    }
+    if foreign_keys != _FOREIGN_KEYS:
+        return "Prompt archive has invalid foreign key definitions"
+    return None
+
+
+def prompt_archive_schema_error(
+    conn: sqlite3.Connection, *, allow_missing: bool = True
+) -> str | None:
+    """Reject an unsupported archive schema or values; reads only.
+
+    A missing table is acceptable for backups that predate it unless
+    ``allow_missing`` is false.
+    """
+    error = _structure_error(conn, allow_missing)
+    if error is not None:
+        return error
+    if not conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
+    ).fetchone():
+        return None
+    if conn.execute(f"PRAGMA foreign_key_check({TABLE})").fetchone():
+        return "Prompt archive contains foreign key violations"
+    if conn.execute(
+        f"SELECT 1 FROM {TABLE} WHERE typeof(archived_at)!='text' "  # nosec B608
+        f"OR length(CAST(archived_at AS BLOB))>{MAX_STAMP_BYTES} "
+        f"OR archived_at NOT GLOB '{_STAMP_GLOB}' LIMIT 1"
+    ).fetchone():
+        return "Prompt archive contains invalid values"
+    return None
+
+
+def migrate_prompt_archive(conn: sqlite3.Connection) -> None:
+    """Install the table atomically; validate before and after (no backfill)."""
+    conn.execute("SAVEPOINT prompt_archive_schema")
+    try:
+        error = _structure_error(conn, True)
+        if error:
+            raise sqlite3.DatabaseError(error)
+        conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
+        error = _structure_error(conn, False)
+        if error:
+            raise sqlite3.DatabaseError(error)
+        conn.execute("RELEASE prompt_archive_schema")
+    except BaseException:
+        conn.execute("ROLLBACK TO prompt_archive_schema")
+        conn.execute("RELEASE prompt_archive_schema")
+        raise

database/restore_stage.py

@@ -0,0 +1,376 @@
+"""A restore in two steps: stage the upload, then publish it (#249 System S8).
+
+**Stage** (``stage_restore``) never touches the live database or the source.
+In a new folder ``.pm-restore-*`` in the library's own folder (#318: never
+the system temp folder, which may be another disk), it:
+
+1. checks the source (``verify_database_file`` without reading every
+   workflow document: a damaged one is repaired below, not refused);
+2. copies it with SQLite's backup API (a rollback-journal file and a WAL file
+   alike) and opens the copy as a ``PromptModel``, which migrates an older
+   schema;
+3. repairs damaged workflow documents (``repair_damaged_documents``),
+   rotates the history epoch and counts what the copy holds;
+4. makes the copy one self-contained file and verifies it fully (every
+   current table, every document);
+5. records the copy's SHA-256, so publishing binds exactly these bytes.
+
+**Publish** (``publish_staged``) re-hashes the stage (``StageChanged`` if it
+differs), keeps the current database as a uniquely named, standalone safety
+backup beside it (``database/safety_backup.py``; not verified, because the
+database being replaced may be the damaged one), copies the stage over the
+live pages through the backup API (no file is swapped under open handles; an
+``OperationalError`` there leaves the live data intact, #226), then
+checkpoints the WAL with TRUNCATE so the copied pages do not leave a
+database-sized ``-wal`` behind. A reader that keeps a snapshot past the busy
+wait is logged, not fatal: the restore itself is done.
+
+Every handle on the stage is closed before its folder is removed (Windows).
+``sweep_restore_temps`` removes what an interrupted restore left (startup).
+"""
+
+import hashlib
+import logging
+import os
+import sqlite3
+import tempfile
+from contextlib import closing
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, Optional
+
+from . import workflow_documents as wd
+from .execution_history_schema import rotate_history_epoch
+from .safety_backup import take_safety_backup
+
+logger = logging.getLogger("prompt_manager.database.restore_stage")
+
+STAGE_PREFIX = ".pm-restore-"
+UPLOAD_PREFIX = ".pm-restore-upload-"
+STAGED_NAME = "staged.db"
+# How long the post-restore TRUNCATE checkpoint waits for readers to leave.
+CHECKPOINT_BUSY_MS = 30_000
+_HASH_CHUNK = 1024 * 1024
+SUMMARY_KEYS = (
+    "prompts",
+    "positives",
+    "pairs",
+    "archived_pairs",
+    "image_links",
+    "runs",
+    "saved_views",
+    "collections",
+)
+
+
+class StageChanged(ValueError):
+    """The staged file is no longer the bytes that were reviewed."""
+
+
+@dataclass
+class StagedRestore:
+    """A verified, migrated copy of an upload, ready to publish."""
+
+    path: str
+    digest: str
+    size: int
+    summary: Dict[str, int]
+    schema: Dict[str, bool]
+    documents: Dict[str, int] = field(
+        default_factory=lambda: {"repaired": 0, "cleared": 0, "images": 0}
+    )
+
+    @property
+    def directory(self) -> str:
+        return os.path.dirname(self.path)
+
+    def discard(self) -> None:
+        """Remove the stage folder (idempotent; its handles are all closed)."""
+        remove_stage_folder(self.directory)
+
+
+def remove_stage_folder(directory: str) -> None:
+    """Remove a stage folder and its files, logging what cannot be removed."""
+    if not directory or not os.path.isdir(directory):
+        return
+    for name in os.listdir(directory):
+        try:
+            os.remove(os.path.join(directory, name))
+        except OSError as error:
+            logger.warning(f"Could not remove restore stage file {name}: {error}")
+    try:
+        os.rmdir(directory)
+    except OSError as error:
+        logger.warning(f"Could not remove the restore stage folder: {error}")
+
+
+def file_digest(path: str) -> str:
+    """``sha256:<hex>`` of the file's bytes (blocking)."""
+    digest = hashlib.sha256()
+    with open(path, "rb") as handle:
+        for chunk in iter(lambda: handle.read(_HASH_CHUNK), b""):
+            digest.update(chunk)
+    return "sha256:" + digest.hexdigest()
+
+
+def _readonly(path: str) -> sqlite3.Connection:
+    return sqlite3.connect(Path(path).resolve().as_uri() + "?mode=ro", uri=True)
+
+
+def _schema(path: str) -> list:
+    """The file's schema objects, to tell whether staging migrated it."""
+    with closing(_readonly(path)) as conn:
+        return sorted(
+            conn.execute(
+                "SELECT type, name, tbl_name, sql FROM sqlite_master "
+                "WHERE name NOT LIKE 'sqlite_%'"
+            ).fetchall(),
+            key=repr,
+        )
+
+
+def _count(conn: sqlite3.Connection, table: str, where: str = "") -> int:
+    exists = conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
+    ).fetchone()
+    if not exists:
+        return 0
+    # ``table`` and ``where`` are constants of library_summary, never input.
+    return conn.execute(f"SELECT COUNT(*) FROM {table} {where}").fetchone()[0]  # nosec
+
+
+def library_summary(conn: sqlite3.Connection) -> Dict[str, int]:
+    """What a library holds, for a restore's review (reads only).
+
+    ``positives`` are the Library's prompts (``library_counts``'s
+    ``positive_prompts``); ``pairs`` the active saved pairs and
+    ``archived_pairs`` the archived ones; ``image_links`` the indexed image
+    rows; ``runs`` the recorded executions; ``collections`` the image
+    collections (#331).
+    """
+    from .library import _ELIGIBLE
+    from .prompt_archive import SHELVED, TABLE as ARCHIVE
+
+    has_state = conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND "
+        "name='prompt_library_state'"
+    ).fetchone()
+    has_pairs = conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name='prompt_pairs'"
+    ).fetchone()
+    has_archive = conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (ARCHIVE,)
+    ).fetchone()
+    if has_state and has_pairs:
+        # Archived parents (#443) are off the shelf, as in library_counts.
+        positives = conn.execute(
+            "SELECT COUNT(*) FROM prompts p WHERE "
+            + _ELIGIBLE
+            + (" AND " + SHELVED if has_archive else "")
+        ).fetchone()[0]
+    else:
+        positives = _count(conn, "prompts")
+    return {
+        "prompts": _count(conn, "prompts"),
+        "positives": positives,
+        "pairs": _count(conn, "prompt_pairs", "WHERE archived=0"),
+        "archived_pairs": _count(conn, "prompt_pairs", "WHERE archived<>0"),
+        "image_links": _count(conn, "generated_images"),
+        "runs": _count(conn, "execution_runs"),
+        "saved_views": _count(conn, "saved_views"),
+        "collections": _count(conn, "image_collections"),
+    }
+
+
+# The newest row of each table a restore would overwrite: with the counts,
+# a write after the review (a new run, image, prompt or edit) changes it.
+_FINGERPRINT_SQL = (
+    ("prompts", "SELECT MAX(id), MAX(updated_at), MAX(last_used_at) FROM prompts"),
+    ("generated_images", "SELECT MAX(id) FROM generated_images"),
+    ("prompt_pairs", "SELECT MAX(id), MAX(updated_at) FROM prompt_pairs"),
+    ("execution_runs", "SELECT COUNT(*), MAX(rowid) FROM execution_runs"),
+    ("saved_views", "SELECT MAX(rowid) FROM saved_views"),
+    ("image_collections", "SELECT MAX(rowid) FROM image_collections"),
+)
+
+
+def live_fingerprint(conn: sqlite3.Connection):
+    """``(summary, fingerprint)`` of a live library, read in one snapshot.
+
+    The fingerprint changes when rows are added to or removed from the
+    tables a restore replaces, or when a prompt, pair, run or saved view is
+    the newest edited (counts plus the newest row and edit of each table);
+    the apply refuses with ``plan-changed`` then. It does not see every
+    edit (a tag, favorite or collection membership change on an older row);
+    those are kept in the restore's safety backup. A read transaction the
+    caller already has is used and left open; one opened here is ended here.
+    """
+    import json
+
+    opened = not conn.in_transaction
+    if opened:
+        conn.execute("BEGIN")  # one read snapshot for every count
+    try:
+        summary = library_summary(conn)
+        newest = []
+        for table, sql in _FINGERPRINT_SQL:
+            try:
+                newest.append([table, list(conn.execute(sql).fetchone())])
+            except sqlite3.OperationalError:
+                newest.append([table, None])  # no such table or column
+    finally:
+        if opened:
+            conn.rollback()  # it only read
+    text = json.dumps([summary, newest], sort_keys=True, default=str)
+    return summary, "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def _copy(src_path: str, staged_path: str) -> None:
+    source = _readonly(src_path)
+    try:
+        with closing(sqlite3.connect(staged_path)) as destination:
+            source.backup(destination)
+    finally:
+        source.close()
+
+
+def _standalone(path: str) -> None:
+    with closing(sqlite3.connect(path)) as conn:
+        conn.execute("PRAGMA journal_mode=DELETE").fetchone()
+
+
+def stage_restore(model_cls, db_path: str, src_path: str) -> StagedRestore:
+    """Stage ``src_path`` beside ``db_path`` (blocking); see the module docstring.
+
+    Raises ValueError with a readable reason when the source or the migrated
+    stage does not verify; the stage folder is then already removed.
+    """
+    ok, reason = model_cls.verify_database_file(src_path, documents=False)
+    if not ok:
+        raise ValueError(reason)
+    before = _schema(src_path)
+    with closing(_readonly(src_path)) as conn:
+        library_state = bool(
+            conn.execute(
+                "SELECT 1 FROM sqlite_master WHERE type='table' AND "
+                "name='prompt_library_state'"
+            ).fetchone()
+        )
+    folder = os.path.dirname(os.path.abspath(db_path))
+    directory = tempfile.mkdtemp(prefix=STAGE_PREFIX, dir=folder)
+    staged_path = os.path.join(directory, STAGED_NAME)
+    staged_key = os.path.normcase(os.path.realpath(staged_path))
+    try:
+        _copy(src_path, staged_path)
+        staged = None
+        try:
+            staged = model_cls(staged_path)
+            conn = staged.get_connection()
+            documents = wd.repair_damaged_documents(conn)
+            rotate_history_epoch(conn)
+            summary = library_summary(conn)
+        finally:
+            if staged is not None:
+                staged.close_all()
+        _standalone(staged_path)
+        ok, reason = model_cls.verify_database_file(
+            staged_path, require_library_state=True
+        )
+        if not ok:
+            raise ValueError(reason)
+        result = StagedRestore(
+            path=staged_path,
+            digest=file_digest(staged_path),
+            size=os.path.getsize(staged_path),
+            summary=summary,
+            schema={
+                "migrated": _schema(staged_path) != before,
+                "library_state": library_state,
+            },
+            documents=documents,
+        )
+    except BaseException:
+        remove_stage_folder(directory)
+        raise
+    finally:
+        with model_cls._init_lock:
+            model_cls._initialized_paths.discard(staged_key)
+    return result
+
+
+def checkpoint_truncate(conn: sqlite3.Connection) -> bool:
+    """Checkpoint the WAL and truncate it to 0 bytes; False when a reader held it."""
+    previous = conn.execute("PRAGMA busy_timeout").fetchone()[0]
+    try:
+        conn.execute(f"PRAGMA busy_timeout = {int(CHECKPOINT_BUSY_MS)}")
+        busy = conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()[0]
+    finally:
+        conn.execute(f"PRAGMA busy_timeout = {int(previous)}")
+    return not busy
+
+
+def publish_staged(model, staged: StagedRestore) -> str:
+    """Publish ``staged`` over ``model``'s database; return the safety backup path.
+
+    "" when there was no database to back up. Raises StageChanged before
+    anything is written when the stage's bytes changed since staging, and
+    SafetyBackupFailed when the current database could not be kept. An
+    ``OperationalError`` from the page copy propagates with
+    ``_prompt_manager_restore_backup_created`` set; the live data is intact.
+    """
+    if not os.path.isfile(staged.path) or file_digest(staged.path) != staged.digest:
+        raise StageChanged("The staged restore changed since it was reviewed")
+    backup_path = ""
+    if os.path.exists(model.db_path):
+        backup_path = take_safety_backup(model, verify=False)
+        logger.info(f"Current database backed up to: {os.path.basename(backup_path)}")
+    try:
+        model._replace_contents_from(staged.path)
+    except sqlite3.OperationalError as exc:
+        exc._prompt_manager_restore_backup_created = bool(backup_path)
+        raise
+    with model._init_lock:
+        model._initialized_paths.add(model._schema_key())
+    _truncate_wal(model)
+    return backup_path
+
+
+def _truncate_wal(model) -> Optional[bool]:
+    """TRUNCATE-checkpoint the live WAL after a publish (never raises)."""
+    try:
+        conn = model.get_connection()
+        with conn.using():
+            truncated = checkpoint_truncate(conn)
+    except sqlite3.Error as error:
+        logger.warning(f"The database log could not be truncated: {error}")
+        return None
+    if not truncated:
+        logger.warning(
+            "The database log could not be truncated after the restore: a "
+            "reader kept it open. It shrinks at the next checkpoint with no "
+            "reader."
+        )
+    return truncated
+
+
+def sweep_restore_temps(db_path: str) -> list:
+    """Remove the stages and uploads an interrupted restore left (startup)."""
+    folder = os.path.dirname(os.path.abspath(db_path))
+    removed = []
+    try:
+        names = os.listdir(folder)
+    except OSError:
+        return removed
+    for name in names:
+        path = os.path.join(folder, name)
+        if name.startswith(UPLOAD_PREFIX) and os.path.isfile(path):
+            try:
+                os.remove(path)
+                removed.append(name)
+            except OSError:
+                continue
+        elif name.startswith(STAGE_PREFIX) and os.path.isdir(path):
+            remove_stage_folder(path)
+            if not os.path.exists(path):
+                removed.append(name)
+    return removed

database/review.py

@@ -0,0 +1,648 @@
+"""Needs review items (#249): persist and read capture-bound ambiguous evidence.
+
+An item is one ambiguous resolver result for one annotation byte record (the
+whole-file SHA-256 of one physical file at one locator), keyed by
+``(byte_id, resolver_version, evidence_hash)``. Recording is idempotent: the
+same evidence for the same bytes is a duplicate and never touches an existing
+item, its state, revision or decisions. Changed bytes are a new byte record
+and therefore a new pending item; the old one stays.
+
+Callers pass verified facts (the capture's digest, size and ``mtime_ns``);
+nothing here reads the filesystem. Writes run in one ``_pair_transaction``
+fenced by the storage epoch the capture was taken under.
+"""
+
+import hashlib
+import json
+import os
+import re
+import sqlite3
+from typing import Iterator
+
+try:
+    from ..utils.hashing import generate_prompt_hash
+except ImportError:
+    from utils.hashing import generate_prompt_hash
+
+from .annotation_schema import (
+    BYTES_TABLE,
+    FILES_TABLE,
+    METADATA_TABLE as ANNOTATION_METADATA_TABLE,
+)
+from .annotations import _new_id, _validate_locator, _validate_target
+from .models import normalize_image_path, utc_now_iso
+from .review_changes import notify_review_change
+from .review_schema import (
+    DECISIONS_TABLE,
+    ITEMS_TABLE,
+    MAX_CANDIDATES,
+    MAX_SPELLED_PATH_BYTES,
+    METADATA_TABLE,
+    canonical_evidence,
+    evidence_digest,
+    evidence_offers_choice,
+)
+
+# Only this module's table constants are interpolated into the SQL below.
+_INSERT_ITEM = (
+    f"INSERT OR IGNORE INTO {ITEMS_TABLE}(id,byte_id,spelled_path,"  # nosec B608
+    "resolver_version,evidence_hash,evidence,candidate_count,limited,state,"
+    "revision,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,'pending',0,?,?)"
+)
+_BUMP_REVIEW_INDEX = (
+    f"UPDATE {METADATA_TABLE} SET "  # nosec B608
+    "review_index_revision=review_index_revision+1 WHERE singleton_id=1"
+)
+# A later spelling of the same locator (a case-only rename on a case-folding
+# filesystem) replaces the recorded one, so targets keep formatting.
+_REFRESH_SPELLING = (
+    f"UPDATE {ITEMS_TABLE} SET spelled_path=? "  # nosec B608
+    f"WHERE spelled_path<>? AND byte_id IN (SELECT b.id FROM {BYTES_TABLE} b "
+    f"JOIN {FILES_TABLE} f ON f.id=b.file_id WHERE f.locator=?)"
+)
+_REVIEW_INDEX = (
+    f"SELECT review_index_revision FROM {METADATA_TABLE} "  # nosec B608
+    "WHERE singleton_id=1"
+)
+_BUMP_IMAGE_INDEX = (
+    f"UPDATE {ANNOTATION_METADATA_TABLE} SET "  # nosec B608
+    "image_index_revision=image_index_revision+1 WHERE singleton_id=1"
+)
+_ITEMS_OF_LOCATOR = (
+    "SELECT i.id,i.byte_id,i.resolver_version,i.evidence_hash,"  # nosec B608
+    "i.evidence,i.candidate_count,i.limited,i.state,i.revision,i.created_at,"
+    f"i.updated_at,b.sha256 FROM {FILES_TABLE} f "
+    f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
+    f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
+    "WHERE f.locator=? ORDER BY i.created_at,i.id"
+)
+_QUEUED = (
+    f"SELECT 1 FROM {FILES_TABLE} f "  # nosec B608
+    f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
+    f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
+    "WHERE f.locator=? AND b.size=? AND b.mtime_ns=? AND i.resolver_version=? "
+    "LIMIT 1"
+)
+_PENDING_AT = (
+    f"SELECT 1 FROM {FILES_TABLE} f "  # nosec B608
+    f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
+    f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
+    "WHERE f.locator=? AND b.size=? AND b.mtime_ns=? AND i.resolver_version=? "
+    "AND i.state='pending' LIMIT 1"
+)
+# Bytes verified at this stat whose item carries a decision (any version).
+_DECIDED_AT = (
+    f"SELECT 1 FROM {FILES_TABLE} f "  # nosec B608
+    f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
+    f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
+    "WHERE f.locator=? AND b.size=? AND b.mtime_ns=? "
+    "AND (i.state<>'pending' OR i.revision<>0 OR EXISTS (SELECT 1 FROM "
+    f"{DECISIONS_TABLE} d WHERE d.item_id=i.id)) LIMIT 1"
+)
+# The other items of one byte record (another version or evidence), with
+# whether each may be superseded: pending, revision 0, no decision row.
+_OTHER_ITEMS_OF_BYTES = (
+    "SELECT i.id, (i.state='pending' AND i.revision=0 AND NOT EXISTS "  # nosec B608
+    f"(SELECT 1 FROM {DECISIONS_TABLE} d WHERE d.item_id=i.id)) "
+    f"FROM {ITEMS_TABLE} i WHERE i.byte_id=? "
+    "AND NOT (i.resolver_version=? AND i.evidence_hash=?)"
+)
+# Pending items of the bytes verified at one stat, with their versions.
+_PENDING_AT_STAT = (
+    f"SELECT i.id,i.resolver_version FROM {FILES_TABLE} f "  # nosec B608
+    f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
+    f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
+    "WHERE f.locator=? AND b.size=? AND b.mtime_ns=? AND i.state='pending'"
+)
+# Pending items of one file: the candidates a link settles.
+_PENDING_OF_LOCATOR = (
+    f"SELECT i.id FROM {FILES_TABLE} f "  # nosec B608
+    f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
+    f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
+    "WHERE f.locator=? AND i.state='pending'"
+)
+# Pending items of one file with their evidence: an untraced file's settle.
+_PENDING_EVIDENCE_OF_LOCATOR = (
+    f"SELECT i.id,i.evidence FROM {FILES_TABLE} f "  # nosec B608
+    f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
+    f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
+    "WHERE f.locator=? AND i.state='pending'"
+)
+# Only an undecided item goes: a decision row (RESTRICT) is never deleted.
+_SETTLE_ITEM = (
+    f"DELETE FROM {ITEMS_TABLE} WHERE id=? AND state='pending' "  # nosec B608
+    f"AND revision=0 AND NOT EXISTS (SELECT 1 FROM {DECISIONS_TABLE} d "
+    f"WHERE d.item_id={ITEMS_TABLE}.id)"
+)
+_EPOCH = "SELECT storage_epoch FROM execution_history_metadata WHERE singleton_id=1"
+_PENDING_COUNT = (
+    f"SELECT COUNT(*) FROM {ITEMS_TABLE} WHERE state='pending'"  # nosec B608
+)
+_QUEUE_COLUMNS = (
+    "id",
+    "state",
+    "revision",
+    "created_at",
+    "candidate_count",
+    "limited",
+    "spelled_path",
+    "locator",
+)
+_QUEUE_SELECT = (
+    "SELECT i.id,i.state,i.revision,i.created_at,i.candidate_count,"  # nosec B608
+    f"i.limited,i.spelled_path,f.locator FROM {ITEMS_TABLE} i "
+    f"JOIN {BYTES_TABLE} b ON b.id=i.byte_id "
+    f"JOIN {FILES_TABLE} f ON f.id=b.file_id "
+)
+_QUEUE_FIRST = _QUEUE_SELECT + (
+    "WHERE i.state='pending' ORDER BY i.created_at,i.id LIMIT ?"
+)
+_QUEUE_AFTER = _QUEUE_SELECT + (
+    "WHERE i.state='pending' AND (i.created_at,i.id)>(?,?) "
+    "ORDER BY i.created_at,i.id LIMIT ?"
+)
+_DETAIL_COLUMNS = _QUEUE_COLUMNS + (
+    "byte_id",
+    "resolver_version",
+    "evidence_hash",
+    "evidence",
+    "updated_at",
+    "sha256",
+)
+_DETAIL = (
+    "SELECT i.id,i.state,i.revision,i.created_at,i.candidate_count,"  # nosec B608
+    "i.limited,i.spelled_path,f.locator,i.byte_id,i.resolver_version,"
+    "i.evidence_hash,i.evidence,i.updated_at,b.sha256 "
+    f"FROM {ITEMS_TABLE} i JOIN {BYTES_TABLE} b ON b.id=i.byte_id "
+    f"JOIN {FILES_TABLE} f ON f.id=b.file_id WHERE i.id=?"
+)
+# The legacy rows and pair links the ingester's compatibility rules read.
+_DEPENDENT_IMAGES = (
+    "SELECT gi.id,p.hash,gi.image_path FROM generated_images gi "
+    "JOIN prompts p ON p.id=gi.prompt_id "
+    "WHERE gi.file_path=? OR gi.image_path=? ORDER BY gi.id"
+)
+_DEPENDENT_LINKS = (
+    "SELECT DISTINCT ip.pair_id,n.hash FROM image_prompt_pairs ip "
+    "JOIN prompt_pairs pp ON pp.id=ip.pair_id "
+    "LEFT JOIN prompts n ON n.id=pp.negative_prompt_id "
+    "WHERE ip.image_id=? ORDER BY ip.pair_id"
+)
+# Rows whose spelling ends with the file's name: the candidates for another
+# spelling of the same physical file (the link summaries' filename fallback).
+_SAME_NAME_ROWS = (
+    "SELECT id,image_path FROM generated_images "
+    "WHERE image_path LIKE ? ESCAPE '\\' ORDER BY id"
+)
+_TITLES = (
+    "SELECT p.hash,s.title FROM prompts p "
+    "JOIN prompt_library_state s ON s.prompt_id=p.id "
+    "WHERE p.hash IN ({marks}) AND s.title IS NOT NULL ORDER BY p.id"
+)
+_ITEM_ID = re.compile(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}")
+_ITEM_COLUMNS = (
+    "id",
+    "byte_id",
+    "resolver_version",
+    "evidence_hash",
+    "evidence",
+    "candidate_count",
+    "limited",
+    "state",
+    "revision",
+    "created_at",
+    "updated_at",
+    "sha256",
+)
+
+
+def _validate_spelling(spelled_path: object, locator: str) -> None:
+    if (
+        not isinstance(spelled_path, str)
+        or not spelled_path
+        or len(spelled_path.encode("utf-8", "surrogatepass")) > MAX_SPELLED_PATH_BYTES
+        or os.path.normcase(spelled_path) != locator
+    ):
+        raise ValueError("spelled_path must be the file's spelling of its locator")
+
+
+def dependent_images(conn: sqlite3.Connection, spellings) -> list:
+    """``(id, image_path, positive_hash)`` of every legacy row of the file.
+
+    ``spellings`` are the paths the file is linked under (the Apply path and
+    every alias of the same physical file); each is matched as the ingester
+    matches one path. Rows are distinct and ordered by id.
+    """
+    found = {}
+    for spelled in dict.fromkeys(spellings):
+        for row in conn.execute(
+            _DEPENDENT_IMAGES, (normalize_image_path(spelled), spelled)
+        ):
+            found[row[0]] = (row[0], row[2], row[1])
+    return [found[key] for key in sorted(found)]
+
+
+def same_name_rows(conn: sqlite3.Connection, name: str) -> list:
+    """``(id, image_path)`` of rows whose spelling may end with ``name``."""
+    # Imported here: operations composes this module's mixin.
+    from .operations import escape_like
+
+    return [
+        (row[0], row[1])
+        for row in conn.execute(_SAME_NAME_ROWS, (f"%{escape_like(name)}",))
+    ]
+
+
+def dependency_fingerprint(
+    conn: sqlite3.Connection, image_path: str, aliases=()
+) -> str:
+    """SHA-256 of what Apply's compatibility depends on for ``image_path``.
+
+    The exact legacy rows of the file under ``image_path`` and every alias
+    spelling of it, with their positive hashes, and each row's pair links
+    with the negative hash, read as the ingester reads them. Unrelated prompt
+    changes (tags, titles) do not move it.
+    """
+    rows = []
+    for image_id, _spelled, positive_hash in dependent_images(
+        conn, (image_path, *aliases)
+    ):
+        links = conn.execute(_DEPENDENT_LINKS, (image_id,)).fetchall()
+        rows.append([image_id, positive_hash, [list(link) for link in links]])
+    text = canonical_evidence({"v": 1, "images": rows})
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+class ReviewOperations:
+    """PromptDatabase mixin: insert and read Needs review items."""
+
+    def record_review_item(
+        self,
+        *,
+        locator: str,
+        spelled_path: str,
+        sha256: str,
+        size: int,
+        mtime_ns: int,
+        epoch: str,
+        resolver_version: int,
+        evidence: dict,
+        candidate_count: int,
+        limited: bool,
+    ) -> bool:
+        """Store one ambiguous result for these exact bytes; True when new.
+
+        The byte record is created (or its verified fingerprint refreshed)
+        like an observe. ``spelled_path`` is the file's own spelling (it must
+        fold to ``locator``); targets are formatted from it. Raises
+        ``LibraryCommandError("storage-changed")`` when the storage epoch is
+        no longer ``epoch`` (a restore since the capture) and ``ValueError``
+        for invalid facts or evidence.
+
+        One file never holds two pending items for the same bytes: the new
+        item supersedes the byte record's other items (an older resolver or
+        semantics version) when every one is pending and undecided (the safe
+        delete of the self-heal); when one carries a decision or a later
+        revision, it is kept and nothing new is stored (False).
+        """
+        _validate_target(locator, sha256, size, mtime_ns)
+        _validate_spelling(spelled_path, locator)
+        if type(resolver_version) is not int or resolver_version < 1:
+            raise ValueError("resolver_version must be a positive integer")
+        if (
+            type(candidate_count) is not int
+            or not 0 <= candidate_count <= MAX_CANDIDATES
+        ):
+            raise ValueError(f"candidate_count must be between 0 and {MAX_CANDIDATES}")
+        if not isinstance(limited, bool) or (limited and candidate_count):
+            raise ValueError("A limited item carries no candidates")
+        text = canonical_evidence(evidence)
+        evidence_hash = evidence_digest(text)
+        with self._pair_transaction() as conn:
+            self._library_epoch(conn, epoch)
+            now = utc_now_iso()
+            file_id = self._annotation_owner(conn, locator, now)
+            byte_id, _revision, created, moved = self._annotation_byte(
+                conn, file_id, sha256, size, mtime_ns, now
+            )
+            if created or moved:
+                conn.execute(_BUMP_IMAGE_INDEX)
+            # The byte record is created or refreshed even when the item is
+            # refused below (an observe would do the same); that commits
+            # with the image index bump, and the review queue is unchanged.
+            others = conn.execute(
+                _OTHER_ITEMS_OF_BYTES, (byte_id, resolver_version, evidence_hash)
+            ).fetchall()
+            if any(not removable for _id, removable in others):
+                return False
+            superseded = sum(
+                conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
+                for item_id, _removable in others
+            )
+            cursor = conn.execute(
+                _INSERT_ITEM,
+                (
+                    _new_id(),
+                    byte_id,
+                    spelled_path,
+                    resolver_version,
+                    evidence_hash,
+                    text,
+                    candidate_count,
+                    int(limited),
+                    now,
+                    now,
+                ),
+            )
+            inserted = cursor.rowcount > 0
+            refreshed = _refresh_spelling(conn, locator, spelled_path)
+            if inserted or refreshed or superseded:
+                conn.execute(_BUMP_REVIEW_INDEX)
+        if inserted or refreshed or superseded:
+            notify_review_change(self.model.db_path)
+        return inserted
+
+    def refresh_review_spelling(self, locator: str, spelled_path: str) -> bool:
+        """Record ``spelled_path`` as the spelling of every item at ``locator``.
+
+        The rescan's skip calls it for a file it does not copy again: a
+        case-only rename keeps size and ``mtime_ns``, and a target formatted
+        from the old spelling would be refused. True when a spelling changed
+        (the review index revision moves; item revisions do not, so an open
+        decision stays valid).
+        """
+        _validate_locator(locator)
+        _validate_spelling(spelled_path, locator)
+        with self._pair_transaction() as conn:
+            refreshed = _refresh_spelling(conn, locator, spelled_path)
+            if refreshed:
+                conn.execute(_BUMP_REVIEW_INDEX)
+        if refreshed:
+            notify_review_change(self.model.db_path)
+        return refreshed
+
+    def review_index_revision(self) -> int:
+        """The queue's revision (moved by every queue change)."""
+        row = self.model.get_connection().execute(_REVIEW_INDEX).fetchone()
+        if row is None:
+            raise sqlite3.DatabaseError("Missing review queue metadata")
+        return row[0]
+
+    def review_queue_version(self) -> dict:
+        """``{library_epoch, review_revision}`` of the queue, from one snapshot.
+
+        The pair a list read answers: equal pairs mean the same queue (a
+        restore moves the epoch, every queue change moves the revision).
+        """
+        with self._annotation_read() as conn:
+            epoch = conn.execute(_EPOCH).fetchone()
+            revision = conn.execute(_REVIEW_INDEX).fetchone()
+        if epoch is None or revision is None:
+            raise sqlite3.DatabaseError("Missing review queue metadata")
+        return {"library_epoch": epoch[0], "review_revision": revision[0]}
+
+    def review_items_for(self, locator: str) -> list:
+        """Every item of the file at ``locator``, oldest first, evidence parsed."""
+        _validate_locator(locator)
+        return list(self._iter_review_items(locator))
+
+    def _iter_review_items(self, locator: str) -> Iterator[dict]:
+        conn = self.model.get_connection()
+        for row in conn.execute(_ITEMS_OF_LOCATOR, (locator,)):
+            item = dict(zip(_ITEM_COLUMNS, tuple(row)))
+            item["evidence"] = json.loads(item["evidence"])
+            item["limited"] = bool(item["limited"])
+            yield item
+
+    # -- queue reads (#249 E3) -------------------------------------------------
+
+    def review_fingerprint_queued(
+        self, locator: str, size: int, mtime_ns: int, resolver_version: int
+    ) -> bool:
+        """True when an item cites a byte record verified at this exact stat.
+
+        A rescan uses it to skip the copy and hash of a file it already
+        queued: the same ``(size, mtime_ns)`` the bytes were verified with.
+        """
+        row = (
+            self.model.get_connection()
+            .execute(_QUEUED, (locator, size, mtime_ns, resolver_version))
+            .fetchone()
+        )
+        return row is not None
+
+    def review_decided_at(self, locator: str, size: int, mtime_ns: int) -> bool:
+        """True when bytes verified at this stat carry a decided item.
+
+        Any version: a newer resolver or table never adds an item beside a
+        decision, so the rescan need not copy those bytes again.
+        """
+        row = (
+            self.model.get_connection()
+            .execute(_DECIDED_AT, (locator, size, mtime_ns))
+            .fetchone()
+        )
+        return row is not None
+
+    def review_pending_at(
+        self, locator: str, size: int, mtime_ns: int, resolver_version: int
+    ) -> bool:
+        """True when a *pending* item cites bytes verified at this exact stat.
+
+        ``review_fingerprint_queued`` also matches decided items; a file whose
+        only item is decided is not in the queue.
+        """
+        row = (
+            self.model.get_connection()
+            .execute(_PENDING_AT, (locator, size, mtime_ns, resolver_version))
+            .fetchone()
+        )
+        return row is not None
+
+    def review_file_linked(self, spellings) -> bool:
+        """True when a saved row links the file under any of ``spellings``."""
+        with self._annotation_read() as conn:
+            return bool(dependent_images(conn, spellings))
+
+    def settle_linked_review_items(self, files) -> int:
+        """Remove the pending, undecided items of files that are now linked.
+
+        ``files`` holds ``(locator, spellings)`` per file. An item leaves the
+        queue only when it is still ``pending`` at revision 0 with no decision
+        row and a saved row links its file under one of ``spellings``; a
+        decided item is never touched. Files without a pending item cost one
+        indexed read and open no write transaction. Returns how many items
+        were removed (the review index revision moves when any was).
+        """
+        files = [(locator, tuple(spellings)) for locator, spellings in files]
+        for locator, _spellings in files:
+            _validate_locator(locator)
+        conn = self.model.get_connection()
+        candidates = [
+            (locator, spellings)
+            for locator, spellings in files
+            if conn.execute(_PENDING_OF_LOCATOR, (locator,)).fetchone() is not None
+        ]
+        if not candidates:
+            return 0
+        removed = 0
+        with self._pair_transaction() as conn:
+            for locator, spellings in candidates:
+                if not dependent_images(conn, spellings):
+                    continue
+                for (item_id,) in conn.execute(
+                    _PENDING_OF_LOCATOR, (locator,)
+                ).fetchall():
+                    removed += conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
+            if removed:
+                conn.execute(_BUMP_REVIEW_INDEX)
+        if removed:
+            notify_review_change(self.model.db_path)
+        return removed
+
+    def settle_stale_review_items(
+        self, locator: str, size: int, mtime_ns: int, versions
+    ) -> int:
+        """Remove pending, undecided items of the bytes verified at this stat
+        whose ``resolver_version`` is not one of ``versions``.
+
+        The caller found these bytes untraced under the current versions, so
+        an older version's item could only refuse Apply. The same safe delete
+        as the other settles; one indexed read when nothing is pending.
+        """
+        _validate_locator(locator)
+        current = set(versions)
+        conn = self.model.get_connection()
+        stale = [
+            item_id
+            for item_id, version in conn.execute(
+                _PENDING_AT_STAT, (locator, size, mtime_ns)
+            )
+            if version not in current
+        ]
+        if not stale:
+            return 0
+        removed = 0
+        with self._pair_transaction() as conn:
+            for item_id in stale:
+                removed += conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
+            if removed:
+                conn.execute(_BUMP_REVIEW_INDEX)
+        if removed:
+            notify_review_change(self.model.db_path)
+        return removed
+
+    def settle_untraced_review_items(self, locators) -> int:
+        """Remove the pending, undecided items that offer nothing to choose.
+
+        ``locators`` are files the caller found ambiguous without a choice
+        (fewer than two candidates carry a prompt). Of their items, one leaves
+        the queue only when its own stored evidence offers no choice either
+        (``evidence_offers_choice``) and it is still ``pending`` at revision 0
+        with no decision row, the same safe delete as
+        ``settle_linked_review_items``. Files without a pending item cost one
+        indexed read and open no write transaction. Returns how many items
+        were removed (the review index revision moves when any was).
+        """
+        locators = list(locators)
+        for locator in locators:
+            _validate_locator(locator)
+        conn = self.model.get_connection()
+        pending = [
+            locator
+            for locator in locators
+            if conn.execute(_PENDING_OF_LOCATOR, (locator,)).fetchone() is not None
+        ]
+        if not pending:
+            return 0
+        removed = 0
+        with self._pair_transaction() as conn:
+            for locator in pending:
+                rows = conn.execute(_PENDING_EVIDENCE_OF_LOCATOR, (locator,)).fetchall()
+                for item_id, evidence in rows:
+                    if evidence_offers_choice(json.loads(evidence)):
+                        continue
+                    removed += conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
+            if removed:
+                conn.execute(_BUMP_REVIEW_INDEX)
+        return removed
+
+    def review_queue_page(self, *, limit: int, after: tuple | None = None) -> dict:
+        """One page of pending items, oldest first, from one snapshot.
+
+        ``after`` is the ``(created_at, id)`` keyset of the last item already
+        shown. Returns ``{epoch, revision, total, items, more}``; ``total`` is
+        the exact pending count and ``items`` carry their ``spelled_path`` and
+        ``locator`` (server-side only).
+        """
+        if type(limit) is not int or limit < 1:
+            raise ValueError("limit must be a positive integer")
+        with self._annotation_read() as conn:
+            epoch = conn.execute(_EPOCH).fetchone()
+            revision = conn.execute(_REVIEW_INDEX).fetchone()
+            if epoch is None or revision is None:
+                raise sqlite3.DatabaseError("Missing review queue metadata")
+            total = conn.execute(_PENDING_COUNT).fetchone()[0]
+            if after is None:
+                rows = conn.execute(_QUEUE_FIRST, (limit + 1,)).fetchall()
+            else:
+                rows = conn.execute(_QUEUE_AFTER, (*after, limit + 1)).fetchall()
+        items = [_queue_row(row, _QUEUE_COLUMNS) for row in rows[:limit]]
+        return {
+            "epoch": epoch[0],
+            "revision": revision[0],
+            "total": total,
+            "items": items,
+            "more": len(rows) > limit,
+        }
+
+    def review_item(self, item_id: str) -> dict | None:
+        """One item with its parsed evidence and the pending total, or None."""
+        if not isinstance(item_id, str) or _ITEM_ID.fullmatch(item_id) is None:
+            return None
+        with self._annotation_read() as conn:
+            epoch = conn.execute(_EPOCH).fetchone()
+            if epoch is None:
+                raise sqlite3.DatabaseError("Missing execution history metadata")
+            row = conn.execute(_DETAIL, (item_id,)).fetchone()
+            if row is None:
+                return None
+            item = _queue_row(row, _DETAIL_COLUMNS)
+            item["evidence"] = json.loads(item["evidence"])
+            item["epoch"] = epoch[0]
+            item["total"] = conn.execute(_PENDING_COUNT).fetchone()[0]
+        return item
+
+    def review_dependency_fingerprint(self, image_path: str, aliases=()) -> str:
+        with self._annotation_read() as conn:
+            return dependency_fingerprint(conn, image_path, aliases)
+
+    def review_same_name_rows(self, name: str) -> list:
+        """``(id, image_path)`` of legacy rows that may spell a file ``name``."""
+        with self._annotation_read() as conn:
+            return same_name_rows(conn, name)
+
+    def stored_prompt_titles(self, texts) -> dict:
+        """Library titles of saved prompts by text (dedup hash), when set."""
+        hashes = {generate_prompt_hash(text): text for text in texts if text.strip()}
+        if not hashes:
+            return {}
+        marks = ",".join("?" * len(hashes))
+        found: dict[str, tuple] = {}
+        conn = self.model.get_connection()
+        for prompt_hash, title in conn.execute(
+            _TITLES.format(marks=marks), list(hashes)
+        ):
+            text = hashes[prompt_hash]
+            found[text] = found.get(text, ()) + (title,)
+        return found
+
+
+def _refresh_spelling(conn: sqlite3.Connection, locator: str, spelled: str) -> bool:
+    return conn.execute(_REFRESH_SPELLING, (spelled, spelled, locator)).rowcount > 0
+
+
+def _queue_row(row, columns) -> dict:
+    item = dict(zip(columns, tuple(row)))
+    item["limited"] = bool(item["limited"])
+    return item

database/review_changes.py

@@ -0,0 +1,75 @@
+"""Needs review queue change notices (#249 PR F2).
+
+Every committed change to the pending queue (an item recorded by the scan or
+the monitor, a spelling refresh, a decision) calls ``notify_review_change``
+after its transaction, from whichever thread wrote it. The review event
+stream (``py/api/review_events.py``) subscribes, so an open Needs review page
+re-reads its queue when it changes instead of polling. Observers are keyed by
+the database file, like ``subscribe_history_restore``; an observer that fails
+is logged and never fails the write.
+"""
+
+import os
+import threading
+from typing import Callable
+
+try:
+    from ..utils.logging_config import get_logger
+except ImportError:
+    from utils.logging_config import get_logger
+
+_logger = get_logger("prompt_manager.database.review_changes")
+_lock = threading.Lock()
+_observers: dict[str, dict[object, Callable[[], None]]] = {}
+
+
+def _identity(db_path: str) -> str:
+    return os.path.normcase(os.path.realpath(db_path))
+
+
+def subscribe_review_changes(
+    db_path: str, callback: Callable[[], None]
+) -> Callable[[], None]:
+    """Call ``callback()`` after each committed queue change; return the unsubscribe."""
+    identity = _identity(db_path)
+    token = object()
+    with _lock:
+        _observers.setdefault(identity, {})[token] = callback
+
+    def unsubscribe() -> None:
+        with _lock:
+            observers = _observers.get(identity)
+            if observers is not None:
+                observers.pop(token, None)
+                if not observers:
+                    _observers.pop(identity, None)
+
+    return unsubscribe
+
+
+def observer_count(db_path: str) -> int:
+    """How many observers watch ``db_path`` (tests and diagnostics)."""
+    with _lock:
+        return len(_observers.get(_identity(db_path), {}))
+
+
+def notify_review_change(db_path: str) -> None:
+    """Tell every observer of ``db_path``; call it after the commit."""
+    with _lock:
+        callbacks = tuple(_observers.get(_identity(db_path), {}).values())
+    for callback in callbacks:
+        try:
+            callback()
+        except Exception:
+            _logger.warning("Needs review change observer failed", exc_info=True)
+
+
+def rekey_review_changes(old_path: str, new_path: str) -> None:
+    """Move the observers of ``old_path`` to ``new_path`` (Move library)."""
+    old, new = _identity(old_path), _identity(new_path)
+    if old == new:
+        return
+    with _lock:
+        observers = _observers.pop(old, None)
+        if observers:
+            _observers.setdefault(new, {}).update(observers)

database/review_decisions.py

@@ -0,0 +1,397 @@
+"""Needs review decisions (#249 E3): explicit, auditable and compatible-only.
+
+A decision is one row of ``media_review_decisions`` for one item revision.
+``selected`` links the chosen candidate's pair to the file through the public
+ingester's compatible-only rules (``_ingest_resolved_pair_in``) in the same
+``_pair_transaction`` as the decision row, the item's new state and revision
+and the review index revision; any conflict or later failure rolls all of it
+back. ``manual`` (#536) is the same but for a Library prompt the reviewer chose:
+its link's provenance names a manual choice and carries no prompt sides, so it
+cannot pose as file evidence, and the decision's ``candidate_hash`` (the table
+requires one) is the digest of that choice. ``unresolved`` records the decision
+and links nothing. No decision counts a run, touches usage or calls an execution
+writer.
+
+Callers verify the file first (the capture must be the item's bytes and the
+current file must still hash to them); nothing here reads the filesystem.
+"""
+
+import os
+import sqlite3
+
+from .annotation_schema import BYTES_TABLE
+from .annotations import _new_id
+from .library import _ELIGIBLE, _FROM, SHELVED
+from .library_commands import LibraryCommandError
+from .models import utc_now_iso
+from .review import (
+    _BUMP_REVIEW_INDEX,
+    _DETAIL,
+    _DETAIL_COLUMNS,
+    _PENDING_COUNT,
+    _queue_row,
+    dependency_fingerprint,
+    dependent_images,
+    same_name_rows,
+)
+from .review_changes import notify_review_change
+from .review_schema import (
+    DECISIONS_TABLE,
+    ITEMS_TABLE,
+    canonical_evidence,
+    evidence_digest,
+)
+
+DECISION_KIND = "user-selection"
+MANUAL_KIND = "manual-link"
+_MANUAL_POSITIVE = (
+    f"SELECT p.id,p.text FROM {_FROM} "  # nosec B608
+    f"WHERE p.id=? AND {_ELIGIBLE} AND {SHELVED}"
+)
+_MANUAL_PAIR = (
+    "SELECT COALESCE(n.text,'') AS negative FROM prompt_pairs pp "
+    "LEFT JOIN prompts n ON n.id=pp.negative_prompt_id "
+    "WHERE pp.id=? AND pp.positive_prompt_id=? AND pp.archived=0"
+)
+_ARCHIVED_BARE_PAIR = (
+    "SELECT 1 FROM prompt_pairs WHERE positive_prompt_id=? "
+    "AND negative_prompt_id IS NULL AND archived=1"
+)
+_INSERT_DECISION = (
+    f"INSERT INTO {DECISIONS_TABLE}(id,item_id,decision_revision,"  # nosec B608
+    "created_at,outcome,candidate_hash,evidence,pair_id,image_id) "
+    "VALUES(?,?,?,?,?,?,?,?,?)"
+)
+_ADVANCE_ITEM = (
+    f"UPDATE {ITEMS_TABLE} SET state=?,revision=revision+1,"  # nosec B608
+    "updated_at=? WHERE id=? AND revision=?"
+)
+_ITEM_BYTES = (
+    f"SELECT b.sha256 FROM {ITEMS_TABLE} i "  # nosec B608
+    f"JOIN {BYTES_TABLE} b ON b.id=i.byte_id WHERE i.id=?"
+)
+
+
+class ReviewCommandError(LibraryCommandError):
+    """A decision lost a race or would rewrite an association.
+
+    ``code`` is ``review-conflict`` (the item was decided or changed since it
+    was read), ``association-conflict`` (the choice is not compatible with
+    the file's saved links, or those links changed), ``file-changed`` (the
+    item is not about these bytes) or ``review-missing``. ``item`` is the
+    current item read inside the transaction, ``total`` the pending count and
+    ``fingerprint`` the current dependency fingerprint, when known;
+    ``changed`` marks an association conflict caused by links that changed
+    since the reviewer read them (rather than an incompatible choice).
+    ``prompt-missing``: the Library prompt or recipe of a manual link is gone,
+    archived or not that prompt's.
+    """
+
+    def __init__(
+        self, code, epoch, *, item=None, total=None, fingerprint=None, changed=False
+    ):
+        super().__init__(code, epoch)
+        self.item = item
+        self.total = total
+        self.fingerprint = fingerprint
+        self.changed = changed
+        # The public view of ``item``, set by the API when it formats one.
+        self.view = None
+
+
+class _Rollback(Exception):
+    """Raised inside the transaction so the context manager rolls back."""
+
+    def __init__(self, error: ReviewCommandError):
+        super().__init__(error.code)
+        self.error = error
+
+
+class ReviewDecisionOperations:
+    """PromptDatabase mixin: record one decision on one Needs review item."""
+
+    def decide_review_item(
+        self,
+        *,
+        item_id: str,
+        expected_epoch: str,
+        expected_revision: int,
+        sha256: str,
+        outcome: str,
+        candidate: dict | None = None,
+        manual: dict | None = None,
+        image_path: str | None = None,
+        metadata: dict | None = None,
+        expected_fingerprint: str | None = None,
+        aliases: tuple = (),
+        known_rows: frozenset | None = None,
+    ) -> dict:
+        """Record the decision; return ``{item, total, pair}``.
+
+        ``selected`` needs the evidence ``candidate`` (with its ``hash``) or
+        ``manual`` (``{positive_id, pair_id}``; a null ``pair_id`` is no
+        negative), the
+        ``image_path`` the scan and the monitor link under, the captured
+        ``metadata`` and the ``expected_fingerprint`` the reviewer saw.
+        ``aliases`` are the other spellings legacy rows of the same physical
+        file use (a symlinked, junctioned or mapped root) and ``known_rows``
+        the ids of every same-name row the caller classified; a same-name row
+        it did not see is a changed association.
+        Raises ``LibraryCommandError("storage-changed")`` after a restore and
+        :class:`ReviewCommandError` for every refused decision, with zero
+        writes either way.
+        """
+        if outcome not in ("selected", "unresolved"):
+            raise ValueError("outcome must be selected or unresolved")
+        if outcome == "selected" and (
+            (not isinstance(candidate, dict)) == (not isinstance(manual, dict))
+            or not image_path
+        ):
+            raise ValueError(
+                "A selected decision needs one candidate or manual choice and a file"
+            )
+        try:
+            with self._pair_transaction() as conn:
+                decided = self._decide_in(
+                    conn,
+                    item_id=item_id,
+                    expected_epoch=expected_epoch,
+                    expected_revision=expected_revision,
+                    sha256=sha256,
+                    outcome=outcome,
+                    candidate=candidate,
+                    manual=manual,
+                    image_path=image_path,
+                    metadata=metadata or {},
+                    expected_fingerprint=expected_fingerprint,
+                    aliases=tuple(aliases),
+                    known_rows=known_rows,
+                )
+        except _Rollback as rollback:
+            raise rollback.error from None
+        # After the commit: open Needs review pages re-read their queue.
+        notify_review_change(self.model.db_path)
+        return decided
+
+    def _decide_in(self, conn: sqlite3.Connection, **command) -> dict:
+        epoch = self._library_epoch(conn, command["expected_epoch"])
+        item_id = command["item_id"]
+        current = _current_item(conn, item_id)
+        if current is None:
+            raise _Rollback(ReviewCommandError("review-missing", epoch))
+        total = conn.execute(_PENDING_COUNT).fetchone()[0]
+        if (
+            current["state"] != "pending"
+            or current["revision"] != command["expected_revision"]
+        ):
+            raise _Rollback(
+                ReviewCommandError("review-conflict", epoch, item=current, total=total)
+            )
+        if conn.execute(_ITEM_BYTES, (item_id,)).fetchone()[0] != command["sha256"]:
+            raise _Rollback(ReviewCommandError("file-changed", epoch))
+        decision_id = _new_id()
+        pair = None
+        selected = command["outcome"] == "selected"
+        candidate = command["candidate"] if selected else None
+        manual = command["manual"] if selected else None
+        candidate_hash = candidate["hash"] if candidate is not None else None
+        if candidate is not None:
+            pair = self._apply_candidate(
+                conn, epoch, current, decision_id, total, **command
+            )
+        elif manual is not None:
+            pair = self._apply_manual(
+                conn, epoch, current, decision_id, total, **command
+            )
+            manual = {"positive_id": manual["positive_id"], "pair_id": pair["id"]}
+            candidate_hash = evidence_digest(
+                canonical_evidence({"kind": MANUAL_KIND, **manual})
+            )
+        evidence = {
+            "kind": MANUAL_KIND if manual is not None else DECISION_KIND,
+            "decision_id": decision_id,
+            "outcome": command["outcome"],
+            "reviewed_digest": command["sha256"],
+            "item_evidence_hash": current["evidence_hash"],
+            "resolver_version": current["resolver_version"],
+            "candidate": candidate,
+            "dependency_fingerprint": command["expected_fingerprint"],
+            "pair": pair,
+        }
+        if manual is not None:
+            evidence["manual"] = manual
+        now = utc_now_iso()
+        self._insert_review_decision(
+            conn,
+            (
+                decision_id,
+                item_id,
+                current["revision"] + 1,
+                now,
+                command["outcome"],
+                candidate_hash,
+                canonical_evidence(evidence),
+                pair["id"] if pair else None,
+                pair["image_id"] if pair else None,
+            ),
+        )
+        conn.execute(
+            _ADVANCE_ITEM, (command["outcome"], now, item_id, current["revision"])
+        )
+        conn.execute(_BUMP_REVIEW_INDEX)
+        return {
+            "item": _current_item(conn, item_id),
+            "total": conn.execute(_PENDING_COUNT).fetchone()[0],
+            "pair": pair,
+            "decision_id": decision_id,
+        }
+
+    def _apply_candidate(
+        self, conn, epoch, current, decision_id, total, **command
+    ) -> dict:
+        """Link the candidate's pair, compatible-only; raise on any conflict."""
+        candidate = command["candidate"]
+        positive, negative = candidate["positive"], candidate["negative"]
+        provenance = {
+            "kind": DECISION_KIND,
+            "decision_id": decision_id,
+            "reviewed_digest": command["sha256"],
+            "candidate_hash": candidate["hash"],
+            "resolver_version": current["resolver_version"],
+            "positive": positive,
+            "negative": negative,
+        }
+        return self._link_compatible(
+            conn, epoch, current, total, positive, negative, provenance, command
+        )
+
+    def _apply_manual(self, conn, epoch, current, decision_id, total, **command):
+        """Link the chosen Library prompt, compatible-only; raise otherwise."""
+        missing = _Rollback(
+            ReviewCommandError("prompt-missing", epoch, item=current, total=total)
+        )
+        positive_id = command["manual"]["positive_id"]
+        pair_id = command["manual"]["pair_id"]
+        row = conn.execute(_MANUAL_POSITIVE, (positive_id,)).fetchone()
+        if row is None:
+            raise missing
+        positive_text, negative_text = row[1], ""
+        if pair_id is not None:
+            chosen = conn.execute(_MANUAL_PAIR, (pair_id, positive_id)).fetchone()
+            if chosen is None:
+                raise missing
+            negative_text = chosen[0]
+        elif conn.execute(_ARCHIVED_BARE_PAIR, (positive_id,)).fetchone():
+            raise missing
+        provenance = {
+            "kind": MANUAL_KIND,
+            "decision_id": decision_id,
+            "reviewed_digest": command["sha256"],
+            "resolver_version": current["resolver_version"],
+            "positive_id": positive_id,
+            "pair_id": pair_id,
+        }
+        # The sides name no provenance: only the ingester reads them.
+        pair = self._link_compatible(
+            conn,
+            epoch,
+            current,
+            total,
+            {"text": positive_text},
+            {"text": negative_text},
+            provenance,
+            command,
+        )
+        # A prompt whose stored hash is stale reaches another row: never link that.
+        reached = conn.execute(
+            "SELECT positive_prompt_id FROM prompt_pairs WHERE id=?", (pair["id"],)
+        ).fetchone()[0]
+        if reached != positive_id or pair_id not in (None, pair["id"]):
+            raise missing
+        return pair
+
+    def _link_compatible(
+        self,
+        conn,
+        epoch,
+        current,
+        total,
+        positive,
+        negative,
+        provenance,
+        command,
+    ) -> dict:
+        # Imported here: pair_ingestion imports operations, which composes this.
+        from .pair_ingestion import _ingest_resolved_pair_in
+
+        image_path = command["image_path"]
+        aliases = command["aliases"]
+        fingerprint = dependency_fingerprint(conn, image_path, aliases)
+        known = command["known_rows"]
+        unseen = known is not None and any(
+            row_id not in known
+            for row_id, _path in same_name_rows(conn, os.path.basename(image_path))
+        )
+        if unseen or fingerprint != command["expected_fingerprint"]:
+            raise _Rollback(
+                ReviewCommandError(
+                    "association-conflict",
+                    epoch,
+                    item=current,
+                    total=total,
+                    fingerprint=fingerprint,
+                    changed=True,
+                )
+            )
+        rows = dependent_images(conn, (image_path, *aliases))
+        if len(rows) > 1:
+            # One file linked under several spellings: no single row to enrich.
+            raise _Rollback(
+                ReviewCommandError(
+                    "association-conflict",
+                    epoch,
+                    item=current,
+                    total=total,
+                    fingerprint=fingerprint,
+                )
+            )
+        if rows:
+            # Enrich the one row of this file under the spelling it was
+            # linked with, so the ingester's compatibility rules see it.
+            image_path = rows[0][1]
+        result = _ingest_resolved_pair_in(
+            conn,
+            self,
+            image_path,
+            command["metadata"],
+            {"state": "resolved", "positive": positive, "negative": negative},
+            provenance=provenance,
+        )
+        if result["status"] == "conflict":
+            raise _Rollback(
+                ReviewCommandError(
+                    "association-conflict",
+                    epoch,
+                    item=current,
+                    total=total,
+                    fingerprint=fingerprint,
+                )
+            )
+        return {
+            "id": result["pair_id"],
+            "image_id": result["image_id"],
+            "status": result["status"],
+        }
+
+    @staticmethod
+    def _insert_review_decision(conn: sqlite3.Connection, values: tuple) -> None:
+        conn.execute(_INSERT_DECISION, values)
+
+
+def _current_item(conn: sqlite3.Connection, item_id: str) -> dict | None:
+    row = conn.execute(_DETAIL, (item_id,)).fetchone()
+    if row is None:
+        return None
+    item = _queue_row(row, _DETAIL_COLUMNS)
+    item.pop("evidence")
+    return item

database/review_schema.py

@@ -0,0 +1,515 @@
+"""Needs review schema group (#249): capture-bound ambiguous items and decisions.
+
+The group installs like ``image_annotations``: optional in old backups,
+required once a database is current, and valid only beside a valid image
+annotation group, because every item cites one annotation byte record. It owns
+
+- ``media_review_metadata``: one row, the group's schema version and the
+  review index revision (moved by every queue change);
+- ``media_review_items``: one ambiguous resolver result for one byte record,
+  keyed by ``(byte_id, resolver_version, evidence_hash)``, with the file's
+  spelling when it was recorded (the locator is case-folded on Windows, a
+  target is not), its canonical evidence, candidate count, ``limited`` flag,
+  state and revision;
+- ``media_review_decisions``: the auditable decision history of an item, one
+  row per decision revision, with optional navigation to the pair and image
+  the decision linked (nulled, never cascaded, when those are deleted).
+
+Evidence is canonical JSON (:func:`canonical_evidence`): one object, sorted
+keys, no whitespace, ASCII only (so characters, bytes and the 8 MiB bound
+agree and lone surrogates survive), no NaN or infinity. ``evidence_hash`` is
+the SHA-256 of the item's evidence text.
+
+Installation never backfills anything and never touches the filesystem.
+Validation reads only: catalog objects compared after whitespace compaction,
+columns and foreign keys through the PRAGMAs, persisted values in single
+bounded SQL passes, then one streamed pass that re-canonicalizes each evidence
+document (one row in memory at a time).
+"""
+
+import hashlib
+import json
+import re
+import sqlite3
+from typing import Any, Optional
+
+from .annotation_schema import BYTES_TABLE
+from .annotation_schema import TABLES as ANNOTATION_TABLES
+from .annotation_schema import _catalog as _annotation_catalog
+from .annotation_schema import annotation_schema_error
+
+try:
+    from ..utils.json_strict import dumps_strict
+except ImportError:
+    from utils.json_strict import dumps_strict
+
+SCHEMA_VERSION = 1
+METADATA_TABLE = "media_review_metadata"
+ITEMS_TABLE = "media_review_items"
+DECISIONS_TABLE = "media_review_decisions"
+TABLES = (METADATA_TABLE, ITEMS_TABLE, DECISIONS_TABLE)
+MAX_EVIDENCE_BYTES = 8 * 1024 * 1024
+MAX_CANDIDATES = 16
+# The file's own spelling when it was recorded (bytes of UTF-8 text). Locators
+# are case-folded on Windows, so a target is formatted from this instead.
+MAX_SPELLED_PATH_BYTES = 16384
+ITEM_STATES = ("pending", "selected", "unresolved")
+DECISION_OUTCOMES = ("selected", "unresolved")
+_TIMESTAMP_BYTES = 64
+_HEX64 = "length({c})=64 AND {c} NOT GLOB '*[^0-9a-f]*'"
+_EVIDENCE_CHECK = f"CHECK(length(CAST(evidence AS BLOB)) <= {MAX_EVIDENCE_BYTES})"
+
+_TABLE_DDL = {
+    METADATA_TABLE: (
+        f"CREATE TABLE {METADATA_TABLE} (singleton_id INTEGER PRIMARY KEY "
+        "CHECK(singleton_id=1), schema_version INTEGER NOT NULL "
+        f"CHECK(schema_version={SCHEMA_VERSION}), "
+        "review_index_revision INTEGER NOT NULL DEFAULT 0 "
+        "CHECK(review_index_revision>=0))"
+    ),
+    ITEMS_TABLE: (
+        f"CREATE TABLE {ITEMS_TABLE} (id TEXT PRIMARY KEY CHECK(length(id)=36), "
+        f"byte_id TEXT NOT NULL REFERENCES {BYTES_TABLE}(id) ON DELETE RESTRICT, "
+        "spelled_path TEXT NOT NULL CHECK(length(CAST(spelled_path AS BLOB)) "
+        f"BETWEEN 1 AND {MAX_SPELLED_PATH_BYTES}), "
+        "resolver_version INTEGER NOT NULL, evidence_hash TEXT NOT NULL "
+        f"CHECK({_HEX64.format(c='evidence_hash')}), "
+        f"evidence TEXT NOT NULL {_EVIDENCE_CHECK}, "
+        "candidate_count INTEGER NOT NULL "
+        f"CHECK(candidate_count BETWEEN 0 AND {MAX_CANDIDATES}), "
+        "limited INTEGER NOT NULL CHECK(limited IN (0,1)), "
+        "state TEXT NOT NULL CHECK(state IN ('pending','selected','unresolved')), "
+        "revision INTEGER NOT NULL CHECK(revision>=0), created_at TEXT NOT NULL, "
+        "updated_at TEXT NOT NULL, UNIQUE(byte_id, resolver_version, evidence_hash))"
+    ),
+    DECISIONS_TABLE: (
+        f"CREATE TABLE {DECISIONS_TABLE} (id TEXT PRIMARY KEY "
+        "CHECK(length(id)=36), item_id TEXT NOT NULL "
+        f"REFERENCES {ITEMS_TABLE}(id) ON DELETE RESTRICT, "
+        "decision_revision INTEGER NOT NULL, created_at TEXT NOT NULL, "
+        "outcome TEXT NOT NULL CHECK(outcome IN ('selected','unresolved')), "
+        "candidate_hash TEXT CHECK(candidate_hash IS NULL OR "
+        f"({_HEX64.format(c='candidate_hash')})), "
+        f"evidence TEXT NOT NULL {_EVIDENCE_CHECK}, "
+        "pair_id INTEGER REFERENCES prompt_pairs(id) ON DELETE SET NULL, "
+        "image_id INTEGER REFERENCES generated_images(id) ON DELETE SET NULL, "
+        "UNIQUE(item_id, decision_revision), "
+        "CHECK((outcome='selected') = (candidate_hash IS NOT NULL)))"
+    ),
+}
+_INDEX_DDL = {
+    # The queue reads pending items oldest first with a stable tiebreak.
+    "idx_media_review_items_queue": (
+        "CREATE INDEX idx_media_review_items_queue ON "
+        f"{ITEMS_TABLE}(state, created_at, id)"
+    ),
+    # Deleting a pair or an image nulls decision navigation by index, not scan.
+    "idx_media_review_decisions_pair": (
+        "CREATE INDEX idx_media_review_decisions_pair ON " f"{DECISIONS_TABLE}(pair_id)"
+    ),
+    "idx_media_review_decisions_image": (
+        "CREATE INDEX idx_media_review_decisions_image ON "
+        f"{DECISIONS_TABLE}(image_id)"
+    ),
+}
+# ``sqlite_autoindex_<table>_<n>``: the TEXT primary key and the UNIQUE key.
+_AUTOINDEXES = {ITEMS_TABLE: 2, DECISIONS_TABLE: 2}
+# (name, declared type, not null, default, primary key position)
+_COLUMNS = {
+    METADATA_TABLE: (
+        ("singleton_id", "INTEGER", 0, None, 1),
+        ("schema_version", "INTEGER", 1, None, 0),
+        ("review_index_revision", "INTEGER", 1, "0", 0),
+    ),
+    ITEMS_TABLE: (
+        ("id", "TEXT", 0, None, 1),
+        ("byte_id", "TEXT", 1, None, 0),
+        ("spelled_path", "TEXT", 1, None, 0),
+        ("resolver_version", "INTEGER", 1, None, 0),
+        ("evidence_hash", "TEXT", 1, None, 0),
+        ("evidence", "TEXT", 1, None, 0),
+        ("candidate_count", "INTEGER", 1, None, 0),
+        ("limited", "INTEGER", 1, None, 0),
+        ("state", "TEXT", 1, None, 0),
+        ("revision", "INTEGER", 1, None, 0),
+        ("created_at", "TEXT", 1, None, 0),
+        ("updated_at", "TEXT", 1, None, 0),
+    ),
+    DECISIONS_TABLE: (
+        ("id", "TEXT", 0, None, 1),
+        ("item_id", "TEXT", 1, None, 0),
+        ("decision_revision", "INTEGER", 1, None, 0),
+        ("created_at", "TEXT", 1, None, 0),
+        ("outcome", "TEXT", 1, None, 0),
+        ("candidate_hash", "TEXT", 0, None, 0),
+        ("evidence", "TEXT", 1, None, 0),
+        ("pair_id", "INTEGER", 0, None, 0),
+        ("image_id", "INTEGER", 0, None, 0),
+    ),
+}
+# (column, parent table, parent column, on delete)
+_FOREIGN_KEYS = {
+    METADATA_TABLE: set(),
+    ITEMS_TABLE: {("byte_id", BYTES_TABLE, "id", "RESTRICT")},
+    DECISIONS_TABLE: {
+        ("item_id", ITEMS_TABLE, "id", "RESTRICT"),
+        ("pair_id", "prompt_pairs", "id", "SET NULL"),
+        ("image_id", "generated_images", "id", "SET NULL"),
+    },
+}
+
+
+class _NonCanonical(ValueError):
+    pass
+
+
+def _reject_constant(name: str) -> Any:
+    raise _NonCanonical(f"JSON constant {name} is not allowed")
+
+
+# Built once: validation re-encodes every stored document at startup.
+_ENCODER = json.JSONEncoder(
+    sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False
+)
+_DECODER = json.JSONDecoder(parse_constant=_reject_constant)
+
+
+def canonical_evidence(value: Any) -> str:
+    """Serialize one evidence object canonically; ``ValueError`` when invalid.
+
+    The text is a JSON object with sorted keys, no whitespace, ASCII only and
+    no NaN/infinity, at most ``MAX_EVIDENCE_BYTES`` bytes. A non-finite number
+    (ComfyUI stamps ``NaN`` into graphs) is written as ``null``, the store's
+    policy everywhere (``utils.json_strict.dumps_strict``).
+    """
+    if not isinstance(value, dict):
+        raise ValueError("Review evidence must be a JSON object")
+    try:
+        try:
+            text = _ENCODER.encode(value)
+        except ValueError as error:
+            if "Out of range float" not in str(error):
+                raise
+            text = dumps_strict(
+                value, sort_keys=True, separators=(",", ":"), ensure_ascii=True
+            )
+    except (TypeError, RecursionError) as error:
+        raise ValueError(f"Review evidence is not serializable: {error}") from None
+    if len(text) > MAX_EVIDENCE_BYTES:
+        raise ValueError("Review evidence is larger than 8 MiB")
+    return text
+
+
+def choosable(candidate: Any) -> bool:
+    """True when a candidate offers a prompt to choose: it is ``resolved`` and
+    its positive side carries non-blank text. The page disables any other
+    candidate, so it is never something to choose.
+    """
+    if not isinstance(candidate, dict) or candidate.get("state") != "resolved":
+        return False
+    positive = candidate.get("positive")
+    if not isinstance(positive, dict):
+        return False
+    text = positive.get("text")
+    return isinstance(text, str) and bool(text.strip())
+
+
+# An item is worth a decision only when at least this many candidates offer
+# a prompt; with fewer there is nothing to choose between (maintainer test of
+# #284/#288: 39 items whose candidates were all "unresolved").
+MIN_CHOOSABLE = 2
+
+
+def evidence_offers_choice(evidence: Any) -> bool:
+    """False when stored evidence provably offers nothing to choose.
+
+    Unlimited evidence must list ``MIN_CHOOSABLE`` choosable candidates. A
+    limited item stores none, only ``candidate_total``; it may offer a choice
+    (more than ``MAX_CANDIDATES`` prompts) unless that total is too small.
+    """
+    if not isinstance(evidence, dict):
+        return False
+    # The saver anchor's one pair (#390) is a suggestion to confirm.
+    needed = 1 if evidence.get("anchor") == "saver" else MIN_CHOOSABLE
+    if evidence.get("limited"):
+        total = evidence.get("candidate_total")
+        return type(total) is int and total >= needed
+    candidates = evidence.get("candidates")
+    if not isinstance(candidates, list):
+        return False
+    return sum(1 for c in candidates if choosable(c)) >= needed
+
+
+def evidence_digest(text: str) -> str:
+    """SHA-256 of canonical evidence text (ASCII, hence UTF-8 identical)."""
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def _is_canonical(text: Any) -> bool:
+    if not isinstance(text, str):
+        return False
+    try:
+        value = _DECODER.decode(text)
+        return canonical_evidence(value) == text
+    except (ValueError, RecursionError):
+        return False
+
+
+def _compact(sql: str) -> str:
+    return re.sub(r"\s+", "", sql).lower()
+
+
+def _uuid(column: str) -> str:
+    """SQL: *column* is a lowercase hyphenated 36-character UUID string."""
+    return (
+        f"typeof({column}) = 'text' AND length({column}) = 36 "
+        f"AND substr({column}, 9, 1) = '-' AND substr({column}, 14, 1) = '-' "
+        f"AND substr({column}, 19, 1) = '-' AND substr({column}, 24, 1) = '-' "
+        f"AND length(replace({column}, '-', '')) = 32 "
+        f"AND replace({column}, '-', '') NOT GLOB '*[^0-9a-f]*'"
+    )
+
+
+def _timestamp(column: str) -> str:
+    return (
+        f"typeof({column}) = 'text' "
+        f"AND length(CAST({column} AS BLOB)) <= {_TIMESTAMP_BYTES}"
+    )
+
+
+def _non_negative(column: str) -> str:
+    return f"typeof({column}) = 'integer' AND {column} >= 0"
+
+
+def _hex64(column: str) -> str:
+    return f"typeof({column}) = 'text' AND " + _HEX64.format(c=column)
+
+
+def _evidence(column: str) -> str:
+    # Type and size only: the streamed pass below proves the text canonical
+    # (which implies printable ASCII), so a pattern scan here would read every
+    # document a second time.
+    return (
+        f"typeof({column}) = 'text' "
+        f"AND length(CAST({column} AS BLOB)) BETWEEN 2 AND {MAX_EVIDENCE_BYTES}"
+    )
+
+
+def _nullable_integer(column: str) -> str:
+    return f"({column} IS NULL OR typeof({column}) = 'integer')"
+
+
+def _in(column: str, values: tuple) -> str:
+    return f"typeof({column}) = 'text' AND {column} IN ({repr(values)[1:-1]})"
+
+
+# Every row of a table must satisfy its predicate; one pass each, LIMIT 1.
+_ROW_PREDICATES = {
+    ITEMS_TABLE: " AND ".join(
+        (
+            _uuid("id"),
+            _uuid("byte_id"),
+            "typeof(spelled_path) = 'text' AND length(CAST(spelled_path AS BLOB)) "
+            f"BETWEEN 1 AND {MAX_SPELLED_PATH_BYTES}",
+            "typeof(resolver_version) = 'integer'",
+            _hex64("evidence_hash"),
+            _evidence("evidence"),
+            _non_negative("candidate_count"),
+            f"candidate_count <= {MAX_CANDIDATES}",
+            "typeof(limited) = 'integer' AND limited IN (0, 1)",
+            _in("state", ITEM_STATES),
+            _non_negative("revision"),
+            _timestamp("created_at"),
+            _timestamp("updated_at"),
+        )
+    ),
+    DECISIONS_TABLE: " AND ".join(
+        (
+            _uuid("id"),
+            _uuid("item_id"),
+            _non_negative("decision_revision"),
+            _timestamp("created_at"),
+            _in("outcome", DECISION_OUTCOMES),
+            "(CASE WHEN outcome = 'selected' THEN "
+            f"{_hex64('candidate_hash')} ELSE candidate_hash IS NULL END)",
+            _evidence("evidence"),
+            _nullable_integer("pair_id"),
+            _nullable_integer("image_id"),
+        )
+    ),
+}
+
+
+def _expected_catalog() -> set:
+    objects = {("table", name, _compact(sql)) for name, sql in _TABLE_DDL.items()}
+    objects |= {("index", name, _compact(sql)) for name, sql in _INDEX_DDL.items()}
+    objects |= {
+        ("index", f"sqlite_autoindex_{table}_{n}", "")
+        for table, count in _AUTOINDEXES.items()
+        for n in range(1, count + 1)
+    }
+    return objects
+
+
+def _catalog(conn: sqlite3.Connection) -> list:
+    names = TABLES + tuple(_INDEX_DDL)
+    marks = ",".join("?" * len(names))
+    table_marks = ",".join("?" * len(TABLES))
+    return conn.execute(  # only "?" placeholders are interpolated
+        "SELECT type,name,sql FROM sqlite_master "
+        f"WHERE name IN ({marks}) OR tbl_name IN ({table_marks})",  # nosec B608
+        names + TABLES,
+    ).fetchall()
+
+
+def _structure_error(conn: sqlite3.Connection) -> Optional[str]:
+    for table, expected in _COLUMNS.items():
+        columns = tuple(
+            (row[1], row[2].upper(), row[3], row[4], row[5])
+            for row in conn.execute(f"PRAGMA table_info({table})")
+        )
+        if columns != expected:
+            return f"Review queue table {table} has invalid columns"
+        foreign_keys = {
+            (row[3], row[2], row[4], row[6])
+            for row in conn.execute(f"PRAGMA foreign_key_list({table})")
+        }
+        if foreign_keys != _FOREIGN_KEYS[table]:
+            return f"Review queue table {table} has invalid foreign keys"
+        if conn.execute(f"PRAGMA foreign_key_check({table})").fetchone():
+            return f"Review queue table {table} contains foreign key violations"
+    return None
+
+
+def _evidence_error(conn: sqlite3.Connection) -> Optional[str]:
+    """Stream every evidence document once; one row is held at a time.
+
+    Rows are read with ``fetchone``: iterating a managed connection's cursor
+    would prefetch ``ITER_CHUNK`` (128) rows (``database/managed_connection.py``).
+    """
+    items = conn.execute(
+        f"SELECT evidence_hash, evidence FROM {ITEMS_TABLE}"  # nosec B608
+    )
+    while (row := items.fetchone()) is not None:
+        evidence_hash, text = row[0], row[1]
+        if not _is_canonical(text) or evidence_digest(text) != evidence_hash:
+            return f"Review queue table {ITEMS_TABLE} contains invalid evidence"
+    decisions = conn.execute(f"SELECT evidence FROM {DECISIONS_TABLE}")  # nosec B608
+    while (row := decisions.fetchone()) is not None:
+        text = row[0]
+        if not _is_canonical(text):
+            return f"Review queue table {DECISIONS_TABLE} contains invalid evidence"
+    return None
+
+
+def _value_error(conn: sqlite3.Connection) -> Optional[str]:
+    # Only this module's constants are interpolated into the SQL below.
+    rows = conn.execute(
+        "SELECT COUNT(*), MIN(typeof(singleton_id) = 'integer' "  # nosec B608
+        "AND singleton_id = 1 AND typeof(schema_version) = 'integer' "
+        f"AND schema_version = {SCHEMA_VERSION} "
+        f"AND {_non_negative('review_index_revision')}) FROM {METADATA_TABLE}"
+    ).fetchone()
+    if rows[0] != 1 or rows[1] != 1:
+        return "Review queue metadata is invalid"
+    for table, predicate in _ROW_PREDICATES.items():
+        if conn.execute(
+            f"SELECT 1 FROM {table} WHERE NOT ({predicate}) LIMIT 1"  # nosec B608
+        ).fetchone():
+            return f"Review queue table {table} contains invalid values"
+    return _evidence_error(conn)
+
+
+def review_without_annotations_error(conn: sqlite3.Connection) -> Optional[str]:
+    """Catalog only: a review group present without its annotation tables.
+
+    Startup asks this before the annotation group installs, which would make
+    a review group copied without it look valid. It reads no rows; the full
+    validation runs once in :func:`migrate_review_state`.
+    """
+    if not _catalog(conn):
+        return None
+    present = {
+        name for kind, name, _sql in _annotation_catalog(conn) if kind == "table"
+    }
+    if present != set(ANNOTATION_TABLES):
+        return (
+            "Review queue schema requires a valid image annotation schema: "
+            "Missing image annotation schema"
+        )
+    return None
+
+
+def review_schema_error(
+    conn: sqlite3.Connection,
+    *,
+    allow_missing: bool = True,
+    annotations_valid: bool = False,
+) -> Optional[str]:
+    """Reject partial or unsupported review schemas and invalid rows.
+
+    Reads only: nothing is ever repaired. A database without any of the
+    group's objects is acceptable for backups that predate it unless
+    ``allow_missing`` is false. A present group requires a present, valid
+    image annotation group; ``annotations_valid`` says the caller has just
+    validated it (startup's annotation install), so only its presence is
+    checked again.
+    """
+    objects = _catalog(conn)
+    if not objects:
+        return None if allow_missing else "Missing review queue schema"
+    if annotations_valid:
+        annotation_error = review_without_annotations_error(conn)
+    else:
+        annotation_error = annotation_schema_error(conn, allow_missing=False)
+        if annotation_error:
+            annotation_error = (
+                "Review queue schema requires a valid image annotation schema: "
+                + annotation_error
+            )
+    if annotation_error:
+        return annotation_error
+    present = {name for kind, name, _sql in objects if kind == "table"}
+    if present != set(TABLES):
+        return "Incomplete review queue schema"
+    actual = {(kind, name, _compact(sql or "")) for kind, name, sql in objects}
+    if actual != _expected_catalog():
+        return "Review queue has an invalid schema definition"
+    return _structure_error(conn) or _value_error(conn)
+
+
+def migrate_review_state(conn: sqlite3.Connection) -> None:
+    """Install the group atomically; validate before and after creating it.
+
+    Runs after ``migrate_annotation_state`` (which has just validated the
+    annotation group): the group is only valid beside the annotation group it
+    cites. A present group is validated once, evidence included.
+    """
+    conn.execute("SAVEPOINT media_review_schema")
+    try:
+        installed = bool(_catalog(conn))
+        error = review_schema_error(conn, annotations_valid=True)
+        if error:
+            raise sqlite3.DatabaseError(error)
+        if installed:
+            # Already complete and valid: nothing to create, and the evidence
+            # pass is the expensive part of validation, so it runs once.
+            conn.execute("RELEASE media_review_schema")
+            return
+        for sql in _TABLE_DDL.values():
+            conn.execute(sql.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
+        for sql in _INDEX_DDL.values():
+            conn.execute(sql.replace("INDEX", "INDEX IF NOT EXISTS", 1))
+        conn.execute(
+            f"INSERT OR IGNORE INTO {METADATA_TABLE} "
+            "(singleton_id, schema_version, review_index_revision) VALUES (1, ?, 0)",
+            (SCHEMA_VERSION,),
+        )
+        error = review_schema_error(conn, allow_missing=False, annotations_valid=True)
+        if error:
+            raise sqlite3.DatabaseError(error)
+        conn.execute("RELEASE media_review_schema")
+    except BaseException:
+        conn.execute("ROLLBACK TO media_review_schema")
+        conn.execute("RELEASE media_review_schema")
+        raise

database/run_attribution.py

@@ -0,0 +1,369 @@
+"""Attribute an output file to a stored prompt through recorded history (#342).
+
+Execution history records, per run, the files ComfyUI reported
+(``execution_run_outputs``) and the PromptManager texts the run used
+(``execution_run_contexts``, whose ``positive_prompt_id`` hint is set only
+while the raw text matches exactly one stored prompt). A file is attributed
+only when all of this holds:
+
+- exactly one recorded run reported the file's output location (same output
+  root, subfolder and name);
+- the file was last written while that run was executing, from its dequeue
+  to its terminal event (else the finish callback): a later run that history
+  did not record may have reused the name;
+- every positive context of that run names the same stored prompt.
+
+Anything else returns None: the caller leaves the file unlinked. Read-only.
+
+``recorded_output`` applies the first two rules and returns, for the
+inspector (#386), the one output node (save node) that run reported the file
+from and the texts PromptManager recorded at execution per sampler.
+
+Lookups are batched (``stored_run_prompts``): one parameterized query per
+chunk of at most ``LOOKUP_CHUNK`` distinct file names, matched in memory. The
+history schema is left exactly as it is (no extra index), so older builds can
+still open the library.
+"""
+
+import logging
+import os
+import sqlite3
+import threading
+import weakref
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, Iterable, List, Optional
+
+try:
+    from ..utils.execution_outputs import snapshot_output_roots
+    from ..utils.library_location import import_folder_paths
+except ImportError:
+    from utils.execution_outputs import snapshot_output_roots
+    from utils.library_location import import_folder_paths
+
+# File times may trail or lead the recorded run times a little (coarse file
+# system clocks, the write finishing just after the host reported it).
+MTIME_SLACK_MS = 2000
+
+# Distinct file names per lookup query: well under SQLite's bound-parameter
+# limit (999 on old builds), so a scan batch costs one query per chunk.
+LOOKUP_CHUNK = 500
+
+_RUNS_FOR_NAMES = (
+    "SELECT DISTINCT o.filename, o.subfolder, o.root_key, o.path_style, o.run_id, "
+    "r.observed_dequeue_at, COALESCE(r.terminal_event_at, r.observed_callback_at), "
+    "o.output_node_id "
+    "FROM execution_run_outputs o JOIN execution_runs r ON r.run_id = o.run_id "
+    "WHERE o.folder_type = 'output' AND o.filename IN ({marks})"
+)
+
+
+def host_output_directory() -> Optional[str]:
+    """ComfyUI's output directory, or None outside ComfyUI."""
+    folder_paths = import_folder_paths()
+    if folder_paths is None:
+        return None
+    try:
+        value = folder_paths.get_output_directory()
+    except Exception:
+        return None
+    return value if isinstance(value, str) and value else None
+
+
+def _inside(image_path: str, output_root: str) -> Optional[str]:
+    """``image_path`` relative to ``output_root``, or None when outside it."""
+    try:
+        relative = os.path.relpath(image_path, output_root)
+    except ValueError:  # another drive on Windows
+        return None
+    if os.path.isabs(relative) or relative == os.pardir:
+        return None
+    if relative.startswith(os.pardir + os.sep):
+        return None
+    return relative
+
+
+def _resolved(path: str) -> str:
+    try:
+        return os.path.realpath(path)
+    except (OSError, ValueError):
+        return os.path.abspath(path)
+
+
+def _location(image_path: str, output_root: str, path_style: str):
+    """(root key, subfolder, filename) as history records them, or None.
+
+    The root key is always the host's own spelling of its output folder,
+    because history recorded that one. The file may be spelled through the
+    same folder or through its resolved path (a symlinked or junction output
+    folder, as scan roots are): the lexical spelling is tried first, then
+    both sides resolved.
+    """
+    root_key = snapshot_output_roots(path_style, output_root, output_root).output_key
+    if root_key is None:
+        return None
+    relative = _inside(os.path.abspath(image_path), os.path.abspath(output_root))
+    if relative is None:
+        relative = _inside(_resolved(image_path), _resolved(output_root))
+    if relative is None:
+        return None
+    subfolder, filename = os.path.split(relative)
+    if not filename:
+        return None
+    return root_key, subfolder, filename
+
+
+def in_output_folder(image_path: str, output_root: Optional[str]) -> bool:
+    """Whether history could ever attribute ``image_path`` (inside the host's
+    output folder, in either spelling)."""
+    if not image_path or not output_root:
+        return False
+    style = "windows" if os.name == "nt" else "posix"
+    return _location(image_path, output_root, style) is not None
+
+
+def _written_during(image_path: str, started: Any, finished: Any) -> bool:
+    if not isinstance(started, int) or not isinstance(finished, int):
+        return False
+    try:
+        written = os.stat(image_path).st_mtime_ns // 1_000_000
+    except OSError:
+        return False
+    return started - MTIME_SLACK_MS <= written <= finished + MTIME_SLACK_MS
+
+
+def stored_run_prompt(
+    db: Any,
+    image_path: str,
+    output_root: Optional[str],
+    path_style: Optional[str] = None,
+) -> Optional[int]:
+    """The stored prompt id history attributes to ``image_path``, else None."""
+    if not image_path:
+        return None
+    return stored_run_prompts(db, [image_path], output_root, path_style).get(image_path)
+
+
+def _runs_by_location(db: Any, names: List[str]) -> Optional[Dict[tuple, dict]]:
+    """``{(filename, subfolder, root_key, style): {run_id: (start, end, nodes)}}``.
+
+    ``nodes`` is the set of output node ids that reported the location in that
+    run. One query per ``LOOKUP_CHUNK`` names; None without history tables.
+    """
+    found: Dict[tuple, dict] = {}
+    conn = db.model.get_connection()
+    for start in range(0, len(names), LOOKUP_CHUNK):
+        chunk = names[start : start + LOOKUP_CHUNK]
+        query = _RUNS_FOR_NAMES.format(marks=",".join("?" for _ in chunk))
+        try:
+            rows = conn.execute(query, chunk).fetchall()  # nosec B608
+        except sqlite3.Error:
+            return None  # no history tables (a library without history)
+        for name, subfolder, root_key, style, run_id, began, ended, node in rows:
+            runs = found.setdefault((name, subfolder, root_key, style), {})
+            runs.setdefault(run_id, (began, ended, set()))[2].add(node)
+    return found
+
+
+def recorded_runs(
+    db: Any,
+    image_paths: Iterable[str],
+    output_root: Optional[str],
+    path_style: Optional[str],
+) -> Dict[str, tuple]:
+    """``{path: (run_id, output node ids)}`` for each file history attributes.
+
+    A file is attributed when exactly one recorded run reported its location
+    and it was last written while that run executed.
+    """
+    paths = list(dict.fromkeys(path for path in image_paths if path))
+    if not paths or not output_root:
+        return {}
+    style = path_style or ("windows" if os.name == "nt" else "posix")
+    located = {}
+    for path in paths:
+        location = _location(path, output_root, style)
+        if location is not None:
+            located[path] = location
+    if not located:
+        return {}
+    names = sorted({filename for _key, _sub, filename in located.values()})
+    runs = _runs_by_location(db, names)
+    if not runs:
+        return {}
+    attributed = {}
+    for path, (root_key, subfolder, filename) in located.items():
+        reported = runs.get((filename, subfolder, root_key, style), {})
+        if len(reported) != 1:
+            continue
+        ((run_id, (began, ended, nodes)),) = reported.items()
+        if _written_during(path, began, ended):
+            attributed[path] = (run_id, nodes)
+    return attributed
+
+
+def named_prompt(db: Any, run_id: str) -> Optional[int]:
+    """The one stored prompt every positive context of ``run_id`` names, else None."""
+    contexts = db.get_execution_contexts(run_id).get("contexts") or []
+    named = {
+        context.get("positive_prompt_id")
+        for context in contexts
+        if context.get("positive") is not None
+    }
+    if len(named) != 1:
+        return None
+    (prompt_id,) = named
+    return prompt_id if isinstance(prompt_id, int) else None
+
+
+def stored_run_prompts(
+    db: Any,
+    image_paths: Iterable[str],
+    output_root: Optional[str],
+    path_style: Optional[str] = None,
+) -> Dict[str, Optional[int]]:
+    """``{path: stored prompt id or None}`` for many files in few queries.
+
+    The output lookup is one query per ``LOOKUP_CHUNK`` distinct names; each
+    run that alone reported a file is then read once for its contexts.
+    """
+    paths = list(dict.fromkeys(path for path in image_paths if path))
+    result: Dict[str, Optional[int]] = dict.fromkeys(paths)
+    prompts: Dict[str, Optional[int]] = {}
+    for path, (run_id, _nodes) in recorded_runs(
+        db, paths, output_root, path_style
+    ).items():
+        if run_id not in prompts:
+            prompts[run_id] = named_prompt(db, run_id)
+        result[path] = prompts[run_id]
+    return result
+
+
+@dataclass(frozen=True)
+class RecordedOutput:
+    """What recorded history knows about the run that wrote one file (#386).
+
+    ``saver_id`` is the one output node the run reported the file from (None
+    when several did). ``texts`` maps each sampler anchor of the run to the
+    effective texts PromptManager recorded at execution for its roles
+    (``{"positive": text or None, "negative": text or None}``); a role whose
+    text was only inferred from the graph is None.
+    """
+
+    saver_id: Optional[str]
+    texts: Dict[str, Dict[str, Optional[str]]]
+
+
+def _recorded_texts(db: Any, run_id: str) -> Dict[str, Dict[str, Optional[str]]]:
+    texts = {}
+    for context in db.get_execution_contexts(run_id).get("contexts") or []:
+        evidence = context.get("evidence") or {}
+        roles = {}
+        for role in ("positive", "negative"):
+            values = context.get(role) or {}
+            proof = (evidence.get(role) or {}).get("effective") or {}
+            value = values.get("effective")
+            recorded = proof.get("provenance") == "recorded"
+            roles[role] = value if recorded and isinstance(value, str) else None
+        texts[context["anchor_id"]] = roles
+    return texts
+
+
+def recorded_output(
+    db: Any,
+    image_path: str,
+    output_root: Optional[str],
+    path_style: Optional[str] = None,
+) -> Optional[RecordedOutput]:
+    """The recorded run facts for ``image_path``, else None (#386).
+
+    The file must be attributed to one run (``recorded_runs``: one run
+    reported its location and it was written during that run). Read-only; it
+    never links anything.
+    """
+    if not image_path:
+        return None
+    conn = db.model.get_connection()
+    own_transaction = not conn.in_transaction
+    if own_transaction:  # outputs and texts from one state of history
+        conn.execute("BEGIN")
+    try:
+        found = recorded_runs(db, [image_path], output_root, path_style)
+        if image_path not in found:
+            return None
+        run_id, nodes = found[image_path]
+        saver = next(iter(nodes)) if len(nodes) == 1 else None
+        return RecordedOutput(saver, _recorded_texts(db, run_id))
+    finally:
+        if own_transaction:
+            conn.rollback()
+
+
+_LOGGER = logging.getLogger("prompt_manager.run_attribution")
+
+
+def recorded_output_lookup(
+    db: Any, *, close: bool = False, reference: Optional[dict] = None
+) -> Callable[[Any], Any]:
+    """``lookup(path) -> RecordedOutput or None`` for the inspect routes (#386).
+
+    Uses ComfyUI's output folder. Any failure (no history, a locked library)
+    is logged and names no node, so the file is judged from its own metadata.
+    ``close`` closes the calling thread's connection afterwards, for routes
+    that close theirs after reading. ``reference`` (a history occurrence's
+    output reference) spells the file as history recorded it: the route's
+    target is case-normalized on Windows, and history matches names exactly.
+    """
+
+    def lookup(path: Any) -> Optional[RecordedOutput]:
+        root = host_output_directory()
+        if reference is not None:
+            if reference.get("folder_type") != "output" or not root:
+                return None
+            path = os.path.join(
+                root, reference.get("subfolder") or "", reference.get("filename") or ""
+            )
+        try:
+            return recorded_output(db, str(path), root)
+        except (sqlite3.Error, OSError, ValueError):
+            _LOGGER.warning("Recorded run lookup failed", exc_info=True)
+            return None
+        except Exception:
+            _LOGGER.exception("Recorded run lookup failed unexpectedly")
+            return None
+        finally:
+            if close:
+                db.model.close()
+
+    return lookup
+
+
+_SEALED_LISTENERS: List[weakref.WeakMethod] = []
+_SEALED_LOCK = threading.Lock()
+
+
+def subscribe_outputs_sealed(method: Callable[[str, List[dict]], None]) -> None:
+    """Call bound ``method(run_id, references)`` after a run's outputs are recorded.
+
+    Held weakly: a listener that is garbage collected is dropped.
+    """
+    with _SEALED_LOCK:
+        _SEALED_LISTENERS.append(weakref.WeakMethod(method))
+
+
+def notify_outputs_sealed(run_id: str, references: Iterable[dict]) -> None:
+    """Tell listeners a run's output references were just committed.
+
+    A listener failure is logged and never reaches the history writer.
+    """
+    references = list(references)
+    with _SEALED_LOCK:
+        alive = [ref for ref in _SEALED_LISTENERS if ref() is not None]
+        _SEALED_LISTENERS[:] = alive
+        listeners = [ref() for ref in alive]
+    for listener in listeners:
+        if listener is None:
+            continue
+        try:
+            listener(run_id, references)
+        except Exception:
+            _LOGGER.warning("Output seal listener failed", exc_info=True)

database/safety_backup.py

@@ -0,0 +1,185 @@
+"""Safety backups beside the live database: one writer for every caller.
+
+A safety backup is ``<db>.<label>_<YYYYmmdd_HHMMSS_ffffff>[_n]`` in the
+library's own folder. It is written by SQLite's backup API into a temp file
+beside the database (``SAFETY_TEMP_PREFIX``, swept at startup after a crash),
+made one self-contained file (rollback journal), optionally verified, fsynced,
+made private (``0o600`` on POSIX) and only then given its unique name, so a
+backup name never carries a half-written file. The names are unique within
+one second (microseconds, then a counter): on Windows two backups in the same
+second must not overwrite each other.
+
+Maintenance applies take a verified backup (``py/api/maintenance.py``); a
+restore takes an unverified one, because the database it replaces may be the
+damaged one the user is recovering from (``database/restore_stage.py``).
+"""
+
+import datetime
+import os
+import sqlite3
+import tempfile
+from contextlib import closing
+
+SAFETY_TEMP_PREFIX = ".pm-safety-"
+_NAME_ATTEMPTS = 1000
+
+
+class SafetyBackupFailed(Exception):
+    """The safety backup could not be written, verified or put in place.
+
+    ``reason`` is the verify reason when the copy did not verify.
+    """
+
+    def __init__(self, message, reason=None):
+        super().__init__(message)
+        self.reason = reason
+
+
+def _now():
+    return datetime.datetime.now()
+
+
+def remove_quietly(*paths):
+    """Remove each existing path, ignoring errors (cleanup after a failure)."""
+    for path in paths:
+        try:
+            if path and os.path.exists(path):
+                os.remove(path)
+        except OSError:
+            pass
+
+
+def backup_names(db_path, label="backup"):
+    """Candidate names, unique within one second (microseconds, then ``_n``)."""
+    base = f"{db_path}.{label}_{_now().strftime('%Y%m%d_%H%M%S_%f')}"
+    for attempt in range(_NAME_ATTEMPTS):
+        yield base if attempt == 0 else f"{base}_{attempt}"
+
+
+def reserve_backup_name(db_path, label="backup"):
+    """Create and return an unused ``<db>.<label>_<stamp>[_n]`` (O_EXCL)."""
+    for candidate in backup_names(db_path, label):
+        try:
+            fd = os.open(candidate, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
+        except FileExistsError:
+            continue
+        os.close(fd)
+        return candidate
+    raise SafetyBackupFailed("No free backup name")
+
+
+def place_backup(temp, db_path, label="backup"):
+    """Give the finished ``temp`` its unique backup name; return that path.
+
+    ``os.link`` is atomic and refuses an existing name, so no empty or half
+    file ever carries a backup name, even across a crash. A filesystem
+    without hard links falls back to reserving the name (``O_EXCL``) and
+    ``os.replace``. The temp file is removed by the caller.
+    """
+    for candidate in backup_names(db_path, label):
+        try:
+            os.link(temp, candidate)
+        except FileExistsError:
+            continue
+        except (OSError, NotImplementedError):
+            break  # no hard links here
+        else:
+            return candidate
+    else:
+        raise SafetyBackupFailed("No free backup name")
+    final = reserve_backup_name(db_path, label)
+    try:
+        os.replace(temp, final)
+    except OSError:
+        remove_quietly(final)
+        raise
+    return final
+
+
+def sweep_safety_temps(db_path):
+    """Remove the temp copies an interrupted safety backup left (blocking).
+
+    Called once at startup, before any job can take a backup. Returns the
+    names removed.
+    """
+    folder = os.path.dirname(os.path.abspath(db_path))
+    removed = []
+    try:
+        names = os.listdir(folder)
+    except OSError:
+        return removed
+    for name in names:
+        if not (name.startswith(SAFETY_TEMP_PREFIX) and ".tmp" in name):
+            continue
+        try:
+            os.remove(os.path.join(folder, name))
+            removed.append(name)
+        except OSError:
+            continue
+    return removed
+
+
+def make_standalone(path):
+    """Make the copy one self-contained file (rollback journal, not WAL).
+
+    The backup API copies the live database's WAL flag; a WAL file opened
+    read-only later (a verify, a restore preview) would leave ``-wal`` and
+    ``-shm`` sidecars beside it.
+    """
+    try:
+        with closing(sqlite3.connect(path)) as conn:
+            conn.execute("PRAGMA journal_mode=DELETE").fetchone()
+    except sqlite3.Error as exc:
+        raise SafetyBackupFailed("The backup could not be finished") from exc
+
+
+def fsync_file(path):
+    with open(path, "rb+") as handle:
+        os.fsync(handle.fileno())
+
+
+def safety_temp(db_path):
+    """A new, closed temp file beside the database (``SAFETY_TEMP_PREFIX``)."""
+    fd, temp = tempfile.mkstemp(
+        dir=os.path.dirname(os.path.abspath(db_path)),
+        prefix=SAFETY_TEMP_PREFIX,
+        suffix=".tmp",
+    )
+    os.close(fd)
+    return temp
+
+
+def take_safety_backup(model, label="backup", *, verify=True):
+    """Write (and by default verify) a safety backup; return its path (blocking).
+
+    The copy is made in a temp file beside the database and only renamed
+    onto its final, unique name ``<db>.<label>_<stamp>`` once it is complete
+    (and verified), so a backup file on disk is always a whole one. ``label``
+    "backup" joins the pruned pool; another label (the workflow storage
+    migration's ``pre-workflow-storage``) is kept until the user removes it.
+    Raises SafetyBackupFailed, leaving no file.
+    """
+    db_path = os.path.abspath(model.db_path)
+    temp = safety_temp(db_path)
+    final = None
+    try:
+        if not model.backup_database(temp):
+            raise SafetyBackupFailed("The backup could not be written")
+        make_standalone(temp)
+        if verify:
+            ok, reason = model.verify_database_file(temp)
+            if not ok:
+                raise SafetyBackupFailed("The backup did not verify", reason)
+        fsync_file(temp)
+        if os.name != "nt":
+            os.chmod(temp, 0o600)
+        final = place_backup(temp, db_path, label)
+        return final
+    except OSError as exc:
+        remove_quietly(final)
+        raise SafetyBackupFailed("The backup could not be put in place") from exc
+    except SafetyBackupFailed:
+        remove_quietly(final)
+        raise
+    finally:
+        remove_quietly(temp, temp + "-journal", temp + "-wal", temp + "-shm")

database/saved_views.py

@@ -33,8 +33,14 @@ MAX_TAG_LENGTH = 50
 _IMAGE_SORTS = ("newest", "oldest", "name", "name-desc", "rating")
 _LIBRARY_SORTS = ("newest", "oldest", "name", "rating")
 _MEDIA = ("all", "image", "video")
+# How a Library view's Tags match (#444): every tag, any of them, or parents
+# without tags ("untagged" never goes with tags; the listing refuses both).
+_TAG_MODES = ("all", "any", "untagged")
 _CONTROL = re.compile(r"[\x00-\x1f\x7f]")
 _SOURCE = re.compile(r"all|0|[1-9][0-9]{0,8}")
+# A collection id as the listing's ``collection`` query takes it (#331): none
+# ("") or 1 to 9 digits without a leading zero.
+_COLLECTION = re.compile(r"|[1-9][0-9]{0,8}")
 FALLBACK_NAME = "Saved view"
 
 
@@ -55,12 +61,18 @@ _STATE_FIELDS = {
         "tags": ("tags", None, ()),
         # The Favorites page's filter (#249 D51): a view saved there reopens it.
         "favorites": ("bool", None, False),
+        # A view saved on a collection page (#331 D140) reopens that
+        # collection; never together with favorites.
+        "collection": ("collection", None, ""),
     },
     "library": {
         "q": ("text", 500, ""),
         "category": ("text", 100, ""),
         "sort": ("choice", _LIBRARY_SORTS, "newest"),
         "tags": ("tags", None, ()),
+        # #444: the Tags match and the output folder (relative, "/"-joined).
+        "tag_mode": ("choice", _TAG_MODES, "all"),
+        "folder": ("text", 1024, ""),
     },
 }
 
@@ -125,6 +137,8 @@ def _validate_value(key: str, kind: str, limit, value: object) -> None:
         raise SavedViewInvalid(f"State '{key}' is not a supported value")
     if kind == "source" and not _SOURCE.fullmatch(value):
         raise SavedViewInvalid(f"State '{key}' must be 'all' or a source index")
+    if kind == "collection" and not _COLLECTION.fullmatch(value):
+        raise SavedViewInvalid(f"State '{key}' must be empty or a collection id")
     if kind == "text" and (len(value) > limit or _CONTROL.search(value)):
         raise SavedViewInvalid(f"State '{key}' is too long or has control characters")
 
@@ -142,6 +156,10 @@ def encode_state(page: object, state: object) -> str:
         )
     for key, (kind, limit, _default) in fields.items():
         _validate_value(key, kind, limit, state[key])
+    if state.get("favorites") is True and state.get("collection"):
+        raise SavedViewInvalid("A view cannot filter by favorites and a collection")
+    if state.get("tag_mode") == "untagged" and state.get("tags"):
+        raise SavedViewInvalid("A view cannot list untagged prompts and filter tags")
     encoded = json.dumps(
         {key: state[key] for key in fields}, ensure_ascii=False, separators=(",", ":")
     )
@@ -181,6 +199,13 @@ def normalize_state(page: str, raw: object) -> dict:
         except SavedViewInvalid:
             value = default
         state[key] = list(value) if kind == "tags" else value
+    # Never written by this build; a stored row with both opens the collection,
+    # as the listing does for a URL with both (#331 D140).
+    if state.get("favorites") is True and state.get("collection"):
+        state["favorites"] = False
+    # Never written by this build either: the tags win.
+    if state.get("tag_mode") == "untagged" and state.get("tags"):
+        state["tag_mode"] = "all"
     return state
 
 

database/scanned_prompts.py

@@ -0,0 +1,87 @@
+"""Link an output file to the prompt of one text, creating it when new (#247).
+
+The store step of ingestion from file metadata when the decision is a
+positive text (``utils.file_prompt``). The prompt insert and the link run in
+one ``BEGIN IMMEDIATE`` transaction: two writers never store one text twice,
+a failed link leaves no new prompt behind, and a file stays linked to one
+prompt (a file already linked to another prompt is a ``conflict``).
+"""
+
+import logging
+import os
+from typing import Any, Dict, Optional, Tuple
+
+from .pair_ingestion import Marks, file_link_rows, link_conflict, mark_created
+
+try:
+    from ..utils.hashing import generate_prompt_hash
+except ImportError:
+    from utils.hashing import generate_prompt_hash
+
+_LOGGER = logging.getLogger("prompt_manager.database.scanned_prompts")
+
+
+def store_scanned_prompt_or_raise(
+    db: Any, path: str, canonical: Dict[str, Any], text: str, marks: Marks
+) -> Tuple[str, Optional[int]]:
+    """``(status, prompt id)`` after linking ``path`` to ``text``'s prompt.
+
+    ``added`` (prompt created, carrying ``marks``), ``linked`` (stored text
+    reused), ``unchanged`` (already linked to it) or ``conflict`` (linked to
+    another prompt; nothing written, id None). Database errors are raised
+    and roll the whole store back, so a caller can run it again.
+    """
+    with db._pair_transaction() as conn:
+        rows = file_link_rows(conn, path)
+        if link_conflict(rows, generate_prompt_hash(text)):
+            return "conflict", None
+        if rows:
+            return "unchanged", rows[0]["prompt_id"]
+        prompt_id, created = db._pair_component_outcome(conn, text)
+        if created:
+            mark_created(conn, db, prompt_id, marks, os.path.basename(path))
+        db._insert_image_link(conn, prompt_id, path, canonical)
+        return ("added" if created else "linked"), prompt_id
+
+
+def store_scanned_prompt(
+    db: Any, path: str, canonical: Dict[str, Any], text: str, marks: Marks
+) -> Tuple[str, Optional[int]]:
+    """The scan's form: an error is logged and reported as ``("found", None)``.
+
+    A write that failed because Move library switched runs again on the new
+    library (``survive_library_switch``); the rerun re-checks the link rows.
+    """
+    try:
+        return db.survive_library_switch(
+            lambda: store_scanned_prompt_or_raise(db, path, canonical, text, marks)
+        )
+    except Exception as error:
+        _LOGGER.error(
+            f"Failed to store the prompt of {os.path.basename(path)}: {error}"
+        )
+        return "found", None
+
+
+def link_stored_prompt_or_raise(
+    db: Any, path: str, canonical: Dict[str, Any], prompt_id: int
+) -> Optional[str]:
+    """Link ``path`` to the stored prompt ``prompt_id`` (recorded history, #342).
+
+    ``linked``, ``unchanged`` (already linked to it), ``conflict`` (linked to
+    another prompt; nothing written) or None when the prompt is gone. One
+    ``BEGIN IMMEDIATE`` transaction; database errors are raised.
+    """
+    with db._pair_transaction() as conn:
+        rows = file_link_rows(conn, path)
+        if len(rows) > 1 or (rows and rows[0]["prompt_id"] != prompt_id):
+            return "conflict"
+        if rows:
+            return "unchanged"
+        exists = conn.execute(
+            "SELECT 1 FROM prompts WHERE id = ?", (prompt_id,)
+        ).fetchone()
+        if exists is None:
+            return None
+        db._insert_image_link(conn, prompt_id, path, canonical)
+        return "linked"

database/vocabulary.py

@@ -13,12 +13,15 @@ same connection before commit. Owner rows and their timestamps never change.
 
 Category consolidation is previewed in one ``library_snapshot`` and applied in
 one ``_pair_transaction`` that recomputes the owner projection and requires the
-preview's ``impact_token``; only rows carrying the exact source change.
+preview's ``impact_token``; only rows carrying the exact source change. Image
+rows holding the source (``image_categories``, #332) are owners too: they are
+counted, hashed into the token and renamed with the prompts and pairs.
 """
 
 import sqlite3
 
-from .library import _ELIGIBLE, _limit
+from .library import _ELIGIBLE, SHELVED_PAIR, UNTAGGED, _escape, _limit
+from .prompt_archive import SHELVED
 from .library_commands import _TOKEN, LibraryCommandError, _validate_epoch
 from .library_tokens import canonical_token
 from .models import utc_now_iso
@@ -36,30 +39,76 @@ _IMAGE_FILES = (
 )
 
 
-def _tag_rows(page: str) -> str:
+def _tag_rows(page: str, order: str = "page.id") -> str:
     """Count rows for the registry rows selected by ``page`` (id, name)."""
-    # Eligible parent components carrying a tag in the page, plus the
-    # positive parent of every active pair carrying it (always eligible).
+    # Eligible parent components on the shelf carrying a tag in the page,
+    # plus the positive parent of every active pair carrying it (always
+    # eligible) when that parent is on the shelf: archived parents (#443)
+    # are hidden with their pairs, so Browse lists what was counted.
     return (
         "WITH page AS (" + page + "),"
         "owners(tag_id,pid) AS ("
         "SELECT pt.tag_id,p.id FROM prompts p "
         "JOIN prompt_tags pt ON pt.prompt_id=p.id "
         "WHERE pt.tag_id IN (SELECT id FROM page) AND " + _ELIGIBLE + " "
+        "AND " + SHELVED + " "
         "UNION SELECT ppt.tag_id,pp.positive_prompt_id FROM prompt_pairs pp "
         "JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id "
-        "WHERE pp.archived=0 AND ppt.tag_id IN (SELECT id FROM page)),"
+        "WHERE pp.archived=0 AND " + SHELVED_PAIR + " "
+        "AND ppt.tag_id IN (SELECT id FROM page)),"
         "images(tag_id,n) AS (" + _IMAGE_FILES + " "
         "WHERE mit.tag_id IN (SELECT id FROM page) GROUP BY mit.tag_id) "
         "SELECT page.id,page.name,COUNT(DISTINCT owners.pid) AS prompt_count,"
         "COALESCE(MAX(images.n),0) AS image_count "
         "FROM page LEFT JOIN owners ON owners.tag_id=page.id "
         "LEFT JOIN images ON images.tag_id=page.id "
-        "GROUP BY page.id ORDER BY page.id"
+        "GROUP BY page.id ORDER BY " + order
     )
 
 
 _TAG_PAGE = _tag_rows("SELECT id,name FROM tags WHERE id>? ORDER BY id LIMIT ?")
+# #444 tag sorts: sort -> (key expression on tags/counted, direction). Ties and
+# "created" follow the id, so every order is total and keyset pages are exact.
+TAG_SORTS = {
+    "created": (None, "ASC"),
+    "name": ("name COLLATE NOCASE", "ASC"),
+    "name-desc": ("name COLLATE NOCASE", "DESC"),
+    "prompts": ("prompt_count", "DESC"),
+    "prompts-asc": ("prompt_count", "ASC"),
+}
+_MAX_SEARCH = 200
+_TAG_SEARCH = "name LIKE ? ESCAPE '\\'"
+# Every matching tag ranked by the prompt count its row shows (the owners rule
+# of ``_tag_rows``), then one keyset page of it with its image counts.
+_RANKED_TAG_PAGE = (
+    "WITH owners(tag_id,pid) AS ("
+    "SELECT pt.tag_id,p.id FROM prompts p "
+    "JOIN prompt_tags pt ON pt.prompt_id=p.id WHERE " + _ELIGIBLE + " "
+    "AND " + SHELVED + " "
+    "UNION SELECT ppt.tag_id,pp.positive_prompt_id FROM prompt_pairs pp "
+    "JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id WHERE pp.archived=0 "
+    "AND " + SHELVED_PAIR + "),"
+    "counted AS (SELECT t.id,t.name,COUNT(DISTINCT o.pid) AS prompt_count "
+    "FROM tags t LEFT JOIN owners o ON o.tag_id=t.id WHERE {search} "
+    "GROUP BY t.id),"
+    "page AS (SELECT id,name,prompt_count FROM counted WHERE {after} "
+    "ORDER BY prompt_count {direction},id LIMIT ?),"
+    "images(tag_id,n) AS (" + _IMAGE_FILES + " "
+    "WHERE mit.tag_id IN (SELECT id FROM page) GROUP BY mit.tag_id) "
+    "SELECT page.id,page.name,page.prompt_count,"
+    "COALESCE(images.n,0) AS image_count FROM page "
+    "LEFT JOIN images ON images.tag_id=page.id "
+    "ORDER BY page.prompt_count {direction},page.id"
+)
+_UNTAGGED_COUNT = (
+    "SELECT COUNT(*) FROM prompts p WHERE ("
+    + _ELIGIBLE
+    + ") AND "
+    + UNTAGGED
+    # Untagged prompts opens the shelf, which hides archived parents (#443).
+    + " AND "
+    + SHELVED
+)
 _TAG_ONE = _tag_rows("SELECT id,name FROM tags WHERE id=?")
 # Every owner kind a tag can sit on, for the delete confirmation.
 _TAG_IMPACT = (
@@ -97,16 +146,34 @@ def _validate_tag_id(tag_id: object) -> None:
         raise ValueError("Invalid tag ID")
 
 
-# The /library/options universe: nonblank values on eligible parents or on
-# active pairs, grouped exactly (whitespace and case variants stay distinct).
-_CATEGORY_GROUPS = (
-    "WITH owners(value,pid) AS ("
-    "SELECT p.category,p.id FROM prompts p WHERE p.category IS NOT NULL "
-    "AND TRIM(p.category)!='' AND " + _ELIGIBLE + " "
-    "UNION SELECT pp.category,pp.positive_prompt_id FROM prompt_pairs pp "
+# Category values in Tags & categories (#332 §7): the prompt-backed values
+# (nonblank values on eligible parents or on active pairs; ``prompt_count``
+# counts only parents on the shelf, so a value only archived parents (#443)
+# hold stays listed with 0, as a tag does; grouped exactly, so
+# whitespace and case variants stay distinct; /library/options lists the same
+# prompt-backed set with its own query) plus the values only image rows hold.
+# A prompt-backed value is ordered by its oldest owning positive; an image-only
+# value sorts after all of them (first_pid is SQLite's INTEGER maximum, above
+# any prompt id), then by value. ``image_count`` counts stored rows, including
+# files missing on disk. No FULL OUTER JOIN, so it runs on SQLite 3.31+.
+_IMAGE_ONLY_FIRST_PID = 2**63 - 1
+_CATEGORY_ROWS = (
+    "WITH owners(value,pid,shelved) AS ("
+    "SELECT p.category,p.id," + SHELVED + " FROM prompts p "
+    "WHERE p.category IS NOT NULL AND TRIM(p.category)!='' AND " + _ELIGIBLE + " "
+    "UNION SELECT pp.category,pp.positive_prompt_id," + SHELVED_PAIR + " "
+    "FROM prompt_pairs pp "
     "WHERE pp.archived=0 AND pp.category IS NOT NULL AND TRIM(pp.category)!=''),"
-    "grouped AS (SELECT value,MIN(pid) AS first_pid,"
-    "COUNT(DISTINCT pid) AS prompt_count FROM owners GROUP BY value) "
+    "image_values(value,image_count) AS ("
+    "SELECT category,COUNT(*) FROM image_categories GROUP BY category),"
+    "grouped(value,first_pid,prompt_count) AS ("
+    "SELECT value,MIN(pid),COUNT(DISTINCT CASE WHEN shelved THEN pid END) "
+    "FROM owners GROUP BY value "
+    "UNION ALL SELECT value," + str(_IMAGE_ONLY_FIRST_PID) + ",0 FROM image_values "
+    "WHERE value NOT IN (SELECT value FROM owners)),"
+    "counted AS (SELECT g.value,g.first_pid,g.prompt_count,"
+    "COALESCE(i.image_count,0) AS image_count FROM grouped g "
+    "LEFT JOIN image_values i ON i.value=g.value) "
 )
 
 
@@ -117,15 +184,23 @@ _CATEGORY_COMPONENTS = (
     "WHERE p.category=? ORDER BY p.id"
 )
 _CATEGORY_PAIRS = "SELECT id,archived FROM prompt_pairs WHERE category=? ORDER BY id"
+# Image rows holding one exact value (#332 DC7), an index search.
+_IMAGE_CATEGORY_ROWIDS = (
+    "SELECT rowid FROM image_categories WHERE category=? ORDER BY rowid"
+)
 _CATEGORY_EXISTS = (
     "SELECT EXISTS(SELECT 1 FROM prompts WHERE category=:v)"
     " OR EXISTS(SELECT 1 FROM prompt_pairs WHERE category=:v)"
+    " OR EXISTS(SELECT 1 FROM image_categories WHERE category=:v)"
 )
 
-# Pairs after components, so a failing pair update rolls both back.
+# Pairs after components and image rows last, so a failing update rolls every
+# earlier one back. An image row is keyed by its file, never by its value, so
+# merging into a value other files already hold cannot conflict.
 _CATEGORY_UPDATES = (
     "UPDATE prompts SET category=?,updated_at=? WHERE category=?",
     "UPDATE prompt_pairs SET category=?,updated_at=? WHERE category=?",
+    "UPDATE image_categories SET category=?,updated_at=? WHERE category=?",
 )
 
 
@@ -136,8 +211,15 @@ def category_impact_token(
     target_exists: bool,
     components: list,
     pairs: list,
+    *,
+    images: list = (),
 ) -> str:
-    """Hash a category projection like ``library_edit_token`` (no secret)."""
+    """Hash a category projection like ``library_edit_token`` (no secret).
+
+    ``images`` are the rowids of the image rows holding ``source`` (#332 DC7).
+    With none the preimage has no ``images`` key, so a prompt-only token keeps
+    the value it had before image categories existed.
+    """
     preimage = {
         "epoch": epoch,
         "source": source,
@@ -146,6 +228,8 @@ def category_impact_token(
         "components": sorted([int(i), bool(flag)] for i, flag in components),
         "pairs": sorted([int(i), bool(flag)] for i, flag in pairs),
     }
+    if images:
+        preimage["images"] = sorted(int(i) for i in images)
     return canonical_token(_CATEGORY_DOMAIN, preimage)
 
 
@@ -177,6 +261,28 @@ def _tag_after(after: object) -> int:
     return after
 
 
+def _tag_search(search: object) -> str:
+    if not isinstance(search, str) or len(search) > _MAX_SEARCH:
+        raise ValueError(f"Tag search must be text of at most {_MAX_SEARCH} characters")
+    return search.strip()
+
+
+def _sorted_tag_after(after: object, sort: str) -> tuple | None:
+    """The ``[key, id]`` cursor of a name or prompt-count sort."""
+    if after is None:
+        return None
+    key_type = str if sort.startswith("name") else int
+    if (
+        not isinstance(after, list)
+        or len(after) != 2
+        or type(after[0]) is not key_type
+        or type(after[1]) is not int
+        or after[1] <= 0
+    ):
+        raise ValueError("Invalid vocabulary cursor")
+    return after[0], after[1]
+
+
 def _category_after(after: object) -> tuple[int, str] | None:
     if after is None:
         return None
@@ -325,7 +431,8 @@ class VocabularyOperations:
         pairs = [
             [row[0], bool(row[1])] for row in conn.execute(_CATEGORY_PAIRS, (source,))
         ]
-        if not components and not pairs:
+        images = [row[0] for row in conn.execute(_IMAGE_CATEGORY_ROWIDS, (source,))]
+        if not components and not pairs and not images:
             raise LibraryCommandError("vocabulary-conflict", epoch)
         target_exists = bool(
             conn.execute(_CATEGORY_EXISTS, {"v": target}).fetchone()[0]
@@ -339,9 +446,10 @@ class VocabularyOperations:
                 "other_prompts": len(components) - eligible,
                 "negative_variants": len(pairs) - archived,
                 "archived_negative_variants": archived,
+                "images": len(images),
             },
             "impact_token": category_impact_token(
-                epoch, source, target, target_exists, components, pairs
+                epoch, source, target, target_exists, components, pairs, images=images
             ),
         }
 
@@ -365,7 +473,8 @@ class VocabularyOperations:
     def apply_vocabulary_category(
         self, *, expected_epoch: str, source: str, target: str, impact_token: str
     ) -> dict:
-        """Set the exact ``source`` to ``target`` on every component and pair.
+        """Set the exact ``source`` to ``target`` on every component, pair
+        and image row.
 
         The recomputed projection must still match the preview's token; a
         mismatch is ``vocabulary-conflict`` with nothing written.
@@ -395,22 +504,80 @@ class VocabularyOperations:
         epoch: str,
         *,
         limit: int = 100,
-        after: int | None = None,
+        after: object = None,
+        search: str = "",
+        sort: str = "created",
     ) -> dict:
+        """One page of tags matching ``search`` (a literal, case-insensitive
+        substring) in ``sort`` order, with the untagged parents' count."""
         _limit(limit)
-        start = _tag_after(after)
-        total = conn.execute("SELECT COUNT(*) FROM tags").fetchone()[0]
-        rows = [dict(row) for row in conn.execute(_TAG_PAGE, (start, limit + 1))]
+        search = _tag_search(search)
+        if not isinstance(sort, str) or sort not in TAG_SORTS:
+            raise ValueError("sort must be one of " + ", ".join(TAG_SORTS))
+        where, params = "1=1", []
+        if search:
+            where, params = _TAG_SEARCH, [f"%{_escape(search)}%"]
+        total, untagged = conn.execute(
+            f"SELECT (SELECT COUNT(*) FROM tags WHERE {where}),({_UNTAGGED_COUNT})",
+            params,
+        ).fetchone()
+        rows = [
+            dict(row) for row in self._tag_page(conn, where, params, sort, after, limit)
+        ]
         has_more = len(rows) > limit
         rows = rows[:limit]
+        key, _direction = TAG_SORTS[sort]
+        next_key = None
+        if has_more:
+            last = rows[-1]
+            next_key = (
+                last["id"]
+                if key is None
+                else [
+                    last["prompt_count" if key == "prompt_count" else "name"],
+                    last["id"],
+                ]
+            )
         return {
             "library_epoch": epoch,
             "tags": rows,
             "total": total,
+            "untagged_count": untagged,
+            # Echoed so a client can tell this read applied its search and sort.
+            "search": search,
+            "sort": sort,
             "has_more": has_more,
-            "next_key": rows[-1]["id"] if has_more else None,
+            "next_key": next_key,
         }
 
+    @staticmethod
+    def _tag_page(conn, where, params, sort, after, limit):
+        key, direction = TAG_SORTS[sort]
+        if key is None:
+            page = (
+                f"SELECT id,name FROM tags WHERE {where} AND id>? ORDER BY id LIMIT ?"
+            )
+            return conn.execute(
+                _tag_rows(page), [*params, _tag_after(after), limit + 1]
+            )
+        cursor = _sorted_tag_after(after, sort)
+        operator = ">" if direction == "ASC" else "<"
+        keyset, keys = "1=1", []
+        if cursor is not None:
+            keyset = f"({key}{operator}? OR ({key}=? AND id>?))"
+            keys = [cursor[0], cursor[0], cursor[1]]
+        if key == "prompt_count":
+            sql = _RANKED_TAG_PAGE.format(
+                search=where, after=keyset, direction=direction
+            )
+            return conn.execute(sql, [*params, *keys, limit + 1])
+        page = (
+            f"SELECT id,name FROM tags WHERE {where} AND {keyset} "
+            f"ORDER BY {key} {direction},id LIMIT ?"
+        )
+        order = f"page.{key} {direction},page.id"
+        return conn.execute(_tag_rows(page, order), [*params, *keys, limit + 1])
+
     def _vocabulary_categories(
         self,
         conn: sqlite3.Connection,
@@ -422,15 +589,15 @@ class VocabularyOperations:
         _limit(limit)
         key = _category_after(after)
         total = conn.execute(
-            _CATEGORY_GROUPS + "SELECT COUNT(*) FROM grouped"
+            _CATEGORY_ROWS + "SELECT COUNT(*) FROM counted"
         ).fetchone()[0]
         where, params = "", []
         if key is not None:
             where = " WHERE first_pid>? OR (first_pid=? AND value>?)"
             params = [key[0], key[0], key[1]]
         rows = conn.execute(
-            _CATEGORY_GROUPS
-            + "SELECT value,first_pid,prompt_count FROM grouped"
+            _CATEGORY_ROWS
+            + "SELECT value,first_pid,prompt_count,image_count FROM counted"
             + where
             + " ORDER BY first_pid,value LIMIT ?",
             [*params, limit + 1],
@@ -440,7 +607,11 @@ class VocabularyOperations:
         return {
             "library_epoch": epoch,
             "categories": [
-                {"value": row["value"], "prompt_count": row["prompt_count"]}
+                {
+                    "value": row["value"],
+                    "prompt_count": row["prompt_count"],
+                    "image_count": row["image_count"],
+                }
                 for row in rows
             ],
             "total": total,
@@ -450,9 +621,18 @@ class VocabularyOperations:
             ),
         }
 
-    def list_vocabulary_tags(self, *, limit: int = 100, after: int | None = None):
+    def list_vocabulary_tags(
+        self,
+        *,
+        limit: int = 100,
+        after: object = None,
+        search: str = "",
+        sort: str = "created",
+    ):
         with self.library_snapshot() as (conn, epoch, _ceiling):
-            return self._vocabulary_tags(conn, epoch, limit=limit, after=after)
+            return self._vocabulary_tags(
+                conn, epoch, limit=limit, after=after, search=search, sort=sort
+            )
 
     def list_vocabulary_categories(
         self, *, limit: int = 100, after: list | None = None

database/workflow_documents.py

@@ -0,0 +1,652 @@
+"""Each workflow and prompt graph stored once, compressed (#249 S6 carry-over).
+
+``generated_images.workflow_data`` (the UI workflow) and ``prompt_metadata``
+(the API prompt graph) used to hold the same large JSON text again for every
+image of a run. Now each distinct text is one ``workflow_documents`` row:
+
+- ``sha256``: the SHA-256 of the text's exact UTF-8 bytes (unique key);
+- ``byte_length``: the length of those bytes;
+- ``data``: ``zlib.compress`` of those bytes.
+
+An image row references its texts through ``workflow_document_id`` and
+``prompt_document_id``. Two forms coexist until the migration has rewritten
+every row, and both stay readable forever:
+
+- old form: the text inline in ``workflow_data`` / ``prompt_metadata``, no id;
+- new form: the inline column NULL and the document id set.
+
+The text is stored and hashed exactly as written, never parsed and
+re-serialized, because a JSON round trip can change bytes (key order,
+spacing, number spelling, duplicate keys, ComfyUI's ``NaN`` tokens).
+
+``read_stored_text`` is the one decoder. Inline text, when present, wins (a
+row an older app version rewrote inline stays right); otherwise the document
+is read by its rowid, decompressed within its recorded length and checked
+against its length and hash. A document that fails a check reads as
+unreadable (None, logged), never as other text. ``resolve_image_row`` puts
+the texts back into a ``generated_images`` row and drops the id keys, so
+every payload keeps its shape.
+
+No trigger: every restore validator refuses triggers in a restored file. A
+document is removed in code: ``release_documents`` in the same transaction
+that drops a row's last reference, and ``sweep_documents`` (at startup, after
+the migration and before Reclaim space) for anything a path missed.
+"""
+
+import hashlib
+import logging
+import re
+import sqlite3
+import zlib
+from collections import OrderedDict
+from typing import Any, Dict, Iterable, Optional, Set
+
+logger = logging.getLogger("prompt_manager.database.workflow_documents")
+
+TABLE = "workflow_documents"
+# (inline text column, document id column) on generated_images.
+TEXT_COLUMNS = (
+    ("workflow_data", "workflow_document_id"),
+    ("prompt_metadata", "prompt_document_id"),
+)
+DOCUMENT_ID_COLUMNS = tuple(reference for _column, reference in TEXT_COLUMNS)
+INDEXES = {
+    "workflow_document_id": "idx_image_workflow_document",
+    "prompt_document_id": "idx_image_prompt_document",
+}
+COMPRESSION_LEVEL = 6
+# A text larger than this stays inline: a document is decompressed in one
+# piece, and its recorded length bounds that (a restored file cannot make a
+# small row expand without limit).
+MAX_DOCUMENT_BYTES = 256 * 2**20
+ID_BATCH = 400  # ids bound per statement (under SQLite's old 999 limit)
+
+_DDL = (
+    f"CREATE TABLE {TABLE} (id INTEGER PRIMARY KEY, sha256 BLOB NOT NULL UNIQUE, "
+    "byte_length INTEGER NOT NULL, data BLOB NOT NULL)"
+)
+_AUTOINDEX = f"sqlite_autoindex_{TABLE}_1"
+_REFERENCE_TYPE = f"INTEGER REFERENCES {TABLE}(id)"
+
+_UNREFERENCED = (
+    "NOT EXISTS (SELECT 1 FROM generated_images g "
+    "WHERE g.workflow_document_id=workflow_documents.id) "
+    "AND NOT EXISTS (SELECT 1 FROM generated_images g "
+    "WHERE g.prompt_document_id=workflow_documents.id)"
+)
+LOOKUP_SQL = (
+    "SELECT id, sha256, byte_length, data FROM workflow_documents WHERE sha256=?"
+)
+REPAIR_SQL = "UPDATE workflow_documents SET byte_length=?, data=? WHERE id=?"
+ALL_DOCUMENTS_SQL = "SELECT sha256, byte_length, data, id FROM workflow_documents"
+READ_SQL = "SELECT sha256, byte_length, data FROM workflow_documents WHERE id=?"
+INSERT_SQL = (
+    "INSERT INTO workflow_documents (sha256, byte_length, data) VALUES (?, ?, ?)"
+)
+# Built from constants only; the ids are bound as "?" marks.
+RELEASE_SQL = "DELETE FROM workflow_documents WHERE id IN ({marks}) AND "
+RELEASE_SQL += _UNREFERENCED
+# The sweep walks the ids through the small unique index, not the blobs.
+SWEEP_SQL = (
+    "DELETE FROM workflow_documents WHERE id IN (SELECT id FROM workflow_documents "
+    "INDEXED BY sqlite_autoindex_workflow_documents_1 WHERE "
+)
+SWEEP_SQL += _UNREFERENCED + ")"
+IMAGE_DOCUMENTS_SQL = (
+    "SELECT workflow_document_id, prompt_document_id FROM generated_images WHERE "
+)
+
+# Statements whose query plans tests/test_workflow_documents.py records.
+QUERY_PLAN_STATEMENTS = {
+    "lookup by hash": (LOOKUP_SQL, (b"\0" * 32,)),
+    "read by id": (READ_SQL, (1,)),
+    "release": (RELEASE_SQL.format(marks="?,?"), (1, 2)),
+    "sweep": (SWEEP_SQL, ()),
+    "documents of images": (IMAGE_DOCUMENTS_SQL + "id IN (?,?)", (1, 2)),
+    "documents of prompts": (IMAGE_DOCUMENTS_SQL + "prompt_id IN (?,?)", (1, 2)),
+}
+
+
+class DocumentMismatch(Exception):
+    """A document would not give back exactly the text it was made from."""
+
+
+def _compact(sql: str) -> str:
+    return re.sub(r"\s+", "", sql or "").lower()
+
+
+# ── schema ────────────────────────────────────────────────────────────
+
+
+def _reference_columns(conn: sqlite3.Connection) -> Dict[str, tuple]:
+    return {
+        row[1]: tuple(row)
+        for row in conn.execute("PRAGMA table_info(generated_images)")
+        if row[1] in DOCUMENT_ID_COLUMNS
+    }
+
+
+def _reference_error(conn: sqlite3.Connection, columns: Dict[str, tuple]):
+    for name, info in columns.items():
+        # (cid, name, type, notnull, default, pk)
+        if (str(info[2]).upper(), info[3], info[4], info[5]) != ("INTEGER", 0, None, 0):
+            return "Workflow document references have an invalid column"
+    keys = {
+        (row[3], row[2], row[4])
+        for row in conn.execute("PRAGMA foreign_key_list(generated_images)")
+        if row[3] in DOCUMENT_ID_COLUMNS
+    }
+    if keys != {(name, TABLE, "id") for name in DOCUMENT_ID_COLUMNS}:
+        return "Workflow document references have invalid foreign keys"
+    for column, index in INDEXES.items():
+        info = conn.execute(f'PRAGMA index_info("{index}")').fetchall()
+        owner = conn.execute(
+            "SELECT tbl_name FROM sqlite_master WHERE type='index' AND name=?",
+            (index,),
+        ).fetchone()
+        if [row[2] for row in info] != [column] or not owner:
+            return "Workflow document references are missing their index"
+        if owner[0] != "generated_images":
+            return "Workflow document references are missing their index"
+    return None
+
+
+def _value_error(conn: sqlite3.Connection):
+    if conn.execute(
+        "SELECT 1 FROM workflow_documents WHERE typeof(sha256)!='blob' "
+        "OR length(sha256)!=32 OR typeof(byte_length)!='integer' OR byte_length<0 "
+        "OR byte_length>? OR typeof(data)!='blob' LIMIT 1",
+        (MAX_DOCUMENT_BYTES,),
+    ).fetchone():
+        return "Workflow documents contain invalid values"
+    for column in DOCUMENT_ID_COLUMNS:
+        # typeof() reads no overflow page; a dangling id is a rowid probe.
+        # ``column`` is one of DOCUMENT_ID_COLUMNS, never caller data.
+        sql = (
+            f"SELECT 1 FROM generated_images g WHERE typeof(g.{column}) "  # nosec B608
+            f"NOT IN ('null','integer') OR (g.{column} IS NOT NULL AND NOT EXISTS "
+            f"(SELECT 1 FROM workflow_documents d WHERE d.id=g.{column})) LIMIT 1"
+        )
+        if conn.execute(sql).fetchone():
+            return "Workflow document references point at missing documents"
+    return None
+
+
+def damaged_message(document_id: int) -> str:
+    """The verify reason for a damaged document: which one, and what to do."""
+    return (
+        f"Workflow document {document_id} is damaged: its text no longer matches "
+        "its checksum, and the images that use it show no workflow. To fix it, "
+        "restore a backup from before the damage, or store a new image with the "
+        "same workflow (that repairs it)."
+    )
+
+
+def _damaged_document(conn: sqlite3.Connection) -> Optional[int]:
+    """The first document failing its length or hash check, or None."""
+    for row in conn.execute(ALL_DOCUMENTS_SQL):
+        if _decode(row[:3], row[3], quiet=True) is None:
+            return row[3]
+    return None
+
+
+def workflow_documents_schema_error(
+    conn: sqlite3.Connection, *, allow_missing: bool = True, documents: bool = False
+) -> Optional[str]:
+    """Reject an unsupported document schema or invalid stored values (reads only).
+
+    A database from before the dedupe has neither the table nor the id
+    columns; that is acceptable unless ``allow_missing`` is false. Half of
+    the schema never is.
+    """
+    objects = conn.execute(
+        "SELECT type,name,sql FROM sqlite_master WHERE name=? "
+        "OR (type IN ('index','trigger') AND tbl_name=?)",
+        (TABLE, TABLE),
+    ).fetchall()
+    has_images = conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name='generated_images'"
+    ).fetchone()
+    columns = _reference_columns(conn) if has_images else {}
+    if not objects and not columns:
+        if allow_missing or not has_images:
+            return None
+        return "Missing workflow document schema"
+    if not objects or set(columns) != set(DOCUMENT_ID_COLUMNS):
+        return "Incomplete workflow document schema"
+    tables = [sql for kind, _name, sql in objects if kind == "table"]
+    others = {(kind, name, sql) for kind, name, sql in objects if kind != "table"}
+    if len(tables) != 1 or others != {("index", _AUTOINDEX, None)}:
+        return "Workflow documents name is reserved by an unsupported schema object"
+    # The exact definition fixes the columns, their types and constraints.
+    if _compact(tables[0]) != _compact(_DDL):
+        return "Workflow documents have an invalid table definition"
+    error = _reference_error(conn, columns) or _value_error(conn)
+    if error is None and documents:
+        # Restore and backup verification only: every document is read.
+        damaged = _damaged_document(conn)
+        if damaged is not None:
+            return damaged_message(damaged)
+    return error
+
+
+def migrate_workflow_documents(conn: sqlite3.Connection) -> None:
+    """Install the table, the id columns and their indexes atomically.
+
+    Runs after every older migration (the table rebuilds only run on
+    databases without ``file_path``, before this). Validates before and
+    after, then sweeps documents no row references.
+    """
+    conn.execute("SAVEPOINT workflow_documents_schema")
+    try:
+        error = workflow_documents_schema_error(conn)
+        if error:
+            raise sqlite3.DatabaseError(error)
+        conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
+        present = _reference_columns(conn)
+        for column in DOCUMENT_ID_COLUMNS:
+            if column not in present:
+                conn.execute(
+                    f"ALTER TABLE generated_images ADD COLUMN {column} "
+                    f"{_REFERENCE_TYPE}"
+                )
+        for column, index in INDEXES.items():
+            conn.execute(
+                f"CREATE INDEX IF NOT EXISTS {index} ON generated_images({column})"
+            )
+        error = workflow_documents_schema_error(conn, allow_missing=False)
+        if error:
+            raise sqlite3.DatabaseError(error)
+        sweep_documents(conn)
+        conn.execute("RELEASE workflow_documents_schema")
+    except BaseException:
+        conn.execute("ROLLBACK TO workflow_documents_schema")
+        conn.execute("RELEASE workflow_documents_schema")
+        raise
+
+
+# ── write ─────────────────────────────────────────────────────────────
+
+
+def storable(text: Any) -> bool:
+    """Whether ``text`` can be a document (text within MAX_DOCUMENT_BYTES)."""
+    if not isinstance(text, str):
+        return False
+    # UTF-8 takes at most 4 bytes a character: most texts need no encoding.
+    if len(text) * 4 <= MAX_DOCUMENT_BYTES:
+        return True
+    return len(text.encode("utf-8")) <= MAX_DOCUMENT_BYTES
+
+
+def store_text(
+    conn: sqlite3.Connection, text: Optional[str], verified: Optional[set] = None
+) -> Optional[int]:
+    """The id of the document holding exactly ``text``; None for None.
+
+    Reuses the document with the same hash, or inserts a new one whose
+    compressed bytes were checked to decompress to ``text``'s bytes. A reused
+    document is decompressed and checked first; a damaged one (a restored
+    file's corruption under a valid hash) is repaired from ``text``, whose
+    hash is its key, so every row that references it reads right again.
+    ``verified``, when given, holds the ids already checked (a migration
+    batch), which are then not read again. Opens a write transaction first
+    when none is open, so a concurrent ``release_documents`` cannot remove a
+    reused document before the caller references it. Raises TypeError for a
+    non-text value and ValueError for a text over MAX_DOCUMENT_BYTES.
+    """
+    if text is None:
+        return None
+    if not isinstance(text, str):
+        raise TypeError("Only text is stored as a workflow document")
+    raw = text.encode("utf-8")
+    if len(raw) > MAX_DOCUMENT_BYTES:
+        raise ValueError("Text is too large for a workflow document")
+    digest = hashlib.sha256(raw).digest()
+    if not conn.in_transaction:
+        conn.execute("BEGIN IMMEDIATE")
+    row = conn.execute(LOOKUP_SQL, (digest,)).fetchone()
+    if row is not None and verified is not None and row[0] in verified:
+        return row[0]
+    data = zlib.compress(raw, COMPRESSION_LEVEL)
+    if zlib.decompress(data) != raw:  # pragma: no cover - zlib is exact
+        raise DocumentMismatch("Compression did not round-trip")
+    if row is None:
+        document_id = conn.execute(INSERT_SQL, (digest, len(raw), data)).lastrowid
+    else:
+        document_id = row[0]
+        if _decode(row[1:], document_id, quiet=True) != text:
+            logger.warning(f"Workflow document {document_id} was damaged; repaired")
+            conn.execute(REPAIR_SQL, (len(raw), data, document_id))
+    if verified is not None:
+        verified.add(document_id)
+    return document_id
+
+
+def stored_values(
+    conn: sqlite3.Connection, texts: Dict[str, Optional[str]]
+) -> Dict[str, Any]:
+    """New-form column values for ``{text column: text}`` (writers).
+
+    Each text becomes ``{column: None, id column: document id}``; None
+    stays ``{column: None, id column: None}``; a value that cannot be a
+    document (not text, or too large) stays inline with no id.
+    """
+    references = dict(TEXT_COLUMNS)
+    values: Dict[str, Any] = {}
+    for column, text in texts.items():
+        reference = references[column]
+        if text is None or not storable(text):
+            values[column], values[reference] = text, None
+        else:
+            values[column], values[reference] = None, store_text(conn, text)
+    return values
+
+
+# ── read ──────────────────────────────────────────────────────────────
+
+
+def _decode(row, document_id, quiet=False) -> Optional[str]:
+    """``(sha256, byte_length, data)`` as checked text, or None (logged)."""
+    if row is None:
+        logger.warning(f"Workflow document {document_id} is missing")
+        return None
+    digest, length, data = bytes(row[0]), row[1], row[2]
+    try:
+        if not isinstance(length, int) or not 0 <= length <= MAX_DOCUMENT_BYTES:
+            raise ValueError("invalid length")
+        inflater = zlib.decompressobj()
+        raw = inflater.decompress(bytes(data), length + 1)
+        if len(raw) != length or inflater.unconsumed_tail or not inflater.eof:
+            raise ValueError("length differs")
+        if hashlib.sha256(raw).digest() != digest:
+            raise ValueError("hash differs")
+        return raw.decode("utf-8")
+    except (zlib.error, ValueError, TypeError) as error:
+        if not quiet:
+            logger.warning(f"Workflow document {document_id} is unreadable: {error}")
+        return None
+
+
+def read_stored_text(
+    conn: sqlite3.Connection, inline: Any, document_id: Optional[int]
+) -> Any:
+    """The stored text of one column: inline when present, else the document.
+
+    Returns None when there is neither, or when the document is missing or
+    fails its checks (logged). An inline value is returned unchanged.
+    """
+    if inline is not None:
+        return inline
+    if document_id is None:
+        return None
+    return _decode(conn.execute(READ_SQL, (document_id,)).fetchone(), document_id)
+
+
+class DocumentReader:
+    """``read_stored_text`` with a small memo for list reads on one connection."""
+
+    MEMO = 32
+
+    def __init__(self, conn: sqlite3.Connection):
+        self.conn = conn
+        self._memo: "OrderedDict[int, Optional[str]]" = OrderedDict()
+
+    def text(self, inline: Any, document_id: Optional[int]) -> Any:
+        if inline is not None or document_id is None:
+            return read_stored_text(self.conn, inline, document_id)
+        if document_id in self._memo:
+            self._memo.move_to_end(document_id)
+            return self._memo[document_id]
+        text = read_stored_text(self.conn, None, document_id)
+        self._memo[document_id] = text
+        if len(self._memo) > self.MEMO:
+            self._memo.popitem(last=False)
+        return text
+
+
+def resolve_image_row(reader: DocumentReader, row) -> Dict[str, Any]:
+    """A ``generated_images`` row as a dict with its texts read back.
+
+    The id columns are dropped, so a payload built from it has the shape it
+    had before the dedupe. Columns the row did not select stay absent.
+    """
+    data = dict(row)
+    for column, reference in TEXT_COLUMNS:
+        has_reference = reference in data
+        document_id = data.pop(reference, None)
+        if column in data or has_reference:
+            data[column] = reader.text(data.get(column), document_id)
+    return data
+
+
+# ── garbage ───────────────────────────────────────────────────────────
+
+
+def _chunks(values):
+    values = list(values)
+    for start in range(0, len(values), ID_BATCH):
+        yield values[start : start + ID_BATCH]
+
+
+def image_documents(conn: sqlite3.Connection, where: str, params) -> Set[int]:
+    """The document ids the image rows matching ``where`` reference.
+
+    ``where`` is a fixed condition written in this package (``id IN (...)``,
+    ``prompt_id IN (...)``), never caller data; values go in ``params``.
+    """
+    ids: Set[int] = set()
+    sql = IMAGE_DOCUMENTS_SQL + where
+    for row in conn.execute(sql, list(params)):  # nosec B608
+        ids.update(value for value in row if value is not None)
+    return ids
+
+
+def documents_of_rows(
+    conn: sqlite3.Connection, column: str, values: Iterable[Any]
+) -> Set[int]:
+    """Document ids of the image rows whose ``column`` is in ``values``.
+
+    ``column`` is ``id`` or ``prompt_id`` (the rows a prompt delete cascades to).
+    """
+    if column not in ("id", "prompt_id"):
+        raise ValueError(f"Unsupported column {column!r}")
+    ids: Set[int] = set()
+    for chunk in _chunks(values):
+        marks = ",".join("?" for _ in chunk)
+        ids |= image_documents(conn, f"{column} IN ({marks})", chunk)
+    return ids
+
+
+def release_documents(conn: sqlite3.Connection, ids: Iterable[Optional[int]]) -> int:
+    """Delete the documents in ``ids`` that no image row references now.
+
+    Runs in the caller's transaction, right after the delete or update that
+    dropped the references. Returns how many documents were removed.
+    """
+    removed = 0
+    for chunk in _chunks(sorted({i for i in ids if i is not None})):
+        marks = ",".join("?" for _ in chunk)
+        # Only "?" marks are interpolated.
+        sql = RELEASE_SQL.format(marks=marks)
+        removed += conn.execute(sql, chunk).rowcount  # nosec B608
+    return removed
+
+
+def sweep_documents(conn: sqlite3.Connection) -> int:
+    """Delete every document no image row references; returns the count."""
+    return conn.execute(SWEEP_SQL).rowcount
+
+
+# ── repair (a restore's stage) ────────────────────────────────────────
+
+INLINE_TEXTS_SQL = (
+    "SELECT workflow_data, prompt_metadata FROM generated_images "
+    "WHERE workflow_data IS NOT NULL OR prompt_metadata IS NOT NULL"
+)
+REKEY_SQL = "UPDATE workflow_documents SET sha256=?, byte_length=? WHERE id=?"
+
+
+def _rows(conn: sqlite3.Connection, sql: str, params=()):
+    """Each row of ``sql``, read one at a time.
+
+    Iterating a managed connection's cursor prefetches ``ITER_CHUNK`` (128)
+    rows, and a document or inline text can be megabytes.
+    """
+    cursor = conn.execute(sql, params)
+    while (row := cursor.fetchone()) is not None:
+        yield row
+
+
+def _inflated(data) -> Optional[bytes]:
+    """The complete text bytes ``data`` inflates to, or None.
+
+    Complete means the zlib stream ends (its Adler-32 checked) within
+    MAX_DOCUMENT_BYTES with nothing after it: the body is intact even when
+    the row's recorded hash or length is not.
+    """
+    try:
+        inflater = zlib.decompressobj()
+        raw = inflater.decompress(bytes(data), MAX_DOCUMENT_BYTES + 1)
+        if not inflater.eof or inflater.unconsumed_tail or inflater.unused_data:
+            return None
+        if len(raw) > MAX_DOCUMENT_BYTES:
+            return None
+        raw.decode("utf-8")
+        return raw
+    except (zlib.error, TypeError, UnicodeDecodeError):
+        return None
+
+
+def _one_column_off(row, raw: bytes) -> bool:
+    """Whether only the recorded hash or only the recorded length is wrong.
+
+    The two columns check the body independently. When the body inflates
+    completely and one of them still agrees with it, that column vouches for
+    the body and the other one is the damaged value. When both disagree, the
+    body itself is the suspect (another valid stream), so it is not trusted.
+    """
+    hash_ok = hashlib.sha256(raw).digest() == bytes(row[0])
+    length_ok = row[1] == len(raw)
+    return hash_ok != length_ok
+
+
+def _inline_copies(conn: sqlite3.Connection, wanted: Dict[bytes, int]):
+    """``{document id: text}`` for each wanted hash an inline row still holds."""
+    found: Dict[int, str] = {}
+    for row in _rows(conn, INLINE_TEXTS_SQL):
+        for text in row:
+            if not isinstance(text, str) or not storable(text):
+                continue
+            document_id = wanted.get(hashlib.sha256(text.encode("utf-8")).digest())
+            if document_id is not None and document_id not in found:
+                found[document_id] = text
+        if len(found) == len(wanted):
+            break
+    return found
+
+
+def _rekey(conn: sqlite3.Connection, document_id: int, raw: bytes) -> Optional[int]:
+    """Give an intact body its true key, or merge it into the document that has it.
+
+    Returns the holder's id when its own body was damaged and has just been
+    rewritten from ``raw`` (it is repaired, never to be cleared), else None.
+    """
+    digest = hashlib.sha256(raw).digest()
+    holder = conn.execute(LOOKUP_SQL, (digest,)).fetchone()
+    if holder is None or holder[0] == document_id:
+        conn.execute(REKEY_SQL, (digest, len(raw), document_id))
+        return None
+    for column in DOCUMENT_ID_COLUMNS:
+        # ``column`` is one of DOCUMENT_ID_COLUMNS, never caller data.
+        conn.execute(
+            f"UPDATE generated_images SET {column}=? WHERE {column}=?",  # nosec B608
+            (holder[0], document_id),
+        )
+    conn.execute("DELETE FROM workflow_documents WHERE id=?", (document_id,))
+    if _decode(holder[1:], holder[0], quiet=True) is None:
+        conn.execute(
+            REPAIR_SQL, (len(raw), zlib.compress(raw, COMPRESSION_LEVEL), holder[0])
+        )
+        return holder[0]
+    return None
+
+
+def repair_damaged_documents(conn: sqlite3.Connection) -> Dict[str, int]:
+    """Repair or clear every document failing its checks; commit (writes).
+
+    For a restore's stage only, never the live library. In order:
+
+    1. A document whose body inflates completely and whose recorded hash or
+       length (not both) disagrees is re-keyed from its text, or merged
+       into the document that already holds that text (``_one_column_off``).
+    2. A damaged body whose exact text an inline row still holds (the same
+       SHA-256, its key) is stored again from that text.
+    3. Any other damaged document is unreadable for good: its images'
+       references are cleared (they already showed no workflow) and the
+       document is removed, so the stage verifies.
+
+    Returns ``{repaired, cleared, images}``: documents repaired (1 and 2),
+    documents cleared and the distinct image rows whose reference was
+    cleared. Rows are read one at a time.
+    """
+    damaged: Dict[bytes, int] = {}
+    intact: Dict[int, bytes] = {}
+    for row in _rows(conn, ALL_DOCUMENTS_SQL):
+        if _decode(row[:3], row[3], quiet=True) is None:
+            raw = _inflated(row[2])
+            if raw is not None and _one_column_off(row, raw):
+                intact[row[3]] = raw
+            else:
+                damaged[bytes(row[0])] = row[3]
+    result = {"repaired": 0, "cleared": 0, "images": 0}
+    if not damaged and not intact:
+        return result
+    with conn:
+        healed = []
+        for document_id, raw in intact.items():
+            holder = _rekey(conn, document_id, raw)
+            if holder is not None and holder in damaged.values():
+                # Rewritten from the merged body: repaired, not to be cleared
+                # (it now also holds the merged images' references).
+                damaged = {k: v for k, v in damaged.items() if v != holder}
+                healed.append(holder)
+        copies = _inline_copies(conn, damaged) if damaged else {}
+        for document_id, text in copies.items():
+            raw = text.encode("utf-8")
+            data = zlib.compress(raw, COMPRESSION_LEVEL)
+            conn.execute(REPAIR_SQL, (len(raw), data, document_id))
+        lost = sorted(set(damaged.values()) - set(copies))
+        images: Set[int] = set()
+        for chunk in _chunks(lost):
+            marks = ",".join("?" * len(chunk))
+            # ``marks`` is only "?" placeholders; the ids are bound.
+            for (image_id,) in _rows(
+                conn,
+                "SELECT id FROM generated_images WHERE "  # nosec B608
+                f"workflow_document_id IN ({marks}) OR prompt_document_id IN ({marks})",
+                chunk + chunk,
+            ):
+                images.add(image_id)
+            for column in DOCUMENT_ID_COLUMNS:
+                conn.execute(
+                    f"UPDATE generated_images SET {column}=NULL "  # nosec B608
+                    f"WHERE {column} IN ({marks})",
+                    chunk,
+                )
+            conn.execute(
+                f"DELETE FROM workflow_documents WHERE id IN ({marks})",  # nosec B608
+                chunk,
+            )
+    for document_id in list(intact) + healed + list(copies):
+        logger.warning(f"Workflow document {document_id} was damaged; repaired")
+    if lost:
+        logger.warning(
+            f"{len(lost)} damaged workflow document(s) could not be repaired; "
+            f"{len(images)} image(s) show no workflow"
+        )
+    result.update(
+        repaired=len(intact) + len(healed) + len(copies),
+        cleared=len(lost),
+        images=len(images),
+    )
+    return result

database/workflow_storage.py

@@ -0,0 +1,455 @@
+"""The workflow storage migration and Reclaim space (#249 S6 carry-over).
+
+``migrate_workflow_storage`` rewrites old-form image rows (workflow and
+prompt graph text inline) to the new form (a ``workflow_documents`` id,
+``database/workflow_documents.py``) in batches:
+
+- each batch is one ``BEGIN IMMEDIATE`` transaction over at most
+  ``BATCH_ROWS`` rows or about ``BATCH_BYTES`` of text, found by a rowid
+  range search (``NEXT_IDS_SQL``) and read one row at a time, so memory is
+  bounded by one row plus the batch's distinct documents;
+- each text is stored (or its document reused) and read back through the
+  one accessor before its row changes; a text that would not come back
+  identical fails the batch (``DocumentMismatch``) and rolls it back whole;
+- a value that is not text (or is over ``MAX_DOCUMENT_BYTES``) stays inline
+  and is counted as protected;
+- each row runs in its own savepoint: one that cannot be rewritten (text
+  SQLite cannot decode, a text that would not come back) stays inline and
+  is listed, and the rest of the batch goes on;
+- a batch ends after ``BATCH_SECONDS``, and the write lock is then left
+  free for ``BATCH_PAUSE``, so other writers are never starved.
+
+A crash or a stop between batches leaves committed batches in the new form
+and the rest in the old form; both read the same, and a rerun continues
+from the first old-form row. Only database rows change: no image or video
+file is opened.
+
+``reclaim_space`` is the VACUUM afterwards: it sweeps unreferenced
+documents, rebuilds the file and checkpoints the WAL with TRUNCATE, so the
+freed pages go back to the disk.
+"""
+
+import logging
+import os
+import sqlite3
+import time
+from contextlib import closing
+
+from .workflow_documents import (
+    TEXT_COLUMNS,
+    DocumentMismatch,
+    read_stored_text,
+    release_documents,
+    storable,
+    store_text,
+    sweep_documents,
+)
+
+OPERATION = "workflow_storage"
+# The first apply's backup: ``<db>.pre-workflow-storage_<stamp>``, outside the
+# pruned ``<db>.backup_*`` pool. Only one is ever made; later applies take a
+# pooled safety backup.
+PRE_MIGRATION_LABEL = "pre-workflow-storage"
+BATCH_ROWS = 200
+BATCH_BYTES = 16 * 2**20  # characters of text rewritten per transaction
+# A batch holds the write lock: it ends after this long, and the loop then
+# leaves the lock free for BATCH_PAUSE, longer than SQLite's longest busy
+# poll (100 ms), so the image monitor and the run counting get in between.
+BATCH_SECONDS = 0.5
+BATCH_PAUSE = 0.15
+
+
+def _row_error(error) -> bool:
+    """Whether ``error`` is one row's own cause, which keeps only that row inline.
+
+    Text SQLite cannot decode for Python, and a text that would not come
+    back identical. Anything else (a full disk, an I/O error, a malformed
+    database, a bug) fails the whole batch. The decode error is matched by
+    its message: ``sqlite_errorcode`` needs Python 3.11.
+    """
+    if isinstance(error, (DocumentMismatch, UnicodeError)):
+        return True
+    return isinstance(error, sqlite3.OperationalError) and str(error).startswith(
+        "Could not decode"
+    )
+
+
+logger = logging.getLogger("prompt_manager.database.workflow_storage")
+_clock = time.monotonic
+
+# typeof() reads a column's type from the record header: no overflow page.
+NEXT_IDS_SQL = (
+    "SELECT id FROM generated_images WHERE id>? AND "
+    "(typeof(workflow_data)!='null' OR typeof(prompt_metadata)!='null') "
+    "ORDER BY id LIMIT ?"
+)
+ROW_SQL = (
+    "SELECT workflow_data, prompt_metadata, workflow_document_id, "
+    "prompt_document_id FROM generated_images WHERE id=?"
+)
+UPDATE_SQL = (
+    "UPDATE generated_images SET {assignments} WHERE id=?"  # columns are constants
+)
+# One pass over the record headers: rows with inline text, and inline
+# values that are not text (they stay inline).
+INLINE_COUNTS_SQL = (
+    "SELECT "
+    "SUM(typeof(workflow_data)='text' OR typeof(prompt_metadata)='text'), "
+    "SUM(typeof(workflow_data) NOT IN ('null','text')) "
+    "+ SUM(typeof(prompt_metadata) NOT IN ('null','text')) "
+    "FROM generated_images"
+)
+# octet_length() (SQLite 3.43+) reads no overflow page either.
+INLINE_BYTES_SQL = (
+    "SELECT SUM(octet_length(workflow_data)), SUM(octet_length(prompt_metadata)) "
+    "FROM generated_images"
+)
+
+QUERY_PLAN_STATEMENTS = {
+    "next old-form rows": (NEXT_IDS_SQL, (0, BATCH_ROWS)),
+    "one row": (ROW_SQL, (1,)),
+}
+
+
+def _rewrite_row(conn, image_id, row, verified, checked):
+    """Store the row's inline texts as documents; returns (changed, protected, size).
+
+    ``verified`` maps document ids already read back in this batch to their
+    text, and ``checked`` holds the reused ids ``store_text`` already
+    checked, so a document shared by many rows is decompressed once.
+    """
+    changes, replaced = {}, []
+    protected = size = 0
+    for index, (column, reference) in enumerate(TEXT_COLUMNS):
+        value, current = row[index], row[index + len(TEXT_COLUMNS)]
+        if value is None:
+            continue
+        if not storable(value):
+            protected += 1
+            continue
+        document_id = store_text(conn, value, checked)
+        if document_id not in verified:
+            verified[document_id] = read_stored_text(conn, None, document_id)
+        if verified[document_id] != value:
+            raise DocumentMismatch(f"Image {image_id}: {column} would change")
+        changes[column], changes[reference] = None, document_id
+        if current is not None and current != document_id:
+            replaced.append(current)
+        size += len(value)
+    if changes:
+        assignments = ",".join(f"{key}=?" for key in changes)
+        conn.execute(
+            UPDATE_SQL.format(assignments=assignments),
+            [*changes.values(), image_id],
+        )
+    release_documents(conn, replaced)
+    return bool(changes), protected, size
+
+
+def migrate_batch(
+    conn,
+    after_id,
+    max_rows=BATCH_ROWS,
+    max_bytes=BATCH_BYTES,
+    max_seconds=BATCH_SECONDS,
+):
+    """Rewrite the next batch after ``after_id`` in one transaction.
+
+    The batch ends after ``max_rows`` rows, about ``max_bytes`` of text or
+    ``max_seconds``, whichever comes first. Each row runs in its own
+    savepoint: a row that cannot be rewritten (``_row_error``) is rolled
+    back alone, stays inline and is listed in ``errors``; anything else rolls
+    the whole batch back and propagates. Returns ``{last_id, rows,
+    protected, errors, done}``: the last row looked at, rows rewritten,
+    inline values kept (not text), the ids kept by an error and whether
+    there was nothing left to look at.
+    """
+    if conn.in_transaction:
+        raise RuntimeError("The migration needs its own transaction")
+    started = _clock()
+    conn.execute("BEGIN IMMEDIATE")
+    try:
+        ids = [row[0] for row in conn.execute(NEXT_IDS_SQL, (after_id, max_rows))]
+        result = {
+            "last_id": after_id,
+            "rows": 0,
+            "protected": 0,
+            "errors": [],
+            "done": not ids,
+        }
+        verified, checked, size = {}, set(), 0
+        for image_id in ids:
+            conn.execute("SAVEPOINT workflow_row")
+            try:
+                row = conn.execute(ROW_SQL, (image_id,)).fetchone()
+                changed, protected, row_size = _rewrite_row(
+                    conn, image_id, row, verified, checked
+                )
+            except Exception as error:
+                if not _row_error(error):
+                    raise
+                conn.execute("ROLLBACK TO workflow_row")
+                conn.execute("RELEASE workflow_row")
+                verified.clear()  # the row's documents were rolled back too
+                checked.clear()
+                logger.warning(f"Image record {image_id} kept inline: {error}")
+                result["errors"].append(image_id)
+                changed, protected, row_size = False, 0, 0
+            else:
+                conn.execute("RELEASE workflow_row")
+            result["rows"] += int(changed)
+            result["protected"] += protected
+            result["last_id"] = image_id
+            size += row_size
+            if size >= max_bytes or _clock() - started >= max_seconds:
+                break
+        conn.commit()
+        return result
+    except BaseException:
+        conn.rollback()
+        raise
+
+
+def pre_migration_backups(db_path):
+    """The kept pre-migration backups beside ``db_path``, earliest first."""
+    db_path = os.path.abspath(db_path)
+    prefix = os.path.basename(db_path) + "." + PRE_MIGRATION_LABEL + "_"
+    folder = os.path.dirname(db_path)
+    try:
+        names = os.listdir(folder)
+    except OSError:
+        return []
+    return sorted(
+        name
+        for name in names
+        if name.startswith(prefix) and os.path.isfile(os.path.join(folder, name))
+    )
+
+
+def _file_bytes(db_path):
+    return sum(
+        os.path.getsize(path)
+        for path in (db_path, db_path + "-wal")
+        if os.path.exists(path)
+    )
+
+
+class WorkflowStorageOperations:
+    """PromptDatabase mixin: the workflow storage migration and Reclaim space."""
+
+    def migrate_workflow_storage(
+        self,
+        *,
+        on_batch=None,
+        should_stop=None,
+        max_rows=BATCH_ROWS,
+        max_bytes=BATCH_BYTES,
+    ) -> dict:
+        """Rewrite every old-form row, batch by batch (blocking).
+
+        ``on_batch(progress)`` runs after each committed batch with the
+        running totals ``{rows, protected, batches, last_id}``;
+        ``should_stop()`` is asked before each batch. Each committed batch is
+        followed by ``BATCH_PAUSE`` with the write lock free. Returns
+        ``{rows, protected, batches, stopped, errors, documents_removed}``
+        (``errors``: the ids of rows kept inline by an error). Unreferenced
+        documents are swept once every row was looked at.
+        """
+        totals = {"rows": 0, "protected": 0, "batches": 0, "last_id": 0}
+        errors = []
+        stopped = False
+        while True:
+            if should_stop is not None and should_stop():
+                stopped = True
+                break
+            batch = self.migrate_workflow_batch(totals["last_id"], max_rows, max_bytes)
+            if batch["done"]:
+                break
+            time.sleep(BATCH_PAUSE)  # leave the write lock to other writers
+            errors += batch["errors"]
+            totals["rows"] += batch["rows"]
+            totals["protected"] += batch["protected"]
+            totals["batches"] += 1
+            totals["last_id"] = batch["last_id"]
+            if on_batch is not None:
+                on_batch(dict(totals))
+        removed = 0 if stopped else self.sweep_workflow_documents()
+        return {
+            "rows": totals["rows"],
+            "protected": totals["protected"],
+            "batches": totals["batches"],
+            "stopped": stopped,
+            "errors": errors,
+            "documents_removed": removed,
+        }
+
+    def migrate_workflow_batch(
+        self, after_id, max_rows=BATCH_ROWS, max_bytes=BATCH_BYTES
+    ) -> dict:
+        """One batch on the calling thread's connection (``migrate_batch``)."""
+        return migrate_batch(self.model.get_connection(), after_id, max_rows, max_bytes)
+
+    def sweep_workflow_documents(self) -> int:
+        """Remove every document no image row references (one transaction)."""
+        conn = self.model.get_connection()
+        with conn:
+            return sweep_documents(conn)
+
+    def workflow_storage_status(self) -> dict:
+        """``workflow_storage_counts`` on one read-only snapshot (blocking)."""
+        from .maintenance import _read_snapshot
+
+        return _read_snapshot(self.model.db_path, self.workflow_storage_counts)
+
+    def workflow_storage_counts(self, conn) -> dict:
+        """``{rows, protected, documents, inline_bytes}`` on ``conn`` (reads only).
+
+        ``inline_bytes`` is None where SQLite has no ``octet_length``.
+        """
+        rows, protected = conn.execute(INLINE_COUNTS_SQL).fetchone()
+        documents = conn.execute("SELECT COUNT(*) FROM workflow_documents").fetchone()
+        inline_bytes = None
+        if sqlite3.sqlite_version_info >= (3, 43, 0):
+            row = conn.execute(INLINE_BYTES_SQL).fetchone()
+            inline_bytes = sum(value or 0 for value in row)
+        return {
+            "rows": rows or 0,
+            "protected": protected or 0,
+            "documents": documents[0],
+            "inline_bytes": inline_bytes,
+        }
+
+    def reclaim_space(self) -> dict:
+        """Sweep, VACUUM and truncate the WAL (blocking); file bytes before/after.
+
+        The caller holds the registry's ``exclusive`` slot. Raises whatever
+        stopped VACUUM; the database is then unchanged.
+        """
+        db_path = self.model.db_path
+        removed = self.sweep_workflow_documents()
+        before = _file_bytes(db_path)
+        self.model.vacuum_database()
+        with closing(sqlite3.connect(db_path)) as checkpoint:
+            busy = checkpoint.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()[0]
+        return {
+            "before_bytes": before,
+            "after_bytes": _file_bytes(db_path),
+            "documents_removed": removed,
+            "checkpoint_busy": bool(busy),
+        }
+
+    # ── preview ───────────────────────────────────────────────────────
+
+    def _plan_workflow_storage(self):
+        """The reviewed plan of the migration (one read-only snapshot)."""
+        from .maintenance import (
+            MaintenanceDraft,
+            _Fingerprint,
+            _megabytes,
+            _plural,
+            _read_snapshot,
+        )
+
+        def read(conn):
+            sizes = [
+                conn.execute(f"PRAGMA {name}").fetchone()[0]
+                for name in ("page_size", "page_count", "freelist_count")
+            ]
+            return sizes, self.workflow_storage_counts(conn)
+
+        (page_size, page_count, free_pages), counts = _read_snapshot(
+            self.model.db_path, read
+        )
+        size = page_size * page_count
+        details = [
+            {
+                "label": "Database file",
+                "detail": f"{_megabytes(size)}, of which "
+                f"{_megabytes(free_pages * page_size)} is free pages",
+            },
+            {
+                "label": "Image records",
+                "detail": f"{_plural(counts['rows'], 'record')} keep their workflow "
+                "or prompt graph inline; each distinct text is stored once, "
+                "compressed",
+            },
+        ]
+        if not counts["rows"]:
+            details[1]["detail"] = NOTHING_TO_DO
+        if counts["inline_bytes"] is not None:
+            details.append(
+                {
+                    "label": "Inline text",
+                    "detail": f"{_megabytes(counts['inline_bytes'])} now",
+                }
+            )
+        if counts["documents"]:
+            details.append(
+                {
+                    "label": "Already stored once",
+                    "detail": _plural(counts["documents"], "workflow document"),
+                }
+            )
+        name = os.path.basename(self.model.db_path)
+        kept = pre_migration_backups(self.model.db_path)
+        backup = (
+            POOLED_BACKUP_NOTE.format(
+                name=f"{name}.backup_<date and time>", kept=kept[0]
+            )
+            if kept
+            else BACKUP_NOTE.format(
+                name=f"{name}.{PRE_MIGRATION_LABEL}_<date and time>"
+            )
+        )
+        details += [
+            {"label": "Backup", "detail": backup},
+            {"label": "Reclaim space", "detail": RECLAIM_NOTE},
+            {"label": "Older versions", "detail": OLDER_VERSIONS_NOTE},
+        ]
+        protected = counts["protected"]
+        return MaintenanceDraft(
+            operation=OPERATION,
+            fingerprint=_Fingerprint(OPERATION).hexdigest(),
+            targets=[],
+            scope={
+                "count": counts["rows"],
+                "label": (
+                    _plural(counts["rows"], "image record") + " to rewrite"
+                    if counts["rows"]
+                    else "Nothing to rewrite"
+                ),
+            },
+            protected={
+                "count": protected,
+                "reasons": (
+                    [f"{_plural(protected, 'stored value')} not text; kept as is"]
+                    if protected
+                    else []
+                ),
+            },
+            media={"deletes_files": False, "label": MEDIA_NOTE},
+            details=details,
+        )
+
+
+MEDIA_NOTE = "No image or video file is opened or changed; only the database."
+RECLAIM_NOTE = (
+    "When every record is rewritten, Reclaim space runs alone to give the freed "
+    "space back to the disk (VACUUM). The database is locked while it runs."
+)
+BACKUP_NOTE = (
+    "Always taken first, whatever the safety backup setting: {name} beside "
+    "the database, verified, and kept until you remove it (it is never pruned)."
+)
+POOLED_BACKUP_NOTE = (
+    "A safety backup is taken first, whatever the safety backup setting: {name} "
+    "among the pooled backups; the pre-migration backup {kept} is already kept."
+)
+NOTHING_TO_DO = "Every image record already stores its workflow once."
+OLDER_VERSIONS_NOTE = (
+    "This version stores the workflows of images it records in the new form, "
+    "whether or not this job runs, and this job rewrites the rest. An older "
+    "Prompt Manager version shows those images without their workflow until you "
+    "update again. The way back to an older version is the earliest "
+    "pre-workflow-storage backup."
+)

docs/application-redesign.md

@@ -12,7 +12,8 @@ code. This plan proposes implementation order and acceptance gates.
 - The native Prompt Library merged in [#210](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/210) for [#208](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/208): one positive parent, saved negative variants, full selected text and a shared filtered image gallery. It does not invent chronological runs from counters or scan dates.
 - [Durable history storage](prompt-history-storage.md) records job attempts and outcomes from [#212](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/212), frozen prompt contexts from [#213](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/213), and observed run-owned output locators from [#214](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/214). Those locators do not verify file bytes or establish gallery-image associations. That association contract and the chronological execution timeline remain pending under [#209](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/209).
 - [#215](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/215) brings the shell, Images and natural-height Prompt Library presentation into the approved geometry and themes while retaining their existing operations. [#216](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/216) adds the shared current-file inspector with exact generation facts, scoped provenance, raw/download evidence and loaded-collection viewer controls, including a slideshow that pauses at native video. This delivery does not establish whole-application parity, change the default entry point or constitute a release. Recorded chronological run/context browsing continues under [#209](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/209); image tagging/review, System consolidation and remaining organization capabilities continue under #196.
-- Local Inspect (#249, PRs L1–L2) replaces the `#/inspect` handoff with the concept's native "Inspect a file" page: one local image or video previews inline and opens the shared viewer and full inspector on a record read entirely in the browser, with nothing uploaded, imported or stored. The legacy metadata page (`web/metadata.html`, served at `/prompt_manager/gallery.html`) stays reachable from Overview; its capabilities map to the shared inspector, except its one-click COPY ALL summary, whose ruled replacement ("Copy settings summary") is pending. Recorded deviations D60–D66 are in [TESTING.md](../TESTING.md). Local roles equal to the server's resolver follow in PR L3.
+- Local Inspect (#249, PRs L1–L3) replaces the `#/inspect` handoff with the concept's native "Inspect a file" page: one local image or video previews inline and opens the shared viewer and full inspector on a record read entirely in the browser, with nothing uploaded, imported or stored. The legacy metadata page (`web/metadata.html`, served at `/prompt_manager/gallery.html`) stays reachable from Overview; its capabilities map to the shared inspector, including its one-click COPY ALL summary, which is the shared inspector's "Copy settings summary" (D117, LI-4). PR L3 ([#286](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/286)) shipped that button and local roles equal to the server's resolver (`web/js/workspace/local-resolver.mjs`, TESTING.md "Local roles (#249 PR L3, decision LI-1)"). Recorded deviations D60–D66 and D117 are in [TESTING.md](../TESTING.md).
+- Collections ([#331](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/331), three PRs) adds named, hand-picked groups of images, stored in the library database by file location as favorites are: a COLLECTIONS sidebar group whose rows open Images filtered to one collection (`#/collection/<id>`) with rename and delete, Add to collection and Remove from collection in the Images selection bar, a Collections… checklist in the shared inspector for the side panel and the viewer, and saved views that keep a collection. Membership survives restarts, source reordering and System › Maintenance duplicate cleanup; no file is copied, moved or changed.
 
 ## Scope and compatibility
 

docs/prompt-send-bridge.md

@@ -18,9 +18,13 @@ protocol and peer checks reject unrelated messages. Same-origin scripts are
 trusted; this transport is not authentication against other scripts on that origin.
 Unsupported messaging is reported without breaking existing ComfyUI nodes.
 
+Apply works in one click: without a preview, the sender requests the ticket itself
+and applies it. Preview remains optional, and shows the exact change first.
+
 An apply timeout or abort after posting means the outcome is unknown. It cannot
 prove the host cancelled the write. The sender must inspect a fresh destination
-preview before deciding to send again; automatic retry must not overwrite edits.
+preview or choose the destination again before sending again; automatic retry must
+not overwrite edits. An error the host returned is a refusal: nothing was written.
 
 ## PromptManager adapter
 
@@ -31,7 +35,9 @@ field. Unknown/ambiguous metadata must not become an empty string.
 
 Previews show original and requested text plus positive prepend/append composition.
 Connected composition is marked unknown rather than guessing upstream values.
-Connected destination fields reject writes. Applying rechecks graph, node, widgets,
+Connected destination fields reject writes. The sender compares the destination's revision
+(node object and graph version) from discovery with the one at the preview, and refuses if the workflow changed.
+Applying rechecks graph, node, widgets,
 values and connections. Ordinary serialization cache synchronization is accepted
 only when it preserves live values. Both requested roles are checked before writing.
 

prompt_manager.py

@@ -26,13 +26,13 @@ except ImportError:
     InputTypeDict = dict
 
 try:
-    from .prompt_manager_base import PromptManagerBase
+    from .prompt_manager_base import SAVE_TO_LIBRARY_INPUT, PromptManagerBase
 except ImportError:
     import os
     import sys
 
     sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-    from prompt_manager_base import PromptManagerBase
+    from prompt_manager_base import SAVE_TO_LIBRARY_INPUT, PromptManagerBase
 
 
 class PromptManager(PromptManagerBase, ComfyNodeABC):
@@ -115,6 +115,7 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
                         )
                     },
                 ),
+                "save_to_library": SAVE_TO_LIBRARY_INPUT,
             },
             "hidden": {
                 # The queued graph and this node's id: used to tell the positive
@@ -152,6 +153,7 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
         append_text: str = "",
         prompt: Optional[Dict[str, Any]] = None,
         unique_id: Optional[str] = None,
+        save_to_library: bool = True,
     ) -> Tuple[Any]:
         """
         Encode the text prompt and save it to the database.
@@ -164,6 +166,7 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
             search_text: Text to search for in past prompts
             prepend_text: Text to prepend to the main prompt
             append_text: Text to append to the main prompt
+            save_to_library: Off: nothing is saved, counted or tracked (#90)
 
         Returns:
             Tuple containing the conditioning for the diffusion model and the final
@@ -205,7 +208,9 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
         # Save prompt to database and set execution context for gallery tracking
         prompt_id = None
         extended_tags = []
-        if storage_text and storage_text.strip():
+        if not save_to_library:
+            self._skip_not_saved(prompt, unique_id)
+        elif storage_text and storage_text.strip():
             self.logger.debug(f"Processing prompt text: {storage_text[:100]}...")
 
             # Add prepend/append info to tags if they exist
@@ -238,6 +243,9 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
             except Exception as e:
                 # Log error but don't fail the encoding
                 self.logger.warning(f"Failed to save prompt to database: {e}")
+                self._mark_untracked(prompt, unique_id)
+        else:
+            self._skip_empty_prompt(prompt, unique_id)
 
         # Perform standard CLIP text encoding using the combined text
         self.logger.debug(
@@ -246,6 +254,11 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
         tokens = clip.tokenize(encoding_text)
         conditioning = clip.encode_from_tokens_scheduled(tokens)
 
+        self.logger.info(f"CLIP encoding completed, text: {repr(encoding_text)[:80]}")
+        result = (conditioning, encoding_text)
+        if not save_to_library:
+            return result  # #90: no metadata registration, no history text
+
         # Register with ComfyUI integration for standard metadata compatibility
         node_id = f"promptmanager_{int(time.time() * 1000)}"
         self.comfyui_integration.register_prompt(
@@ -259,12 +272,15 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
                 "append_text": append_text.strip() if append_text else None,
             },
         )
-
-        self.logger.info(f"CLIP encoding completed, text: {repr(encoding_text)[:80]}")
-        result = (conditioning, encoding_text)
         record_runtime_context(
             type(self).__name__, text, prepend_text, append_text, encoding_text
         )
+        self._record_cached_item(
+            unique_id,
+            (encoding_text,),
+            storage_text,
+            fields=((text, prepend_text, append_text),),
+        )
         return result
 
     @classmethod

prompt_manager_base.py

@@ -23,8 +23,16 @@ try:
     from .utils.image_monitor import get_image_monitor
     from .utils.prompt_tracker import get_prompt_tracker
     from .utils.hashing import generate_prompt_hash
-    from .utils.prompt_graph import resolve_text, run_prompt_nodes
+    from .utils.prompt_graph import resolve_text, run_prompt_nodes, save_setting
+    from .utils.run_keys import executing_run_id, prompt_node_key
     from .utils.usage_tracking import PENDING_FIRST_USE
+    from .utils.cached_runs import (
+        CACHED_RUNS,
+        Role,
+        Seen,
+        own_counts,
+        stored_facts,
+    )
 except ImportError:
     import sys
 
@@ -34,8 +42,32 @@ except ImportError:
     from utils.image_monitor import get_image_monitor
     from utils.prompt_tracker import get_prompt_tracker
     from utils.hashing import generate_prompt_hash
-    from utils.prompt_graph import resolve_text, run_prompt_nodes
+    from utils.prompt_graph import resolve_text, run_prompt_nodes, save_setting
+    from utils.run_keys import executing_run_id, prompt_node_key
     from utils.usage_tracking import PENDING_FIRST_USE
+    from utils.cached_runs import (
+        CACHED_RUNS,
+        Role,
+        Seen,
+        own_counts,
+        stored_facts,
+    )
+
+
+# The optional input every PromptManager node declares last (#90): declared
+# last so no widget value of a workflow saved before it shifts.
+SAVE_TO_LIBRARY_INPUT = (
+    "BOOLEAN",
+    {
+        "default": True,
+        "label_on": "save",
+        "label_off": "skip",
+        "tooltip": (
+            "Save this node's prompt to the library, count its runs and link its "
+            "images. Off: PromptManager ignores the run; the output is unchanged."
+        ),
+    },
+)
 
 
 class PromptManagerBase:
@@ -69,8 +101,10 @@ class PromptManagerBase:
         """Save the prompt to the SQLite database.
 
         Runs are normally counted by the queue hook (utils/usage_tracking.py), so a
-        save only counts one when asked to (batch items whose text the hook could not
-        see) or when the hook queued this prompt before it existed.
+        save only counts one when asked to (text the hook could not see, unless
+        this job already counted the node from ComfyUI's cache) or when the hook
+        queued this prompt before it existed. The two are independent: the
+        pending catch-up is written even when ``count_run`` is False.
 
         Args:
             text: The prompt text
@@ -138,7 +172,9 @@ class PromptManagerBase:
         resolved = resolve_text(prompt_graph, node_id)
         return {
             "is_run_prompt": node_id in run_prompt_nodes(prompt_graph),
+            # The hook skips a switch it cannot read (#90): the node counts
             "seen_at_queue": resolved is not None
+            and save_setting(prompt_graph, node_id) is True
             and generate_prompt_hash(resolved) == generate_prompt_hash(text),
             "known": True,
         }
@@ -165,17 +201,137 @@ class PromptManagerBase:
         batch_item = (
             role["known"] and role["is_run_prompt"] and not role["seen_at_queue"]
         )
-        prompt_id = self._save_prompt_to_database(
-            text=text, category=category, tags=tags, count_run=batch_item
+        counts = own_counts(
+            type(self).__name__,
+            stored_facts(text),
+            Role(
+                frozenset({"text"} if role["known"] and role["is_run_prompt"] else ())
+            ),
+            Seen(text=role["seen_at_queue"]),
         )
-        if prompt_id and role["is_run_prompt"]:
+        prompt_id = self._save_prompt_to_database(
+            text=text,
+            category=category,
+            tags=tags,
+            count_run=bool(counts) and not self._replayed(unique_id),
+        )
+        if role["is_run_prompt"] and not prompt_id:
+            # The save failed: an older entry of this key must not stand in (#342)
+            self._mark_untracked(prompt_graph, unique_id)
+        elif role["is_run_prompt"]:
             self.prompt_tracker.set_current_prompt(
                 prompt_text=encoding_text,
                 additional_data={**additional_data, "prompt_id": prompt_id},
                 push_to_queue=batch_item or not role["known"],
+                **self._run_identity(prompt_graph, unique_id),
             )
         return prompt_id
 
+    def _cached_runs(self) -> Any:
+        return getattr(self, "cached_runs", CACHED_RUNS)
+
+    def _replayed(self, unique_id: Any) -> bool:
+        """This job already counted the node from ComfyUI's cache (#347).
+
+        A RAM-pressure cache can free the entry after ``execution_cached``, so
+        the node executes in a job that replayed it: its own count is skipped.
+        """
+        try:
+            return unique_id is not None and self._cached_runs().replayed(unique_id)
+        except Exception as error:
+            self.logger.debug(f"Could not check the cached-run replay: {error}")
+            return False
+
+    def _record_cached_item(
+        self,
+        unique_id: Any,
+        outputs: tuple,
+        text: Any,
+        negative_text: Any = None,
+        fields: Optional[tuple] = None,
+    ) -> None:
+        """Remember this execution's outputs and stored text for cached jobs (#347).
+
+        ``fields`` are the ``(raw, prepend, append)`` texts this execution gave
+        ``record_runtime_context`` for each output, so a cached job's history
+        can record them too (#403).
+        """
+        if unique_id is None:
+            return
+        try:
+            self._cached_runs().record_item(
+                unique_id, outputs, stored_facts(text, negative_text), fields
+            )
+        except Exception as error:
+            self.logger.debug(f"Could not record the cached-run receipt: {error}")
+
+    @staticmethod
+    def _run_identity(prompt_graph: Any, unique_id: Any) -> Dict[str, Any]:
+        """The tracker's proof that an image belongs to this execution (#342)."""
+        return {
+            "run_key": prompt_node_key(prompt_graph, unique_id),
+            "run_id": executing_run_id(),
+        }
+
+    def _skip_not_saved(self, prompt_graph: Any = None, unique_id: Any = None) -> None:
+        """``save_to_library`` is off: nothing is saved, counted or tracked (#90).
+
+        A literal off makes the node no run prompt (``utils.prompt_graph``), so
+        no image looks up its run key. A switch wired from a node the graph
+        cannot read keeps the node a run prompt: its run key is marked like an
+        empty editor's, so an earlier run's entry of that key never links this
+        run's images.
+        """
+        message = (
+            "Save to library is off for this node: its prompt is not saved, the "
+            "run is not counted and its images are not linked"
+        )
+        if isinstance(prompt_graph, dict) and unique_id is not None:
+            self._mark_untracked(prompt_graph, unique_id, message)
+        else:
+            self.logger.info(message)
+
+    def _skip_empty_prompt(self, prompt_graph: Any, unique_id: Any) -> None:
+        """The ``text`` editor is empty: save and track nothing (#342).
+
+        Prepend and append text are deliberately never stored as the prompt, so
+        an empty editor has no prompt to save, count or link. For the run's
+        positive node this is logged at info and the node's run key is marked
+        empty (``_mark_untracked``).
+        """
+        self._mark_untracked(
+            prompt_graph,
+            unique_id,
+            "Prompt text is empty (prepend and append text are not stored): "
+            "nothing saved or tracked, and this run's images stay unlinked",
+        )
+
+    def _mark_untracked(
+        self, prompt_graph: Any, unique_id: Any, message: Optional[str] = None
+    ) -> None:
+        """This execution of a positive node has no prompt to link (#342).
+
+        An empty editor, or a save or tracking failure. The node's run key is
+        marked with a placeholder, so no earlier prompt of that key can be
+        linked to this run's images, and a batch item keeps its queue place.
+        ``message`` is logged at info for a positive node.
+        """
+        try:
+            role = self._prompt_role(prompt_graph, unique_id, "")
+            if not role["is_run_prompt"]:
+                if message:
+                    self.logger.debug("Empty text on a non-positive node; not saved")
+                return
+            if message:
+                self.logger.info(message)
+            self.prompt_tracker.mark_empty(
+                push_to_queue=role["known"],
+                **self._run_identity(prompt_graph, unique_id),
+            )
+        except Exception as error:
+            # Tracking must never block generation
+            self.logger.warning(f"Could not mark the untracked prompt: {error}")
+
     def _record_use(self, prompt_id: int, times: int = 1) -> None:
         """Count runs; a tracking failure must never block generation."""
         try:
@@ -213,8 +369,10 @@ class PromptManagerBase:
 
         cache = get_trigger_cache()
 
-        # Lazy-load cache on first use
-        if not cache.is_loaded and IntegrationConfig.LORA_MANAGER_PATH:
+        # Lazy-load cache on first use; a failed load waits before the next try
+        # (#355: never a rescan of LoraManager's folders on every run).
+        retry_due = getattr(cache, "retry_due", lambda: True)
+        if not cache.is_loaded and IntegrationConfig.LORA_MANAGER_PATH and retry_due():
             cache.load(IntegrationConfig.LORA_MANAGER_PATH)
 
         modified, injected = inject_trigger_words(text, cache)

prompt_manager_pair.py

@@ -12,9 +12,21 @@ else:
 
 try:
     from .prompt_manager import IO, InputTypeDict, PromptManagerFocused
+    from .utils.cached_runs import (
+        PAIR,
+        Role,
+        Seen,
+        own_counts,
+        pair_components,
+        stored_facts,
+    )
     from .utils.hashing import generate_prompt_hash
     from .utils.pair_execution import record_pair_execution
-    from .utils.prompt_graph import intrinsic_pair_nodes, run_prompt_components
+    from .utils.prompt_graph import (
+        intrinsic_pair_nodes,
+        run_prompt_components,
+        save_setting,
+    )
     from .utils.prompt_pair_resolver import resolve_text_input
     from .utils.usage_tracking import (
         PENDING_FIRST_USE,
@@ -23,9 +35,21 @@ try:
     )
 except ImportError:
     from prompt_manager import IO, InputTypeDict, PromptManagerFocused
+    from utils.cached_runs import (
+        PAIR,
+        Role,
+        Seen,
+        own_counts,
+        pair_components,
+        stored_facts,
+    )
     from utils.hashing import generate_prompt_hash
     from utils.pair_execution import record_pair_execution
-    from utils.prompt_graph import intrinsic_pair_nodes, run_prompt_components
+    from utils.prompt_graph import (
+        intrinsic_pair_nodes,
+        run_prompt_components,
+        save_setting,
+    )
     from utils.prompt_pair_resolver import resolve_text_input
     from utils.usage_tracking import (
         PENDING_FIRST_USE,
@@ -74,8 +98,13 @@ class PromptManagerPair(PromptManagerFocused):
         prompt: Optional[Dict[str, Any]] = None,
         unique_id: Optional[str] = None,
         extra_pnginfo: Optional[Dict[str, Any]] = None,
+        save_to_library: bool = True,
     ) -> Tuple[Any, Any, str, str]:
-        """Encode both sides once; database and tracking failures never block CLIP."""
+        """Encode both sides once; database and tracking failures never block CLIP.
+
+        With ``save_to_library`` off nothing is saved, counted, tracked or
+        recorded (#90); the four outputs are the same.
+        """
         if clip is None:
             raise RuntimeError("ERROR: clip input is invalid: None")
         if not isinstance(text, str) or not isinstance(negative_text, str):
@@ -93,6 +122,10 @@ class PromptManagerPair(PromptManagerFocused):
         negative_conditioning = clip.encode_from_tokens_scheduled(
             clip.tokenize(negative_text)
         )
+        result = positive_conditioning, negative_conditioning, positive, negative_text
+        if not save_to_library:
+            self._skip_not_saved(prompt, unique_id)
+            return result
         record_pair_execution(
             extra_pnginfo,
             unique_id,
@@ -117,85 +150,29 @@ class PromptManagerPair(PromptManagerFocused):
         )
         is_positive = "text" in used_fields
         pair_used = node_id in intrinsic_pair_nodes(prompt) if known else True
-        try:
-            pair_id = self.db.save_prompt_pair(
-                text,
-                negative_text,
-                category=category.strip() or None,
-                tags=self._parse_tags(tags),
-            )
-            pair = self.db.get_prompt_pair(pair_id)
-            prompt_id = pair["positive_prompt_id"]
-            resolved_positive = (
-                resolve_text_input(prompt, node_id, "text") if known else None
-            )
-            resolved_negative = (
-                resolve_text_input(prompt, node_id, "negative_text") if known else None
-            )
-            positive_seen = resolved_positive is not None and generate_prompt_hash(
-                resolved_positive
-            ) == generate_prompt_hash(text)
-            pair_seen = (
-                positive_seen
-                and resolved_negative is not None
-                and generate_prompt_hash(resolved_negative)
-                == generate_prompt_hash(negative_text)
-            )
-            positive_pending = getattr(self, "pending_first_use", PENDING_FIRST_USE)
-            pair_pending = getattr(
-                self, "pending_pair_first_use", PENDING_PAIR_FIRST_USE
-            )
-            counted = set()
-            for field, component_text, resolved_text, component_id in (
-                ("text", text, resolved_positive, prompt_id),
-                (
-                    "negative_text",
+        if not text.strip():
+            # Nothing to store: prepend/append are never the stored prompt (#342)
+            self._skip_empty_prompt(prompt, unique_id)
+        else:
+            try:
+                pair_id, prompt_id = self._save_and_track_pair(
+                    text,
                     negative_text,
-                    resolved_negative,
-                    pair["negative_prompt_id"],
-                ),
-            ):
-                component_hash = generate_prompt_hash(component_text)
-                if (
-                    field not in used_fields
-                    or not component_id
-                    or component_hash in counted
-                ):
-                    continue
-                counted.add(component_hash)
-                seen_at_queue = (
-                    resolved_text is not None
-                    and generate_prompt_hash(resolved_text) == component_hash
+                    positive,
+                    category=category,
+                    tags=tags,
+                    prepend_text=prepend_text,
+                    append_text=append_text,
+                    prompt=prompt,
+                    unique_id=unique_id,
+                    used_fields=used_fields,
+                    is_positive=is_positive,
+                    pair_used=pair_used,
                 )
-                runs = positive_pending.consume(component_hash) + int(not seen_at_queue)
-                if runs:
-                    self.db.record_prompt_use(component_id, times=runs)
-            if pair_used:
-                runs = pair_pending.consume(pair_run_key(text, negative_text)) + int(
-                    not pair_seen
-                )
-                if runs:
-                    self.db.record_prompt_pair_use(pair_id, times=runs)
-            if is_positive:
-                self.prompt_tracker.set_current_prompt(
-                    prompt_text=positive,
-                    additional_data={
-                        "prompt_id": prompt_id,
-                        "pair_id": pair_id if pair_used else None,
-                        "positive_text": positive,
-                        "negative_text": negative_text,
-                        "positive_storage_text": text.strip(),
-                        "negative_storage_text": negative_text.strip(),
-                        "prepend_text": prepend_text.strip(),
-                        "append_text": append_text.strip(),
-                        "final_text": positive,
-                        "category": category.strip() or None,
-                        "tags": self._parse_tags(tags) or [],
-                    },
-                    push_to_queue=not positive_seen or (pair_used and not pair_seen),
-                )
-        except Exception as error:
-            self.logger.warning(f"Could not save or track paired prompt: {error}")
+            except Exception as error:
+                self.logger.warning(f"Could not save or track paired prompt: {error}")
+                if is_positive:
+                    self._mark_untracked(prompt, unique_id)
         try:
             self.comfyui_integration.register_prompt(
                 f"promptmanager_pair_{time.time_ns()}",
@@ -209,7 +186,6 @@ class PromptManagerPair(PromptManagerFocused):
             )
         except Exception as error:
             self.logger.warning(f"Could not register paired metadata: {error}")
-        result = positive_conditioning, negative_conditioning, positive, negative_text
         record_runtime_context(
             type(self).__name__,
             text,
@@ -221,8 +197,115 @@ class PromptManagerPair(PromptManagerFocused):
         record_runtime_context(
             type(self).__name__, negative_text, "", "", negative_text, role="negative"
         )
+        self._record_cached_item(
+            unique_id,
+            (positive, negative_text),
+            text,
+            negative_text,
+            fields=((text, prepend_text, append_text), (negative_text, "", "")),
+        )
         return result
 
+    def _save_and_track_pair(
+        self,
+        text: str,
+        negative_text: str,
+        positive: str,
+        *,
+        category: str,
+        tags: str,
+        prepend_text: str,
+        append_text: str,
+        prompt: Optional[Dict[str, Any]],
+        unique_id: Optional[str],
+        used_fields: set,
+        is_positive: bool,
+        pair_used: bool,
+    ) -> Tuple[Optional[int], Optional[int]]:
+        """Save the recipe, count its runs and register the positive for linking."""
+        known = isinstance(prompt, dict) and unique_id is not None
+        node_id = str(unique_id)
+        pair_id = self.db.save_prompt_pair(
+            text,
+            negative_text,
+            category=category.strip() or None,
+            tags=self._parse_tags(tags),
+            # The Library shows the prompt's own category and tags, not the pair's.
+            fill_new_positive=True,
+        )
+        pair = self.db.get_prompt_pair(pair_id)
+        prompt_id = pair["positive_prompt_id"]
+        resolved_positive = (
+            resolve_text_input(prompt, node_id, "text") if known else None
+        )
+        resolved_negative = (
+            resolve_text_input(prompt, node_id, "negative_text") if known else None
+        )
+        if known and save_setting(prompt, node_id) is not True:
+            # The hook skips a switch it cannot read (#90): the node counts
+            resolved_positive = resolved_negative = None
+        positive_seen = resolved_positive is not None and generate_prompt_hash(
+            resolved_positive
+        ) == generate_prompt_hash(text)
+        pair_seen = (
+            positive_seen
+            and resolved_negative is not None
+            and generate_prompt_hash(resolved_negative)
+            == generate_prompt_hash(negative_text)
+        )
+        negative_seen = resolved_negative is not None and generate_prompt_hash(
+            resolved_negative
+        ) == generate_prompt_hash(negative_text)
+        positive_pending = getattr(self, "pending_first_use", PENDING_FIRST_USE)
+        pair_pending = getattr(self, "pending_pair_first_use", PENDING_PAIR_FIRST_USE)
+        facts = stored_facts(text, negative_text)
+        role = Role(frozenset(used_fields), pair_used)
+        # Own counts are skipped when this job already counted the node from
+        # ComfyUI's cache (#347); the pending first-use catch-up never is.
+        counts = (
+            []
+            if self._replayed(unique_id)
+            else own_counts(
+                PAIR, facts, role, Seen(positive_seen, negative_seen, pair_seen)
+            )
+        )
+        component_ids = {"text": prompt_id, "negative_text": pair["negative_prompt_id"]}
+        for field, component_hash in pair_components(facts, role):
+            component_id = component_ids[field]
+            if not component_id:
+                continue
+            runs = positive_pending.consume(component_hash) + int(
+                ("prompt", component_hash) in counts
+            )
+            if runs:
+                self.db.record_prompt_use(component_id, times=runs)
+        if pair_used:
+            runs = pair_pending.consume(pair_run_key(text, negative_text)) + int(
+                ("pair", facts.positive_hash, facts.negative_hash) in counts
+            )
+            if runs:
+                self.db.record_prompt_pair_use(pair_id, times=runs)
+        if is_positive:
+            self.prompt_tracker.set_current_prompt(
+                prompt_text=positive,
+                additional_data={
+                    "prompt_id": prompt_id,
+                    "pair_id": pair_id if pair_used else None,
+                    "positive_text": positive,
+                    "negative_text": negative_text,
+                    "positive_storage_text": text.strip(),
+                    "negative_storage_text": negative_text.strip(),
+                    "prepend_text": prepend_text.strip(),
+                    "append_text": append_text.strip(),
+                    "final_text": positive,
+                    "category": category.strip() or None,
+                    "tags": self._parse_tags(tags) or [],
+                },
+                push_to_queue=not positive_seen or (pair_used and not pair_seen),
+                **self._run_identity(prompt, unique_id),
+            )
+        return pair_id, prompt_id
+
     @classmethod
     def IS_CHANGED(
         cls,

prompt_manager_text.py

@@ -26,13 +26,13 @@ except ImportError:
     InputTypeDict = dict
 
 try:
-    from .prompt_manager_base import PromptManagerBase
+    from .prompt_manager_base import SAVE_TO_LIBRARY_INPUT, PromptManagerBase
 except ImportError:
     import os
     import sys
 
     sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-    from prompt_manager_base import PromptManagerBase
+    from prompt_manager_base import SAVE_TO_LIBRARY_INPUT, PromptManagerBase
 
 
 class PromptManagerText(PromptManagerBase, ComfyNodeABC):
@@ -115,6 +115,7 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
                         ),
                     },
                 ),
+                "save_to_library": SAVE_TO_LIBRARY_INPUT,
             },
             "hidden": {
                 # The queued graph and this node's id: used to tell the positive
@@ -149,6 +150,7 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
         append_text: str = "",
         prompt: Optional[Dict[str, Any]] = None,
         unique_id: Optional[str] = None,
+        save_to_library: bool = True,
     ) -> Tuple[str]:
         """
         Process the text prompt and save it to the database.
@@ -160,6 +162,7 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
             search_text: Text to search for in past prompts
             prepend_text: Text to prepend to the main prompt
             append_text: Text to append to the main prompt
+            save_to_library: Off: nothing is saved, counted or tracked (#90)
 
         Returns:
             Tuple containing the final processed text string
@@ -184,7 +187,9 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
         # Save prompt to database and set execution context for gallery tracking
         prompt_id = None
         extended_tags = []
-        if storage_text and storage_text.strip():
+        if not save_to_library:
+            self._skip_not_saved(prompt, unique_id)
+        elif storage_text and storage_text.strip():
             self.logger.debug(f"Processing prompt text: {storage_text[:100]}...")
 
             # Add prepend/append info to tags if they exist
@@ -216,6 +221,14 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
             except Exception as e:
                 # Log error but don't fail the processing
                 self.logger.warning(f"Failed to save prompt to database: {e}")
+                self._mark_untracked(prompt, unique_id)
+        else:
+            self._skip_empty_prompt(prompt, unique_id)
+
+        self.logger.debug(f"Text processing completed: {final_text[:100]}...")
+        result = (final_text,)
+        if not save_to_library:
+            return result  # #90: no metadata registration, no history text
 
         # Register with ComfyUI integration for standard metadata compatibility
         node_id = f"promptmanagertext_{int(time.time() * 1000)}"
@@ -230,12 +243,15 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
                 "append_text": append_text.strip() if append_text else None,
             },
         )
-
-        self.logger.debug(f"Text processing completed: {final_text[:100]}...")
-        result = (final_text,)
         record_runtime_context(
             type(self).__name__, text, prepend_text, append_text, final_text
         )
+        self._record_cached_item(
+            unique_id,
+            (final_text,),
+            storage_text,
+            fields=((text, prepend_text, append_text),),
+        )
         return result
 
     @classmethod

py/api/__init__.py

@@ -5,9 +5,24 @@ Split into domain-specific mixins for maintainability:
 - ImageRoutesMixin: gallery, thumbnails, image serving and linking
 - AdminRoutesMixin: duplicates, stats, settings, diagnostics, maintenance, backup, scan
 - LoggingRoutesMixin: log management endpoints
+- LogCleanupMixin: the reviewed rotated-log cleanup (a maintenance plan)
+- DiagnosticsReportMixin: the shareable diagnostic report, without secrets or paths
 - AutotagRoutesMixin: auto-tagging model management and tagging
 - JobRoutesMixin: the session job registry behind Activity (py/api/jobs.py)
 - ImageTagJobsMixin: bulk Add tags / Auto-tag as image-tags jobs
+- ThumbnailJobsMixin: thumbnail plan, thumbnail job and the gallery stream on it
+- ThumbnailConvertMixin: Convert outdated thumbnails (#317), a thumbnail job mode
+- AutoThumbnailMixin: previews made on first view, on capture and after a scan
+- ReviewRoutesMixin: the Needs review queue, item detail and decisions
+- ReviewRecordingMixin: Needs review recording for the scan and the monitor
+- ReviewEventsMixin: the Needs review change stream an open page re-reads on
+- SystemRoutesMixin: System health check, Overview and usage statistics
+- LibraryLocationRoutesMixin: where the library lives, Move library, Keep it here
+- MaintenanceRoutesMixin: reviewed maintenance plans (preview, apply)
+- LibraryDeleteRoutesMixin: the Library's Delete… (archive paired) and Restore
+- WorkflowStorageRoutesMixin: store each workflow once, then Reclaim space
+- DuplicateFilesMixin: duplicate image files as find and removal jobs
+- LoraImportRoutesMixin: the reviewed, staged LoRA reference import job
 """
 
 import asyncio
@@ -15,33 +30,60 @@ import datetime
 import functools
 import gzip as gzip_module
 import json
+import math
 import ntpath
 import os
 import re
+import threading
 from pathlib import Path
 
 from aiohttp import web
 from PIL import Image
 
 from .prompts import PromptRoutesMixin
+from .revalidation import revalidated_response
 from .pairs import PairRoutesMixin
 from .library import LibraryRoutesMixin
+from .library_delete import LibraryDeleteRoutesMixin
 from .vocabulary import VocabularyRoutesMixin
 from .annotations import AnnotationRoutesMixin
+from .review import ReviewRoutesMixin
+from .review_recording import ReviewRecordingMixin
+from .review_events import ReviewEventsMixin
 from .saved_views import SavedViewRoutesMixin
+from .collections import CollectionRoutesMixin
 from .history import HistoryRoutesMixin
 from .images import ImageRoutesMixin
+from .gallery_listing import WALK_STOPPED, RootListing, WalkOutcome
 from .admin import AdminRoutesMixin
 from .settings_routes import SettingsRoutesMixin
 from .job_routes import JobRoutesMixin
 from .image_tag_jobs import ImageTagJobsMixin
+from .thumbnail_jobs import ThumbnailJobsMixin
+from .thumbnail_convert import ThumbnailConvertMixin
+from .thumbnail_auto import AutoThumbnailMixin
+from .system import SystemRoutesMixin
+from .library_location import LibraryLocationRoutesMixin
+from .maintenance import MaintenanceRoutesMixin
+from .backup_restore import BackupRestoreRoutesMixin
+from .workflow_storage import WorkflowStorageRoutesMixin
+from .duplicate_files import DuplicateFilesMixin
+from .maintenance_plans import PlanStore
 from .jobs import JobRegistry
 from .logging_routes import LoggingRoutesMixin
+from .log_cleanup import LogCleanupMixin
+from .diagnostics_report import DiagnosticsReportMixin
 from .autotag_routes import AutotagRoutesMixin
 from .lora_integration import LoraIntegrationMixin
+from .lora_import_routes import LoraImportRoutesMixin
+from .image_categories import ImageCategoryRoutesMixin
+from .image_delete import ImageDeleteMixin
+from .image_return import ImageReturnMixin
+from .source_rescope import SourceRescopeMixin
 
 try:
     from ...database.operations import PromptDatabase
+    from ...utils.file_prompt import parse_comfyui_prompt
     from ...utils.logging_config import get_logger
     from ...utils.video_metadata import is_video_path, read_video_metadata
 except ImportError:
@@ -51,6 +93,7 @@ except ImportError:
         0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
     )
     from database.operations import PromptDatabase
+    from utils.file_prompt import parse_comfyui_prompt
     from utils.logging_config import get_logger
     from utils.video_metadata import is_video_path, read_video_metadata
 
@@ -60,6 +103,18 @@ def _get_project_root():
     return os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
+# How long an executor thread waits for the loop to drop the listings.
+_OFF_LOOP_DROP_WAIT = 30
+
+
+def _running_loop():
+    """The event loop running in this thread, or None."""
+    try:
+        return asyncio.get_running_loop()
+    except RuntimeError:
+        return None
+
+
 def _public_path(path) -> str:
     """Render a server path for an API response without revealing the layout.
 
@@ -250,6 +305,10 @@ async def _gzip_middleware(request, handler):
     if len(compressed) >= len(body):
         return response
 
+    # A length the handler declared is the uncompressed one: drop it so the
+    # compressed body's own length is sent (#329: a stale length made the
+    # browser wait for bytes that never came).
+    response.headers.pop("Content-Length", None)
     response.body = compressed
     response.headers["Content-Encoding"] = "gzip"
     response.headers["Vary"] = "Accept-Encoding"
@@ -259,9 +318,14 @@ async def _gzip_middleware(request, handler):
 class PromptManagerAPI(
     PairRoutesMixin,
     LibraryRoutesMixin,
+    LibraryDeleteRoutesMixin,
     VocabularyRoutesMixin,
     AnnotationRoutesMixin,
+    ReviewRoutesMixin,
+    ReviewRecordingMixin,
+    ReviewEventsMixin,
     SavedViewRoutesMixin,
+    CollectionRoutesMixin,
     HistoryRoutesMixin,
     PromptRoutesMixin,
     ImageRoutesMixin,
@@ -269,9 +333,25 @@ class PromptManagerAPI(
     SettingsRoutesMixin,
     JobRoutesMixin,
     ImageTagJobsMixin,
+    ThumbnailJobsMixin,
+    ThumbnailConvertMixin,
+    AutoThumbnailMixin,
+    SystemRoutesMixin,
+    LibraryLocationRoutesMixin,
+    MaintenanceRoutesMixin,
+    BackupRestoreRoutesMixin,
+    WorkflowStorageRoutesMixin,
+    DuplicateFilesMixin,
     LoggingRoutesMixin,
+    LogCleanupMixin,
+    DiagnosticsReportMixin,
     AutotagRoutesMixin,
     LoraIntegrationMixin,
+    LoraImportRoutesMixin,
+    ImageCategoryRoutesMixin,
+    ImageDeleteMixin,
+    ImageReturnMixin,
+    SourceRescopeMixin,
 ):
     """REST API handler for PromptManager operations and web interface.
 
@@ -299,11 +379,28 @@ class PromptManagerAPI(
         # Private media captures of the annotation routes (py/api/annotations.py),
         # created on first use with their restore subscription.
         self._captures = None
+        # Capture tokens Generate suggestions ran on: a save of one marks the
+        # image auto-tagged (#379).
+        self._suggested_tokens = set()
         self._captures_unsubscribe = None
+        # Set by close_annotation_captures (shutdown): no store is recreated.
+        self._captures_closed = False
         self._cached_output_dir = None  # Lazy-cached by _find_comfyui_output_dir()
         self._html_cache = {}  # Cached HTML file contents keyed by path
-        self._gallery_cache = {}  # dict: path_str -> (files, timestamp)
-        self._gallery_cache_ttl = 30  # Cache TTL in seconds
+        # One shared listing per output root, keyed by str(root), and the
+        # background walks refreshing them (py/api/gallery_listing.py, #348).
+        self._root_listings: dict[str, RootListing] = {}
+        self._listing_tasks: set = set()
+        # The loop the walks run on, set by the startup warm-up or the first
+        # walk; the monitor's thread hops onto it (notify_output_settled).
+        self._listing_loop = None
+        # Set by cleanup_gallery_listings: no background walk starts after it.
+        self._listings_closed = False
+        # The workspace caches rebuilt behind a stored walk that changed the
+        # listing, once the host's startup warm-up has run (#348).
+        self._workspace_warm_enabled = False
+        self._workspace_warm_task = None
+        self._workspace_warm_again = False
         # Workspace listing values and sort orders derived from a cached scan
         # (py/api/images.py _workspace_listed/_workspace_orders).
         self._workspace_listed_cache = {}
@@ -323,8 +420,40 @@ class PromptManagerAPI(
         self._jobs = JobRegistry(self.logger)
         # The latest metadata scan job (also in self._jobs), if one was started.
         self._scan_job = None
-        # The current duplicate scan (a py/api/jobs.Job outside the registry).
-        self._duplicate_job = None
+        # Duplicate-file finds by job id, with the groups each one holds for
+        # review (py/api/duplicate_files.py); the last few only.
+        self._duplicate_finds = {}
+        # The last System health report this session and the database
+        # generation it describes, bumped when a restore replaces self.db
+        # (py/api/system.py).
+        self._health_report = None
+        self._db_generation = 0
+        # Reviewed maintenance plans under single-use tokens; a restart
+        # forgets them (py/api/maintenance.py).
+        self._maintenance_plans = PlanStore()
+        # #480: the removal marks the workspace listing hides, read once now so
+        # a listing page reads no table until a mark or the database changes.
+        try:
+            self._removal_marks()
+        except Exception:
+            self.logger.warning("Removal marks could not be read", exc_info=True)
+
+        # A safety backup interrupted by a crash leaves its temp copy behind.
+        try:
+            from .maintenance import sweep_safety_temps
+
+            from .backup_restore import restore_stage
+
+            swept = sweep_safety_temps(self.db.model.db_path)
+            swept += restore_stage.sweep_restore_temps(self.db.model.db_path)
+            if swept:
+                self.logger.info(f"Removed {len(swept)} interrupted backup copies")
+        except Exception as e:
+            self.logger.warning(f"Could not sweep interrupted backup copies: {e}")
+
+        # A crash between Move library's switch and its rename left the old
+        # node-folder files under their own names (#249 S6b): rename them now.
+        self.finish_interrupted_library_move()
 
         # Run cleanup on initialization to remove any existing duplicates
         try:
@@ -397,23 +526,122 @@ class PromptManagerAPI(
             return await loop.run_in_executor(None, call)
         return await loop.run_in_executor(None, func, *args)
 
-    def invalidate_gallery_cache(self):
-        """Invalidate the cached gallery file listing.
+    def _root_listing(self, root) -> RootListing:
+        """The shared listing record of one output root (created empty)."""
+        return self._root_listings.setdefault(str(root), RootListing())
 
-        Called by the image monitor when new files are detected. The workspace
-        keeps its per-file values and card facts, so the next request rebuilds
-        only new or changed files (see ImageRoutesMixin._workspace_listed and
-        _card_facts); its sort orders
-        are dropped, as the rescan replaces them anyway, and so is the
-        favorites lookup, which holds the old scan list.
+    def invalidate_gallery_cache(self, retire=False):
+        """Drop every root's listing, so the next read waits for a new walk.
+
+        Called after files are deleted through the app (duplicate cleanup and
+        the legacy duplicate delete), never for thumbnail writes. A walk
+        running now is not stored (its generation moved on). The workspace
+        keeps its per-file values and card facts, so the next request
+        rebuilds only new or changed files (see ImageRoutesMixin.
+        _workspace_listed and _card_facts); its sort orders are dropped, as
+        the walk replaces them anyway, and so is the favorites lookup, which
+        holds the old list.
+
+        ``retire`` (the roots changed): no background walk of a root starts
+        again until a reader or the warm-up asks for it.
         """
-        self._gallery_cache = {}
+        loop = self._listing_loop
+        if loop is not None and loop.is_running() and _running_loop() is not loop:
+            # Called from an executor thread (a settings save, a test seed):
+            # listing records are touched only on the loop, and the caller
+            # expects the drop done when this returns.
+            dropped = threading.Event()
+
+            def drop():
+                try:
+                    self._drop_listings(retire)
+                finally:
+                    dropped.set()
+
+            loop.call_soon_threadsafe(drop)
+            if not dropped.wait(_OFF_LOOP_DROP_WAIT):
+                self.logger.warning("The Images listing drop is still queued")
+            return
+        self._drop_listings(retire)
+
+    def _drop_listings(self, retire=False):
+        for record in list(self._root_listings.values()):
+            record.drop()
+            if retire:
+                # The roots changed: a removed root must not come back through
+                # a pending timer, a trailing walk or a monitor mark.
+                record.retired = True
+                record.dirty = False
+                handle, record.rewalk_handle = record.rewalk_handle, None
+                if handle is not None:
+                    handle.cancel()
         self._workspace_order_cache = {}
         self._favorite_listing_cache = {}
 
+    async def warm_gallery_listings(self, app=None):
+        """Start one background walk per output root (aiohttp on_startup).
+
+        Registered by the root ``__init__.py``, so the first Images open after
+        ComfyUI starts finds the listing built. It never fails startup: a
+        root that cannot be walked is retried by the first read.
+        """
+        # Each stored walk that changes a listing then rebuilds the workspace's
+        # per-file values, orders and favorites behind it (_warm_workspace).
+        self._workspace_warm_enabled = True
+        self._listing_loop = asyncio.get_running_loop()
+        self._listings_closed = False
+        try:
+            for root in self._get_all_output_dirs():
+                self._root_listing(root).retired = False
+                self._schedule_walk(root)
+        except Exception as e:
+            self.logger.warning(f"Could not warm the Images listing: {e}")
+
+    def notify_output_settled(self, path):
+        """The image monitor settled an output at *path* (any thread).
+
+        Called on the monitor's worker thread with the resolved path. It only
+        hands the path to the event loop (``output_settled``) and never
+        touches a listing from this thread; before the loop is known, or once
+        the listings are cleaned up, it does nothing.
+        """
+        loop = self._listing_loop
+        if loop is None or self._listings_closed or not loop.is_running():
+            return
+        try:
+            loop.call_soon_threadsafe(self.output_settled, path)
+        except RuntimeError:
+            pass  # the loop closed meanwhile (shutdown)
+
+    async def cleanup_gallery_listings(self, app=None):
+        """Stop the background listing walks (aiohttp on_cleanup)."""
+        self._workspace_warm_enabled = False
+        # No mark, deferred walk or trailing walk starts a walk from now on.
+        self._listings_closed = True
+        # Before any await, so no delayed refresh can start a walk meanwhile.
+        for record in self._root_listings.values():
+            handle, record.rewalk_handle = record.rewalk_handle, None
+            if handle is not None:
+                handle.cancel()
+        warm, self._workspace_warm_task = self._workspace_warm_task, None
+        if warm is not None:
+            warm.cancel()
+            await asyncio.gather(warm, return_exceptions=True)
+        tasks = list(self._listing_tasks)
+        for task in tasks:
+            task.cancel()
+        if tasks:
+            await asyncio.gather(*tasks, return_exceptions=True)
+        self._listing_tasks.clear()
+        # A walk cancelled before its first step never ran its own cleanup.
+        for record in self._root_listings.values():
+            flight, record.flight = record.flight, None
+            if flight is not None and not flight.done():
+                flight.set_result(WalkOutcome.failed(RuntimeError(WALK_STOPPED)))
+
     def reset_gallery_roots_cache(self):
         """Drop every cached listing and derived workspace value (roots changed)."""
-        self.invalidate_gallery_cache()
+        self.invalidate_gallery_cache(retire=True)
         self._workspace_listed_cache = {}
         self._workspace_card_cache = {}
 
@@ -439,52 +667,71 @@ class PromptManagerAPI(
 
         @routes.get("/prompt_manager/workspace")
         async def serve_workspace_ui(request: web.Request) -> web.Response:
-            return self._serve_html_page("workspace.html", "Workspace")
+            return self._serve_html_page(request, "workspace.html", "Workspace")
 
         @routes.get("/prompt_manager/web")
         async def serve_web_ui(request):
-            return self._serve_html_page("index.html", "Web UI", cache=False)
+            return self._serve_html_page(request, "index.html", "Web UI", cache=False)
 
         @routes.get("/prompt_manager/gallery.html")
         async def serve_gallery_ui(request):
-            return self._serve_html_page("metadata.html", "Gallery")
+            return self._serve_html_page(request, "metadata.html", "Gallery")
 
         @routes.get("/prompt_manager/admin")
         async def serve_admin_ui(request):
-            return self._serve_html_page("admin.html", "Admin UI")
+            return self._serve_html_page(request, "admin.html", "Admin UI")
 
         @routes.get("/prompt_manager/gallery")
         async def serve_gallery_admin_ui(request):
-            return self._serve_html_page("gallery.html", "Gallery")
+            return self._serve_html_page(request, "gallery.html", "Gallery")
 
         # ── Static file serving ───────────────────────────────────────
 
         @routes.get("/prompt_manager/lib/{filepath:.*}")
         async def serve_lib_static(request):
             """Serve static library files (JS, CSS) from web/lib directory."""
-            return self._serve_static_file("lib", request.match_info.get("filepath"))
+            return self._serve_static_file(
+                request, "lib", request.match_info.get("filepath")
+            )
 
         @routes.get("/prompt_manager/js/{filepath:.*}")
         async def serve_js_static(request):
             """Serve static JavaScript files from web/js directory."""
-            return self._serve_static_file("js", request.match_info.get("filepath"))
+            return self._serve_static_file(
+                request, "js", request.match_info.get("filepath")
+            )
 
         # ── Register domain-specific routes from mixins ───────────────
 
         self._register_pair_routes(routes)
         self._register_library_routes(routes)
+        self._register_library_delete_routes(routes)
         self._register_vocabulary_routes(routes)
         self._register_annotation_routes(routes)
+        self._register_review_routes(routes)
+        self._register_review_event_routes(routes)
         self._register_saved_view_routes(routes)
+        self._register_collection_routes(routes)
         self._register_history_routes(routes)
         self._register_prompt_routes(routes)
         self._register_image_routes(routes)
         self._register_admin_routes(routes)
         self._register_settings_routes(routes)
         self._register_job_routes(routes)
+        self._register_system_routes(routes)
+        self._register_diagnostics_report_routes(routes)
+        self._register_library_location_routes(routes)
+        self._register_maintenance_routes(routes)
+        self._register_backup_restore_routes(routes)
+        self._register_duplicate_routes(routes)
         self._register_logging_routes(routes)
         self._register_autotag_routes(routes)
         self._register_lora_routes(routes)
+        self._register_lora_import_routes(routes)
+        self._register_image_category_routes(routes)
+        self._register_image_delete_routes(routes)
+        self._register_image_return_routes(routes)
+        self._register_rescope_routes(routes)
 
         # Register gzip compression middleware (once)
         global _gzip_registered
@@ -500,8 +747,11 @@ class PromptManagerAPI(
 
         self.logger.info("All routes registered with decorator pattern")
 
-    def _serve_static_file(self, subdir, filepath):
-        """Serve ``web/<subdir>/<filepath>``; 403 outside it, 404 when missing."""
+    def _serve_static_file(self, request, subdir, filepath):
+        """Serve ``web/<subdir>/<filepath>``; 403 outside it, 404 when missing.
+
+        A served file is revalidated on every reuse (#478, ``revalidation``).
+        """
         root = os.path.join(_get_project_root(), "web", subdir)
         exact = STATIC_EXACT_FILES.get((subdir, filepath))
         # An exact file passes the same containment check, its extension only.
@@ -516,10 +766,13 @@ class PromptManagerAPI(
             return web.Response(text="Not Found", status=404)
         with open(file_path, "rb") as f:
             content = f.read()
-        return web.Response(body=content, content_type=content_type)
+        return revalidated_response(request, content, content_type)
 
-    def _serve_html_page(self, filename, title, cache=True):
-        """Serve ``web/<filename>``; 404 when missing, 500 when unreadable."""
+    def _serve_html_page(self, request, filename, title, cache=True):
+        """Serve ``web/<filename>``; 404 when missing, 500 when unreadable.
+
+        A served page is revalidated on every reuse (#478, ``revalidation``).
+        """
         html_path = os.path.join(_get_project_root(), "web", filename)
         try:
             if cache and html_path in self._html_cache:
@@ -536,7 +789,9 @@ class PromptManagerAPI(
                     content_type="text/html",
                     status=404,
                 )
-            return web.Response(text=content, content_type="text/html", charset="utf-8")
+            return revalidated_response(
+                request, content.encode("utf-8"), "text/html", charset="utf-8"
+            )
         except Exception:
             self.logger.exception(f"Failed to load {title}")
             return web.Response(
@@ -549,7 +804,12 @@ class PromptManagerAPI(
     # ── Shared utilities used by multiple mixins ──────────────────────
 
     def _enrich_images(self, images):
-        """Add url and thumbnail_url to a flat list of image dicts."""
+        """Add url and thumbnail_url to a flat list of image dicts.
+
+        Each dict without ``is_video`` also gets ``is_video`` and
+        ``media_type`` from its extension (#355: a linked LoraManager video
+        outside the output folders is shown as a video, not a picture).
+        """
         from urllib.parse import quote as url_quote
 
         output_dirs = self._get_all_output_dirs()
@@ -559,6 +819,11 @@ class PromptManagerAPI(
             if not image_path_str:
                 continue
 
+            if "is_video" not in image:
+                video = is_video_path(image_path_str)
+                image["is_video"] = video
+                image["media_type"] = "video" if video else "image"
+
             img_path = Path(image_path_str)
 
             # Set fallback url via image ID
@@ -575,16 +840,20 @@ class PromptManagerAPI(
                         f"{url_quote(rel_path.as_posix(), safe='/')}"
                     )
 
-                    # Check for thumbnail
-                    rel_no_ext = rel_path.with_suffix("")
-                    thumb_rel = (
-                        f"thumbnails/{rel_no_ext.as_posix()}_thumb{rel_path.suffix}"
-                    )
+                    # Any preview the file owns, the current format first (#317)
+                    from . import thumbnail_settings
                     from .prompts import thumbnail_url_for
 
-                    thumbnail_url = thumbnail_url_for(
-                        thumb_rel, output_path / thumb_rel
+                    existing = thumbnail_settings.existing_thumbnail(
+                        output_path, rel_path
                     )
+                    thumbnail_url = None
+                    if existing is not None:
+                        thumb_rel = (
+                            thumbnail_settings.thumbnail_stem_rel(rel_path)
+                            + existing.suffix
+                        )
+                        thumbnail_url = thumbnail_url_for(thumb_rel, existing)
                     if thumbnail_url:
                         image["thumbnail_url"] = thumbnail_url
                     break  # Found matching root, stop searching
@@ -600,12 +869,12 @@ class PromptManagerAPI(
         return prompts
 
     def _clean_nan_recursive(self, obj):
-        """Recursively clean NaN values from nested data structures."""
+        """Recursively replace NaN and Infinity in nested data with None."""
         if isinstance(obj, dict):
             return {key: self._clean_nan_recursive(value) for key, value in obj.items()}
         elif isinstance(obj, list):
             return [self._clean_nan_recursive(item) for item in obj]
-        elif isinstance(obj, float) and str(obj) == "nan":
+        elif isinstance(obj, float) and not math.isfinite(obj):
             return None
         else:
             return obj
@@ -728,6 +997,13 @@ class PromptManagerAPI(
         Returns:
             List[Path]: Validated directory paths, possibly empty.
         """
+        return self._output_dirs_with_origin()[0]
+
+    def _output_dirs_with_origin(self):
+        """``(dirs, auto_detected)``: the ``_get_all_output_dirs`` list, and
+        whether it comes from detection or ComfyUI's output rather than from
+        a valid configured root (False when there is no directory at all).
+        """
         from ..config import GalleryConfig
 
         configured = list(GalleryConfig.MONITORING_DIRECTORIES)
@@ -746,15 +1022,16 @@ class PromptManagerAPI(
             output_dirs.append(Path(real))
 
         if output_dirs:
-            return output_dirs
+            return output_dirs, False
 
         if configured:
             # Every configured root was rejected: never let auto-detection
             # resurrect it, serve ComfyUI's own output directory instead.
-            return self._comfyui_output_dir_list()
-
-        fallback = self._find_comfyui_output_dir()
-        return [Path(fallback)] if fallback else []
+            fallback_dirs = self._comfyui_output_dir_list()
+        else:
+            fallback = self._find_comfyui_output_dir()
+            fallback_dirs = [Path(fallback)] if fallback else []
+        return fallback_dirs, bool(fallback_dirs)
 
     def _comfyui_output_dir_list(self):
         """ComfyUI's output directory as a one-element list, or [] if unknown."""
@@ -788,278 +1065,4 @@ class PromptManagerAPI(
 
     def _parse_comfyui_prompt(self, metadata):
         """Parse ComfyUI and A1111 prompt data from metadata."""
-        result = {
-            "prompt": None,
-            "workflow": None,
-            "parameters": {},
-            "positive_prompt": None,
-            "negative_prompt": None,
-        }
-
-        # Check for A1111/Forge style parameters first.
-        # The chunk is one block: a possibly multi-line positive prompt, then a
-        # "Negative prompt:" section, then a trailing "Steps: ..." settings line.
-        # Taking lines[0] as the positive prompt truncates any multi-line prompt
-        # to one line, so split on the settings line instead.
-        if "parameters" in metadata:
-            import re as _re
-
-            params = metadata["parameters"]
-            body = params
-            settings = ""
-            _sm = _re.search(r"\nSteps:\s", body)
-            if _sm:
-                settings = body[_sm.start() + 1 :].strip()
-                body = body[: _sm.start()]
-            negative = None
-            _nm = _re.search(r"(?im)^\s*negative prompt:\s*", body)
-            if _nm:
-                negative = body[_nm.end() :].strip()
-                body = body[: _nm.start()]
-            elif body.lower().lstrip().startswith("negative prompt:"):
-                negative = body.lower().lstrip()[len("negative prompt:") :].strip()
-                body = ""
-            result["positive_prompt"] = body.strip() or None
-            if negative:
-                result["negative_prompt"] = negative
-            # Store raw parameters too
-            result["parameters"]["parameters"] = params
-            if settings:
-                for _f in ("Steps", "Sampler", "Schedule type", "CFG scale", "Seed"):
-                    _fm = _re.search(
-                        r"(?i)(?:^|,\s*)" + _re.escape(_f) + r":\s*([^,]+)", settings
-                    )
-                    if _fm:
-                        result["parameters"][_f.lower().replace(" ", "_")] = _fm.group(
-                            1
-                        ).strip()
-
-        # If no A1111 format found, proceed with ComfyUI parsing
-        if result["positive_prompt"] is None:
-            # Check for direct prompt field
-            if "prompt" in metadata:
-                try:
-                    prompt_data = json.loads(metadata["prompt"])
-                    result["prompt"] = prompt_data
-                except json.JSONDecodeError:
-                    result["prompt"] = metadata["prompt"]
-
-            # Check for workflow
-            if "workflow" in metadata:
-                try:
-                    workflow_data = json.loads(metadata["workflow"])
-                    result["workflow"] = workflow_data
-                except json.JSONDecodeError:
-                    result["workflow"] = metadata["workflow"]
-
-            # Check for other common ComfyUI fields
-            common_fields = [
-                "positive",
-                "negative",
-                "steps",
-                "cfg",
-                "sampler",
-                "scheduler",
-                "seed",
-            ]
-            for field in common_fields:
-                if field in metadata:
-                    try:
-                        result["parameters"][field] = json.loads(metadata[field])
-                    except json.JSONDecodeError:
-                        result["parameters"][field] = metadata[field]
-
-        return result
-
-    def _extract_readable_prompt(self, parsed_data):
-        """Extract human-readable prompt text from ComfyUI/A1111 data."""
-
-        def safe_to_string(value):
-            if isinstance(value, str):
-                return value
-            elif isinstance(value, list):
-                return " ".join(str(item) for item in value if item)
-            elif value is not None:
-                return str(value)
-            return None
-
-        # First check if we already extracted a positive prompt (A1111 format)
-        if parsed_data.get("positive_prompt"):
-            return parsed_data["positive_prompt"]
-
-        # Check if prompt is already a string
-        if isinstance(parsed_data.get("prompt"), str):
-            return parsed_data["prompt"]
-
-        # Check if prompt is a simple value that can be converted
-        if parsed_data.get("prompt") and not isinstance(
-            parsed_data.get("prompt"), dict
-        ):
-            return safe_to_string(parsed_data["prompt"])
-
-        prompt_data = parsed_data.get("prompt")
-        if isinstance(prompt_data, dict):
-            # Use the enhanced logic from parse-metadata.py
-            positive_prompt = self._extract_positive_prompt_from_comfyui_data(
-                prompt_data
-            )
-            if positive_prompt:
-                return positive_prompt
-
-        # Check workflow data if available
-        workflow_data = parsed_data.get("workflow")
-        if isinstance(workflow_data, dict):
-            positive_prompt = self._extract_positive_prompt_from_comfyui_data(
-                workflow_data
-            )
-            if positive_prompt:
-                return positive_prompt
-
-        # Check parameters for positive prompt
-        if parsed_data.get("parameters", {}).get("positive"):
-            return safe_to_string(parsed_data["parameters"]["positive"])
-
-        return None
-
-    def _get_node_inputs(self, node):
-        """Safely get inputs from a node, handling both dict and list formats.
-
-        Old format: inputs is a dict with direct key-value pairs
-            inputs = {"text": "my prompt", "seed": 123}
-
-        New format: inputs is a list of connection objects
-            inputs = [
-                {"name": "text", "type": "STRING", "link": null,
-                 "widget": {"name": "text"}},
-                {"name": "clip", "type": "CLIP", "link": 11}
-            ]
-        """
-        if not isinstance(node, dict):
-            return {}
-
-        inputs = node.get("inputs", {})
-
-        # If inputs is already a dict, return it
-        if isinstance(inputs, dict):
-            return inputs
-
-        # If inputs is a list, convert to dict format
-        if isinstance(inputs, list):
-            inputs_dict = {}
-            for input_item in inputs:
-                if isinstance(input_item, dict) and "name" in input_item:
-                    name = input_item["name"]
-                    inputs_dict[name] = input_item
-            return inputs_dict
-
-        return {}
-
-    def _find_text_in_node(self, node):
-        """Try to find text content in a node using various strategies.
-
-        Handles both old and new workflow formats.
-        """
-        if not isinstance(node, dict):
-            return None
-
-        # Strategy 1: Check normalized inputs for 'text' field
-        inputs = self._get_node_inputs(node)
-        if "text" in inputs and isinstance(inputs["text"], str):
-            return inputs["text"]
-
-        # Strategy 2: For text encoder nodes, check widgets_values
-        class_type = node.get("class_type", node.get("type", ""))
-        text_encoder_types = [
-            "CLIPTextEncode",
-            "CLIPTextEncodeSDXL",
-            "CLIPTextEncodeSDXLRefiner",
-            "CLIPTextEncodeFlux",
-            "PromptManager",
-            "PromptManagerText",
-            "BNK_CLIPTextEncoder",
-            "Text Encoder",
-            "CLIP Text Encode",
-        ]
-
-        if any(
-            encoder_type.lower() in class_type.lower()
-            for encoder_type in text_encoder_types
-        ):
-            widgets_values = node.get("widgets_values", [])
-            if widgets_values and len(widgets_values) > 0:
-                if isinstance(widgets_values[0], str) and widgets_values[0].strip():
-                    return widgets_values[0]
-
-        return None
-
-    def _extract_positive_prompt_from_comfyui_data(self, data):
-        """Extract positive prompt from ComfyUI data.
-
-        Handles both old and new formats.
-        """
-        if not isinstance(data, dict):
-            return None
-
-        # Build nodes dictionary
-        nodes_by_id = {}
-        if "nodes" in data:
-            # Handle nodes array format
-            for node in data["nodes"]:
-                if isinstance(node, dict):
-                    nid = node.get("id")
-                    if nid is not None:
-                        nodes_by_id[nid] = node
-        else:
-            # Handle flat dictionary format (node_id -> node_data)
-            for nid_str, node in data.items():
-                try:
-                    nid = int(nid_str)
-                except (ValueError, TypeError):
-                    nid = nid_str
-                if isinstance(node, dict):
-                    if "id" in node:
-                        nid = node["id"]
-                    nodes_by_id[nid] = node
-
-        if not nodes_by_id:
-            return None
-
-        # First, try to find positive/negative connection pattern
-        pos_id = None
-        for node in nodes_by_id.values():
-            if isinstance(node, dict):
-                inputs = self._get_node_inputs(node)
-                if "positive" in inputs and "negative" in inputs:
-                    try:
-                        pos_input = inputs["positive"]
-                        if isinstance(pos_input, list) and len(pos_input) > 0:
-                            pos_id = int(pos_input[0])
-                            break
-                    except (ValueError, TypeError, IndexError):
-                        continue
-
-        # Get text from the positive node
-        if pos_id is not None and pos_id in nodes_by_id:
-            text_val = self._find_text_in_node(nodes_by_id[pos_id])
-            if text_val:
-                return text_val
-
-        # Fallback: find any text encoder node with text content
-        text_nodes = []
-        for node in nodes_by_id.values():
-            if isinstance(node, dict):
-                text_val = self._find_text_in_node(node)
-                if text_val:
-                    # Try to determine if this is positive or negative
-                    node_title = node.get("title", "").lower()
-                    if "neg" not in node_title and "negative" not in node_title:
-                        # Prioritize non-negative prompts
-                        text_nodes.insert(0, text_val)
-                    else:
-                        text_nodes.append(text_val)
-
-        # Return the first positive-looking prompt
-        if text_nodes:
-            return text_nodes[0]
-
-        return None
+        return parse_comfyui_prompt(metadata)

py/api/annotation_suggest.py

@@ -10,12 +10,15 @@ filesystem or the database.
 
 import math
 
-WD14_MODELS = frozenset({"wd14-swinv2", "wd14-vit"})
-LLM_MODELS = frozenset({"gguf", "hf"})
-MODEL_TYPES = WD14_MODELS | LLM_MODELS
-
-MAX_INSTRUCTION_CHARS = 4000
-MAX_INSTRUCTION_BYTES = 16 * 1024
+from ..tagging_defaults import (  # noqa: F401 - re-exported for the route
+    LLM_MODEL_TYPES as LLM_MODELS,
+    MAX_INSTRUCTION_BYTES,
+    MAX_INSTRUCTION_CHARS,
+    MODEL_TYPES,
+    WD14_MODEL_TYPES as WD14_MODELS,
+    validate_template,
+    validate_threshold,
+)
 
 SUGGEST_FIELDS = frozenset(
     {"target", "capture_token", "expected_epoch", "model_type", "options"}
@@ -24,7 +27,7 @@ _COMMON = frozenset({"suggestion_limit", "use_gpu"})
 _WD14_REQUIRED = frozenset({"general_threshold", "character_threshold"})
 _LLM_REQUIRED = frozenset({"instructions"})
 
-UNAVAILABLE_HINT = "Download it from the legacy Auto-tag page."
+UNAVAILABLE_HINT = "Download it in System › Auto-tagging."
 
 
 class OptionError(ValueError):
@@ -32,27 +35,17 @@ class OptionError(ValueError):
 
 
 def _threshold(name: str, value) -> float:
-    if type(value) not in (int, float) or not math.isfinite(value):
-        raise OptionError(f"{name} must be a finite number")
-    if not 0 <= value <= 1:
-        raise OptionError(f"{name} must be between 0 and 1")
-    return float(value)
+    try:
+        return validate_threshold(name, value)
+    except ValueError as error:
+        raise OptionError(str(error)) from None
 
 
 def _instructions(value) -> str:
-    if not isinstance(value, str) or not value.strip():
-        raise OptionError("instructions must be non-empty text")
-    if len(value) > MAX_INSTRUCTION_CHARS:
-        raise OptionError(
-            f"instructions must be at most {MAX_INSTRUCTION_CHARS} characters"
-        )
     try:
-        encoded = value.encode("utf-8")
-    except UnicodeEncodeError:
-        raise OptionError("instructions must be valid Unicode text") from None
-    if len(encoded) > MAX_INSTRUCTION_BYTES:
-        raise OptionError("instructions are too long")
-    return value
+        return validate_template(value)
+    except ValueError as error:
+        raise OptionError(str(error)) from None
 
 
 def validate_options(model_type, options, *, default_limit: int, max_limit: int):
@@ -93,6 +86,23 @@ def unavailable_message(model_name: str) -> str:
     return f"{model_name} is not downloaded. {UNAVAILABLE_HINT}"
 
 
+def model_refusal(status: dict, model_type: str, model_name: str):
+    """Why *model_type* cannot run, from ``get_models_status()``, else None.
+
+    Not downloaded first (``unavailable_message``), then a backend package
+    that cannot be imported (#424). Both are a 409 ``model-unavailable``.
+    """
+    entry = status.get(model_type) or {}
+    if not entry.get("downloaded"):
+        return unavailable_message(model_name)
+    missing = entry.get("missing_packages") or []
+    if missing:
+        from ..autotag import missing_packages_message
+
+        return missing_packages_message(model_type, [str(name) for name in missing])
+    return None
+
+
 def run_suggestions(service, capture_path: str, model_type: str, options: dict):
     """Blocking: load (or swap to) *model_type*, then suggest for the capture.
 

py/api/annotations.py

@@ -13,6 +13,7 @@ import datetime
 import functools
 import os
 import re
+import threading
 from pathlib import Path
 
 from aiohttp import web
@@ -20,13 +21,18 @@ from aiohttp import web
 from .annotation_suggest import (
     SUGGEST_FIELDS,
     OptionError,
+    model_refusal,
     run_suggestions,
     suggestion_rows,
-    unavailable_message,
     validate_options,
 )
 from .autotag_routes import model_busy_response
-from .images import _forbidden, _has_traversal, _unavailable_link, _verified_title
+
+# The target check is shared with the collection routes (#331); both names
+# stay importable from here.
+from .media_target import MAX_TARGET_PATH  # noqa: F401
+from .media_target import media_target as _target
+from .images import _unavailable_link, _verified_title
 from .prompts import bad_request
 from .. import model_gate
 
@@ -39,7 +45,7 @@ try:
     )
     from ...database.execution_history import subscribe_history_restore
     from ...database.library_commands import LibraryCommandError
-    from ...utils.media_capture import CAPTURE_EXTENSIONS, CaptureError, CaptureStore
+    from ...utils.media_capture import CaptureError, CaptureStore
 except ImportError:
     from database.annotations import (
         MAX_IMAGE_TAGS,
@@ -49,15 +55,13 @@ except ImportError:
     )
     from database.execution_history import subscribe_history_restore
     from database.library_commands import LibraryCommandError
-    from utils.media_capture import CAPTURE_EXTENSIONS, CaptureError, CaptureStore
+    from utils.media_capture import CaptureError, CaptureStore
 
 # Request bodies of the annotation routes are small (a target plus a tag set).
 MAX_BODY_BYTES = 64 * 1024
-MAX_TARGET_PATH = 1024
 CAPTURE_URL = "/prompt_manager/image-annotations/capture/"
 _PREVIEW_CHUNK = 64 * 1024
 
-_TARGET_FIELDS = frozenset({"root_index", "root_scope", "path"})
 _OBSERVE_FIELDS = frozenset({"target"})
 _SAVE_FIELDS = frozenset(
     {
@@ -73,6 +77,9 @@ _HEX64 = re.compile(r"[0-9a-f]{64}")
 _UUID = re.compile(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}")
 # secrets.token_urlsafe(32) is 43 characters; the bound only rejects junk.
 _TOKEN = re.compile(r"[A-Za-z0-9_-]{1,128}")
+# The store is created on first use by the event loop, the scan's executor or
+# the monitor's worker (Needs review), so its creation is serialized.
+_CAPTURE_STORE_LOCK = threading.Lock()
 
 _PREVIEW_TYPES = {
     ".png": "image/png",
@@ -184,37 +191,6 @@ def _capture_error(exc: CaptureError) -> web.Response:
     return _coded(exc.code, _MESSAGES.get(exc.code) or str(exc))
 
 
-def _target(data: object):
-    """``((index, scope, relative), None)`` for a target, else ``(None, error)``.
-
-    Only the listing identity is accepted: an int root index, the root's
-    64-hex scope and a bounded root-relative POSIX media path, never an
-    absolute, drive-qualified, backslash or ``..`` path. Nothing here touches
-    the filesystem or the database.
-    """
-    if not isinstance(data, dict) or set(data) != _TARGET_FIELDS:
-        return None, bad_request("target must name root_index, root_scope and path")
-    index, scope, relative = data["root_index"], data["root_scope"], data["path"]
-    if type(index) is not int or index < 0:
-        return None, bad_request("Invalid gallery root")
-    if not isinstance(scope, str) or _HEX64.fullmatch(scope) is None:
-        return None, bad_request("Invalid gallery root scope")
-    if not isinstance(relative, str) or not relative or len(relative) > MAX_TARGET_PATH:
-        return None, bad_request("path must be a relative media path")
-    parts = relative.split("/")
-    if (
-        "\0" in relative
-        or "\\" in relative
-        or relative.startswith("/")
-        or _has_traversal(relative)
-        or any(part in ("", ".", "..") or ":" in part for part in parts)
-    ):
-        return None, _forbidden()
-    if os.path.splitext(parts[-1])[1].lower() not in CAPTURE_EXTENSIONS:
-        return None, _forbidden("Only media files can be tagged")
-    return (index, scope, relative), None
-
-
 def _expires_at(seconds: float | None) -> str:
     moment = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(
         seconds=max(0.0, seconds or 0.0)
@@ -259,31 +235,47 @@ class AnnotationRoutesMixin:
         Created lazily so an API object that never observes a file (most tests,
         a server that never opens the Tag dialog) owns no temp directory. A
         restore of the database dooms every capture (``invalidate_all``).
+        Thread-safe: Needs review recording calls it off the event loop.
+        Raises ``CaptureError`` after ``close_annotation_captures``.
         """
-        if self._captures is None:
-            self._captures = CaptureStore()
-        if self._captures_unsubscribe is None:
-            self._captures_unsubscribe = subscribe_history_restore(
-                self.db.model.db_path, self._invalidate_captures
-            )
-        return self._captures
+        with _CAPTURE_STORE_LOCK:
+            if getattr(self, "_captures_closed", False):
+                # A recording still in flight at shutdown must not create a
+                # new temp directory after the cleanup removed the old one.
+                raise CaptureError("The capture store is closed")
+            if self._captures is None:
+                self._captures = CaptureStore()
+            if self._captures_unsubscribe is None:
+                self._captures_unsubscribe = subscribe_history_restore(
+                    self.db.model.db_path, self._invalidate_captures
+                )
+            return self._captures
 
     def _invalidate_captures(self) -> None:
         store = self._captures
         if store is not None:
             store.invalidate_all()
 
-    def close_annotation_captures(self) -> None:
-        """Remove every capture file and the owned directory. Idempotent."""
-        unsubscribe, self._captures_unsubscribe = self._captures_unsubscribe, None
+    def close_annotation_captures(self, *, final: bool = False) -> None:
+        """Remove every capture file and the owned directory. Idempotent.
+
+        Taken under the store lock. ``final`` (the shutdown cleanup) also makes
+        ``_capture_store`` refuse to create another store, so a recording
+        still in flight cannot leak a new temp directory after it.
+        """
+        with _CAPTURE_STORE_LOCK:
+            if final:
+                self._captures_closed = True
+            unsubscribe, self._captures_unsubscribe = self._captures_unsubscribe, None
+            store, self._captures = self._captures, None
+            self._suggested_tokens = set()
         if unsubscribe is not None:
             unsubscribe()
-        store, self._captures = self._captures, None
         if store is not None:
             store.close()
 
     async def cleanup_annotation_captures(self, app: web.Application) -> None:
-        self.close_annotation_captures()
+        self.close_annotation_captures(final=True)
 
     # -- shared request steps --------------------------------------------------
 
@@ -329,7 +321,10 @@ class AnnotationRoutesMixin:
         root, error = self._scoped_root(index, scope)
         if error is not None:
             return error
-        store = self._capture_store()
+        try:
+            store = self._capture_store()
+        except CaptureError as exc:  # the server is shutting down
+            return _capture_error(exc)
         try:
             epoch = await self._run_in_executor(self.db.get_execution_epoch)
             capture = await _settled(
@@ -493,6 +488,7 @@ class AnnotationRoutesMixin:
         store = self._captures
         if store is not None and _TOKEN.fullmatch(token) is not None:
             store.release(token)
+        self._suggested_tokens.discard(token)
         return web.json_response(
             {"success": True}, headers={"Cache-Control": "no-store"}
         )
@@ -542,6 +538,9 @@ class AnnotationRoutesMixin:
             return _coded("capture-expired", _OTHER_FILE)
         if facts.sha256 != capture.sha256:
             return _coded("file-changed")
+        # A save after Generate suggestions ran on this capture marks the
+        # image auto-tagged (#379), so a Skip batch leaves the review alone.
+        suggested = token in self._suggested_tokens
         try:
             receipt = await self._run_in_executor(
                 self.db.replace_image_tags,
@@ -552,6 +551,7 @@ class AnnotationRoutesMixin:
                 expected_epoch=expected_epoch,
                 expected_tag_revision=revision,
                 tags=tags,
+                mark_autotagged=suggested,
             )
         except AnnotationCommandError as exc:
             current = exc.current or {}
@@ -568,10 +568,23 @@ class AnnotationRoutesMixin:
         except Exception:
             self.logger.exception("Image tag save failed")
             return _error(500, "save-failed", "The tags could not be saved.")
+        # Dropped only after a saved or replayed receipt: a retry after a
+        # tag-conflict still marks.
+        self._suggested_tokens.discard(token)
         return web.json_response(
             {"success": True, **receipt}, headers={"Cache-Control": "no-store"}
         )
 
+    def _remember_suggested(self, store, token: str) -> None:
+        """Note that Generate ran on *token*; forget tokens whose capture is
+        gone. ``expires_in`` does not reset a capture's idle clock."""
+        self._suggested_tokens = {
+            kept
+            for kept in self._suggested_tokens
+            if store.expires_in(kept) is not None
+        }
+        self._suggested_tokens.add(token)
+
     # -- suggest ---------------------------------------------------------------
 
     async def suggest_image_tags(self, request: web.Request) -> web.Response:
@@ -582,7 +595,7 @@ class AnnotationRoutesMixin:
         it, including a legacy batch), never downloads (``model-unavailable``)
         and never writes.
         """
-        from ..autotag import MODELS, get_autotag_service
+        from ..autotag import MODELS, MissingBackendPackages, get_autotag_service
         from ..config import MAX_SUGGESTION_LIMIT, AutotagConfig
 
         data, error = await self._annotation_body(request, SUGGEST_FIELDS)
@@ -620,9 +633,10 @@ class AnnotationRoutesMixin:
                 return error
             service = get_autotag_service()
             status = await self._run_in_executor(service.get_models_status)
-            if not status.get(model_type, {}).get("downloaded"):
-                name = MODELS.get(model_type, {}).get("name", model_type)
-                return _coded("model-unavailable", unavailable_message(name))
+            name = MODELS.get(model_type, {}).get("name", model_type)
+            refusal = model_refusal(status, model_type, name)
+            if refusal is not None:
+                return _coded("model-unavailable", refusal)
             owner = model_gate.new_owner("image-annotations")
             if not model_gate.try_acquire(owner):
                 return model_busy_response()
@@ -641,6 +655,9 @@ class AnnotationRoutesMixin:
                     options,
                     on_skipped=functools.partial(store.release_use, token),
                 )
+            except MissingBackendPackages as exc:
+                # A package gone since the status read (load_model logged it).
+                return _coded("model-unavailable", str(exc))
             except Exception:
                 self.logger.exception("Tag suggestion failed")
                 return _error(
@@ -653,6 +670,7 @@ class AnnotationRoutesMixin:
         finally:
             if not started:
                 store.release_use(token)
+        self._remember_suggested(store, token)
         return web.json_response(
             {
                 "success": True,

py/api/collections.py

@@ -0,0 +1,350 @@
+"""Collections (#331): named, hand-picked groups of images.
+
+Routes under ``/prompt_manager/collections``. A file is named by its listing
+identity ``{root_index, root_scope, path}`` (``media_target.py``) and stored
+under the favorites' keys (``_favorite_root_key``, ``_favorite_relative_key``),
+which never leave the server. A stale scope (the sources were reordered) is a
+409 ``source-changed`` before anything is written. Responses are
+``{"success": bool, ...}``; an unexpected failure is a logged 500 with a
+generic message, never the exception text.
+"""
+
+import hmac
+import re
+from functools import wraps
+from typing import Callable
+
+from aiohttp import web
+
+from .images import (
+    _favorite_relative_key,
+    _favorite_root_key,
+    _forbidden,
+    _json_body_or_error,
+    collection_not_found,
+)
+from .media_target import media_target
+from .prompts import bad_request
+
+try:
+    from ...database.image_collections import (
+        CollectionFilesMissing,
+        CollectionInvalid,
+        CollectionMissing,
+        CollectionNameTaken,
+        normalize_collection_name,
+    )
+    from ...database.image_favorites import MAX_KEY_LENGTH
+except ImportError:
+    from database.image_collections import (
+        CollectionFilesMissing,
+        CollectionInvalid,
+        CollectionMissing,
+        CollectionNameTaken,
+        normalize_collection_name,
+    )
+    from database.image_favorites import MAX_KEY_LENGTH
+
+BASE = "/prompt_manager/collections"
+# The most files one add or remove names (the bulk tagging cap, MC12).
+MAX_ITEMS = 500
+MAX_ID = 999_999_999
+_PATH_ID = re.compile(r"[1-9][0-9]{0,8}")
+_ROOT_INDEX = re.compile(r"[0-9]{1,9}")
+_ADD_TARGETS = (frozenset({"collection_id", "items"}), frozenset({"name", "items"}))
+_REMOVE_FIELDS = frozenset({"collection_id", "items"})
+_RENAME_FIELDS = frozenset({"name"})
+_MEDIA_MESSAGE = "Only media files can be added to a collection"
+READ_FAILED = "Collections could not be read."
+CHANGE_FAILED = "The collection could not be changed."
+SOURCES_CHANGED = "The sources changed. Refresh the images and try again."
+FILES_NOT_FOUND = (
+    "None of the selected images could be found. Refresh the images and try again."
+)
+NAME_TAKEN = "Another collection already has this name."
+
+
+class _Refused(Exception):
+    """A request refused before any write; carries its response."""
+
+    def __init__(self, response: web.Response):
+        super().__init__(response.status)
+        self.response = response
+
+
+def _coded(status: int, code: str, message: str, **extra) -> web.Response:
+    return web.json_response(
+        {"success": False, "code": code, "error": message, **extra},
+        status=status,
+        headers={"Cache-Control": "no-store"},
+    )
+
+
+def _collection_response(failure: str) -> Callable:
+    """Map the database's refusals to their status; anything else to a 500."""
+
+    def decorate(handler: Callable) -> Callable:
+        @wraps(handler)
+        async def wrapped(self, request: web.Request) -> web.Response:
+            try:
+                return await handler(self, request)
+            except _Refused as refused:
+                return refused.response
+            except CollectionInvalid as exc:
+                return bad_request(str(exc))
+            except CollectionMissing:
+                return collection_not_found()
+            except CollectionFilesMissing:
+                return _coded(404, "files-not-found", FILES_NOT_FOUND)
+            except CollectionNameTaken as exc:
+                return _coded(
+                    409, "name-taken", NAME_TAKEN, collection_id=exc.collection_id
+                )
+            except Exception:
+                self.logger.exception("Collection request failed")
+                return web.json_response(
+                    {"success": False, "error": failure}, status=500
+                )
+
+        return wrapped
+
+    return decorate
+
+
+def _path_id(request: web.Request) -> int:
+    raw = request.match_info["collection_id"]
+    if not raw.isascii() or _PATH_ID.fullmatch(raw) is None:
+        raise _Refused(bad_request("Invalid collection id"))
+    return int(raw)
+
+
+def _body_id(value: object) -> int:
+    if type(value) is not int or not 1 <= value <= MAX_ID:
+        raise _Refused(bad_request("Invalid collection id"))
+    return value
+
+
+async def _body(request: web.Request, allowed: tuple) -> dict:
+    data, error = await _json_body_or_error(request)
+    if error is not None:
+        raise _Refused(error)
+    if frozenset(data) not in allowed:
+        names = " or ".join(", ".join(sorted(fields)) for fields in allowed)
+        raise _Refused(bad_request(f"Request must contain exactly {names}"))
+    return data
+
+
+class CollectionRoutesMixin:
+    """Mixin providing the collection routes (#331)."""
+
+    def _register_collection_routes(self, routes: web.RouteTableDef) -> None:
+        # The fixed paths go first: "/items" must never match "/{collection_id}".
+        routes.get(BASE)(self.list_collections)
+        routes.post(BASE + "/items")(self.add_collection_items)
+        routes.delete(BASE + "/items")(self.remove_collection_items)
+        routes.get(BASE + "/membership")(self.collection_membership)
+        routes.patch(BASE + "/{collection_id}")(self.rename_collection)
+        routes.delete(BASE + "/{collection_id}")(self.delete_collection)
+
+    # -- reads -------------------------------------------------------------
+
+    @_collection_response(READ_FAILED)
+    async def list_collections(self, request: web.Request) -> web.Response:
+        """Every collection with its listed ``count`` and its ``missing`` rows.
+
+        ``count`` is the members listed under every configured source (one
+        primary-key range read per source, matched against its cached
+        listing); ``missing`` the stored rows beyond it. When a scan or a read
+        fails both are null for every collection and the names still return.
+        """
+        collections = await self._run_in_executor(self.db.list_collections)
+        counts = None
+        if collections:
+            try:
+                counts = await self._collection_counts()
+            except Exception:
+                self.logger.warning(
+                    "Collection counts could not be read", exc_info=True
+                )
+        rows = []
+        for collection in collections:
+            listed, stored = (None, None) if counts is None else counts
+            count = None if listed is None else listed.get(collection["id"], 0)
+            stored_rows = None if stored is None else stored.get(collection["id"], 0)
+            rows.append(
+                {
+                    **collection,
+                    "count": count,
+                    # Clamped: never below zero, whatever the snapshot saw.
+                    "missing": None if count is None else max(0, stored_rows - count),
+                }
+            )
+        return web.json_response({"success": True, "collections": rows})
+
+    async def _collection_counts(self) -> tuple[dict, dict]:
+        """``({id: listed members}, {id: stored rows})`` over every source.
+
+        The scans come from the gallery cache; the root keys (realpath), the
+        listings and one read snapshot of every member and stored count run
+        in the executor, so a write between two reads cannot skew them.
+        """
+        roots = list(self._get_all_output_dirs())
+        root_keys = await self._run_in_executor(
+            lambda: [_favorite_root_key(root) for root in roots]
+        )
+        distinct = {}  # one folder configured twice counts once
+        for root, root_key in zip(roots, root_keys):
+            distinct.setdefault(root_key, root)
+        scans = {
+            root_key: await self._get_gallery_files(root)
+            for root_key, root in distinct.items()
+        }
+        return await self._run_in_executor(self._count_members, distinct, scans)
+
+    def _count_members(self, roots: dict, scans: dict) -> tuple[dict, dict]:
+        """Blocking: match one snapshot's members against each listing."""
+        members, stored = self.db.collection_count_snapshot(list(roots))
+        listed: dict[int, int] = {}
+        for root_key, root in roots.items():
+            listing = self._favorite_listing(root, scans[root_key])
+            for collection_id, keys in members.get(root_key, {}).items():
+                found = sum(1 for key in keys if key in listing)
+                listed[collection_id] = listed.get(collection_id, 0) + found
+        return listed, stored
+
+    @_collection_response(READ_FAILED)
+    async def collection_membership(self, request: web.Request) -> web.Response:
+        """GET ``?root_index&root_scope&path``: the ids holding that file."""
+        query = request.query
+        if len(query) != 3 or set(query) != {"root_index", "root_scope", "path"}:
+            # len() counts a repeated key (?path=a&path=b) twice.
+            return bad_request("Query must name root_index, root_scope and path once")
+        raw_index = query["root_index"]
+        if not raw_index.isascii() or _ROOT_INDEX.fullmatch(raw_index) is None:
+            return bad_request("Invalid gallery root")
+        roots = list(self._get_all_output_dirs())
+        target = {
+            "root_index": int(raw_index),
+            "root_scope": query["root_scope"],
+            "path": query["path"],
+        }
+        root_key, key = await self._run_in_executor(
+            self._collection_key, target, roots, {}
+        )
+        ids = await self._run_in_executor(self.db.file_collections, root_key, key)
+        return web.json_response({"success": True, "collection_ids": ids})
+
+    # -- writes ------------------------------------------------------------
+
+    @_collection_response(CHANGE_FAILED)
+    async def add_collection_items(self, request: web.Request) -> web.Response:
+        """POST ``{collection_id | name, items}``: add the listed files.
+
+        Only files the source's listing holds are added; the rest count in
+        ``not_found``. A new name with nothing to add creates nothing (404
+        ``files-not-found``).
+        """
+        data = await _body(request, _ADD_TARGETS)
+        target = {}
+        if "name" in data:
+            target["name"] = normalize_collection_name(data["name"])
+        else:
+            target["collection_id"] = _body_id(data["collection_id"])
+        roots = list(self._get_all_output_dirs())
+        keys = await self._run_in_executor(self._collection_items, data["items"], roots)
+        found, not_found = [], 0
+        listings = {}
+        for index, root_key, key in keys:
+            if index not in listings:
+                root = roots[index]
+                files = await self._get_gallery_files(root)
+                listings[index] = await self._run_in_executor(
+                    self._favorite_listing, root, files
+                )
+            if key in listings[index]:
+                found.append((root_key, key))
+            else:
+                not_found += 1
+        result = await self._run_in_executor(
+            lambda: self.db.add_to_collection(items=found, **target)
+        )
+        return web.json_response({"success": True, **result, "not_found": not_found})
+
+    @_collection_response(CHANGE_FAILED)
+    async def remove_collection_items(self, request: web.Request) -> web.Response:
+        """DELETE ``{collection_id, items}``: listed or not, every named file."""
+        data = await _body(request, (_REMOVE_FIELDS,))
+        collection_id = _body_id(data["collection_id"])
+        roots = list(self._get_all_output_dirs())
+        keys = await self._run_in_executor(self._collection_items, data["items"], roots)
+        result = await self._run_in_executor(
+            self.db.remove_from_collection,
+            collection_id,
+            [(root_key, key) for _index, root_key, key in keys],
+        )
+        return web.json_response({"success": True, **result})
+
+    @_collection_response(CHANGE_FAILED)
+    async def rename_collection(self, request: web.Request) -> web.Response:
+        """PATCH ``{name}``: a case variant of its own name is allowed."""
+        collection_id = _path_id(request)
+        data = await _body(request, (_RENAME_FIELDS,))
+        name = normalize_collection_name(data["name"])
+        collection = await self._run_in_executor(
+            self.db.rename_collection, collection_id, name
+        )
+        return web.json_response({"success": True, "collection": collection})
+
+    @_collection_response(CHANGE_FAILED)
+    async def delete_collection(self, request: web.Request) -> web.Response:
+        """DELETE: the collection and its members; files are never touched."""
+        collection_id = _path_id(request)
+        await self._run_in_executor(self.db.delete_collection, collection_id)
+        return web.json_response({"success": True, "deleted": True})
+
+    # -- request steps -----------------------------------------------------
+
+    def _collection_items(self, items: object, roots: list) -> list[tuple]:
+        """``(index, root key, relative key)`` per distinct file, in order.
+
+        Blocking (realpath per root, once per request): run in the executor.
+        Every item is checked before anything is written: 400 for a malformed
+        shape, 403 for an unsafe path, 409 for a stale scope. Two spellings
+        that fold to one key (Windows) count once.
+        """
+        if not isinstance(items, list) or not 1 <= len(items) <= MAX_ITEMS:
+            raise _Refused(bad_request(f"items must list 1 to {MAX_ITEMS} images"))
+        keys, sources = {}, {}
+        for item in items:
+            index = item.get("root_index") if isinstance(item, dict) else None
+            root_key, key = self._collection_key(item, roots, sources)
+            keys.setdefault((root_key, key), index)
+        return [(index, root_key, key) for (root_key, key), index in keys.items()]
+
+    def _collection_key(
+        self, item: object, roots: list, sources: dict
+    ) -> tuple[str, str]:
+        """The stored keys of one target, or raise its refusal (blocking).
+
+        *sources* caches ``{index: (scope, root key)}`` for one request, so
+        each root's realpath runs once however many items name it.
+        """
+        target, error = media_target(item, media_message=_MEDIA_MESSAGE)
+        if error is not None:
+            raise _Refused(error)
+        index, scope, relative = target
+        if index >= len(roots):
+            raise _Refused(_coded(409, "source-changed", SOURCES_CHANGED))
+        if index not in sources:
+            sources[index] = (
+                self._gallery_root_scope(roots[index], index),
+                _favorite_root_key(roots[index]),
+            )
+        current, root_key = sources[index]
+        if not hmac.compare_digest(scope, current):
+            raise _Refused(_coded(409, "source-changed", SOURCES_CHANGED))
+        key = _favorite_relative_key(relative)
+        if max(len(root_key), len(key)) > MAX_KEY_LENGTH:
+            # A stored key past the schema bound would fail the next restore.
+            raise _Refused(_forbidden("Path is too long to add to a collection"))
+        return root_key, key

py/api/diagnostics_report.py

@@ -0,0 +1,291 @@
+"""The shareable diagnostic report (#329 System S12).
+
+``GET /prompt_manager/system/diagnostics`` answers a report built on the
+server from an explicit list of fields: the project and Viewer.js versions,
+Python and the platform, the four checks of the legacy ``GET /diagnostics``
+(one helper, ``admin.legacy_checks``), the database's public info, source
+counts, the log configuration and statistics, the last health summary and
+the configuration keys that only ``config.json`` holds. It never copies
+``PromptManagerConfig.get_config()``: no path leaf is listed, and the CivitAI
+key is reported only as ``has_civitai_api_key``.
+
+Every path in the report goes through ``report_path``: relative to the
+closest public anchor that is not a filesystem root, else ``"outside
+ComfyUI"``, never a basename that could be a user name. As a backstop,
+``scrub`` replaces any string that still looks like an absolute path, holds
+the saved key or the home folder (with or without its root), and counts each
+replacement in ``redacted`` (0 for a correct builder).
+
+The report opens the database, so it takes a registry lease: it is refused
+(409 ``busy``) while an ``exclusive`` job (a restore) runs, and a restore
+waits for it. It is built off the event loop.
+"""
+
+import datetime
+import json
+import ntpath
+import os
+import platform
+import posixpath
+import re
+from pathlib import Path, PurePosixPath, PureWindowsPath
+
+from aiohttp import web
+
+from . import admin as _admin
+from .job_routes import busy_response
+from .jobs import JobBusy
+from .system import health_summary
+
+try:
+    from ...utils.version import project_version
+except ImportError:  # loaded as the top-level ``py`` package (tests)
+    from utils.version import project_version
+
+DIAGNOSTICS_LEASE = "Diagnostics"
+_ROOT = Path(__file__).resolve().parents[2]
+VIEWERJS_PROVENANCE = str(_ROOT / "web" / "lib" / "viewerjs" / "provenance.json")
+OUTSIDE = "outside ComfyUI"
+REMOVED = "[removed]"
+# The CivitAI key is searched for only when it is long enough to be a key.
+MIN_KEY_LENGTH = 8
+_WINDOWS_ABSOLUTE = re.compile(r"^[A-Za-z]:[\\/]")
+
+
+def anchored_path(canonical, anchors, home, pathmod=os.path):
+    """``canonical`` relative to the closest usable anchor, else ``outside ComfyUI``.
+
+    All three are already canonical (``normcase``) in ``pathmod``'s spelling.
+    A filesystem root is never usable, nor is a strict ancestor of ``home``
+    (``/home``, ``C:\\Users``), whose relative paths would start with a user
+    name; so a path inside the home folder needs an anchor at or below it.
+    """
+    flavor = PureWindowsPath if pathmod is ntpath else PurePosixPath
+    target = flavor(canonical)
+    home_path = flavor(home) if home else None
+    usable = []
+    for anchor in map(flavor, anchors):
+        if anchor.parent == anchor:
+            continue
+        if home_path is not None and home_path != anchor:
+            if home_path.is_relative_to(anchor):
+                continue
+        if target.is_relative_to(anchor):
+            usable.append(anchor)
+    if not usable:
+        return OUTSIDE
+    anchor = max(usable, key=lambda a: len(a.parts))
+    return target.relative_to(anchor).as_posix() or "."
+
+
+def _canonical(path):
+    return os.path.normcase(os.path.realpath(str(path)))
+
+
+def report_path(path):
+    """``path`` relative to a public anchor (POSIX style), else ``outside ComfyUI``."""
+    if not path:
+        return ""
+    from ..config import GalleryConfig
+
+    try:
+        canonical = _canonical(path)
+        anchors = [_canonical(anchor) for anchor in GalleryConfig.path_anchors()]
+        home = os.path.expanduser("~")
+        home = _canonical(home) if home and home != "~" else None
+    except (OSError, ValueError):
+        return OUTSIDE
+    return anchored_path(canonical, anchors, home)
+
+
+def _looks_absolute(text):
+    return (
+        posixpath.isabs(text)
+        or ntpath.isabs(text)
+        or _WINDOWS_ABSOLUTE.match(text) is not None
+        or text.startswith("\\\\")
+    )
+
+
+# The home folder is matched only as whole path parts, never inside a word.
+_PART = r"(?<![\w.-]){}(?![\w.-])"
+MIN_HOME_NAME = 3
+
+
+def _home_patterns():
+    """``(pattern, paths_only)`` for the home folder: in full, rootless, its name.
+
+    Accepted residual risk: only this process's own home folder is known, so
+    another user's home (an output folder under /home/<them> while ComfyUI
+    runs as a service account) can still be named by a report path.
+    """
+    home = os.path.expanduser("~")
+    if not home or home == "~" or len(home) < 4 or home in ("/", "\\"):
+        return ()
+    spellings = {home, home.replace("\\", "/")}
+    relative = os.path.splitdrive(home)[1].strip("\\/")
+    for spelling in (relative, relative.replace("\\", "/")):
+        if "/" in spelling or "\\" in spelling:
+            spellings.add(spelling)
+    patterns = [
+        (re.compile(_PART.format(re.escape(spelling)), re.IGNORECASE), False)
+        for spelling in spellings
+    ]
+    name = re.split(r"[\\/]", relative)[-1] if relative else ""
+    if len(name) >= MIN_HOME_NAME:
+        # The bare name only inside a path ("info/x"), not a word ("INFO").
+        patterns.append(
+            (re.compile(_PART.format(re.escape(name)), re.IGNORECASE), True)
+        )
+    return tuple(patterns)
+
+
+def scrub(report, key):
+    """``report`` with leaked paths and the key replaced, counted in ``redacted``."""
+    key = key if isinstance(key, str) and len(key) >= MIN_KEY_LENGTH else None
+    homes = _home_patterns()
+    count = 0
+
+    def clean(value):
+        nonlocal count
+        if isinstance(value, str):
+            if (
+                _looks_absolute(value)
+                or (key and key in value)
+                or any(
+                    pattern.search(value)
+                    for pattern, paths_only in homes
+                    if not paths_only or "/" in value or "\\" in value
+                )
+            ):
+                count += 1
+                return REMOVED
+            return value
+        if isinstance(value, dict):
+            return {name: clean(item) for name, item in value.items()}
+        if isinstance(value, (list, tuple)):
+            return [clean(item) for item in value]
+        return value
+
+    cleaned = clean(report)
+    cleaned["redacted"] = int(cleaned.get("redacted") or 0) + count
+    return cleaned
+
+
+def viewerjs_version(path=None):
+    """The pinned Viewer.js version from its provenance file, or ``unknown``."""
+    try:
+        with open(path or VIEWERJS_PROVENANCE, encoding="utf-8") as handle:
+            version = json.load(handle).get("version")
+    except (OSError, ValueError, AttributeError):
+        return "unknown"
+    return version if isinstance(version, str) and version else "unknown"
+
+
+def public_config():
+    """The configuration keys only ``config.json`` holds, from an explicit list."""
+    from ..config import GalleryConfig, IntegrationConfig, PromptManagerConfig
+
+    return {
+        "database": {
+            "enable_duplicate_detection": (
+                PromptManagerConfig.ENABLE_DUPLICATE_DETECTION
+            ),
+            "enable_auto_save": PromptManagerConfig.ENABLE_AUTO_SAVE,
+        },
+        "web_ui": {
+            "result_timeout": PromptManagerConfig.RESULT_TIMEOUT,
+            "show_test_button": PromptManagerConfig.SHOW_TEST_BUTTON,
+            "webui_display_mode": PromptManagerConfig.WEBUI_DISPLAY_MODE,
+            "infinite_scroll": PromptManagerConfig.INFINITE_SCROLL,
+        },
+        "performance": {
+            "max_search_results": PromptManagerConfig.MAX_SEARCH_RESULTS,
+            "enable_fuzzy_search": PromptManagerConfig.ENABLE_FUZZY_SEARCH,
+            "auto_backup_interval": PromptManagerConfig.AUTO_BACKUP_INTERVAL,
+            "worker_threads": PromptManagerConfig.WORKER_THREADS,
+        },
+        "gallery": {
+            "processing_delay": GalleryConfig.PROCESSING_DELAY,
+            "prompt_timeout": GalleryConfig.PROMPT_TIMEOUT,
+            "cleanup_interval": GalleryConfig.CLEANUP_INTERVAL,
+            "auto_cleanup_missing_files": GalleryConfig.AUTO_CLEANUP_MISSING_FILES,
+            "max_image_age_days": GalleryConfig.MAX_IMAGE_AGE_DAYS,
+            "max_concurrent_processing": GalleryConfig.MAX_CONCURRENT_PROCESSING,
+            "metadata_extraction_timeout": GalleryConfig.METADATA_EXTRACTION_TIMEOUT,
+            # config.json gallery.monitoring.save_foreign_runs (#247)
+            "save_foreign_runs": GalleryConfig.SAVE_FOREIGN_RUNS,
+        },
+        "integrations": {
+            "lora_manager_enabled": bool(IntegrationConfig.LORA_MANAGER_ENABLED),
+            "has_civitai_api_key": bool(IntegrationConfig.CIVITAI_API_KEY),
+        },
+    }
+
+
+def _source_counts(sources):
+    configured = sources.get("configured") or []
+    effective = sources.get("effective") or []
+    return {
+        "configured": len(configured),
+        "effective": len(effective),
+        "watching": sum(1 for row in effective if row.get("watch") == "watching"),
+        "unavailable": sum(1 for row in configured if row.get("valid") is False),
+    }
+
+
+class DiagnosticsReportMixin:
+    """Mixin providing ``GET /prompt_manager/system/diagnostics``."""
+
+    def _register_diagnostics_report_routes(self, routes):
+        @routes.get("/prompt_manager/system/diagnostics")
+        async def diagnostics_report_route(request):
+            return await self.get_diagnostics_report(request)
+
+    async def get_diagnostics_report(self, request):
+        """The report, built off the loop under a lease (409 ``busy`` in a restore)."""
+        from ..config import IntegrationConfig
+
+        try:
+            async with self._jobs.lease(DIAGNOSTICS_LEASE):
+                report = await self._run_in_executor(self.build_report)
+        except JobBusy as busy:
+            return busy_response(busy)
+        except Exception:
+            self.logger.exception("The diagnostic report could not be built")
+            return web.json_response(
+                {
+                    "success": False,
+                    "code": None,
+                    "error": "The diagnostic report could not be built.",
+                },
+                status=500,
+            )
+        report = scrub(report, IntegrationConfig.CIVITAI_API_KEY)
+        return web.json_response(report, headers={"Cache-Control": "no-store"})
+
+    def build_report(self):
+        """The report's fields (blocking: reads the database and the log folder)."""
+        db = self.db
+        manager = self._get_logger_manager()
+        stats = dict(manager.get_log_stats())
+        stats["log_directory"] = report_path(stats.get("log_directory"))
+        info = dict(db.model.get_database_info() or {})
+        if "database_path" in info:
+            info["database_path"] = report_path(info["database_path"])
+        sources, _monitor = self._source_status()
+        return {
+            "success": True,
+            "generated_at": datetime.datetime.now().replace(microsecond=0).isoformat(),
+            "version": project_version(_ROOT),
+            "python": platform.python_version(),
+            "platform": {"system": platform.system(), "release": platform.release()},
+            "viewerjs": viewerjs_version(VIEWERJS_PROVENANCE),
+            "checks": _admin.legacy_checks(db.model.db_path, report_path),
+            "database_info": info,
+            "sources": _source_counts(sources),
+            "logging": {"config": dict(manager.get_config()), "stats": stats},
+            "health": health_summary(self._current_health_report()),
+            "config": public_config(),
+            "redacted": 0,
+        }

py/api/duplicate_files.py

@@ -0,0 +1,790 @@
+"""Duplicate image files on the job registry (#249 System S6, Task 17).
+
+- ``POST /prompt_manager/jobs/duplicates`` starts the find (202 ``{job}``):
+  kind ``duplicates``, "Find duplicate image files", admission ``read``,
+  cancellable, outcome group ``groups_found``. It hashes the media files of
+  every effective source (``_hash_media_groups``, shared with the legacy
+  scan) and the server keeps the groups it found under the job id: content
+  hash, root-relative paths and each file's source index. A find is refused
+  while a duplicate removal runs, and one find runs at a time.
+- ``GET /prompt_manager/maintenance/duplicates/{job_id}?offset&limit`` pages
+  those groups, at most ``MAX_GROUP_PAGE`` at a time: ``{success, job:{id,
+  state}, total, offset, limit, groups:[{group_id, size, count, files}]}``.
+  Each file names its root-relative ``path``, its source's public label and
+  index and the Images serve URLs; never an absolute path.
+- ``POST /prompt_manager/maintenance/preview {operation:"duplicate_files",
+  options:{job_id, keep:[{group_id, root_index, path}]}}`` reviews a removal:
+  in each group named, the files kept are the ones listed and every other
+  copy is removed. Each keeper and each removal is checked against the held
+  group, with containment decided as text before any filesystem access
+  (``file_problem``): inside its source, a media file outside
+  ``thumbnails``, then a regular file (not a link) whose resolved path is
+  still inside the source, with the same size and content hash. A removal
+  that fails, or whose kept copy fails, is protected and listed with the
+  reason. The plan carries the files' database links, which move to the
+  kept copy.
+- ``POST /prompt_manager/maintenance/apply {token}`` runs the reviewed plan
+  as the ``duplicate_cleanup`` job: admission ``records``, not cancellable,
+  groups ``removed``, ``kept`` and ``errors``. It checks the sources are
+  unchanged, takes the safety backup the policy promises (the database only;
+  removed files are not in it), then for each removal rechecks its kept copy
+  and the file itself (``file_problem``) and removes it with every handle
+  closed (``_unlink``), together with its thumbnail. A file that changed is
+  kept; the first unlink that fails stops the job, a locked file with the
+  coded ``LOCKED_FILE`` message (never the OS text), and the files after it
+  are not attempted. The database links of the files actually removed then
+  move to their kept copy (``PromptDatabase.move_file_links``).
+
+The legacy gallery routes (``/scan_duplicates``, its stream and status, and
+``/delete_duplicate_images``) keep their shapes on the same registry kinds;
+see ``py/api/admin.py``.
+"""
+
+import asyncio
+import os
+from pathlib import Path
+
+from aiohttp import web
+
+from .admin import DUPLICATE_FIND_LEGACY_TITLE, _remove_thumbnail
+from .duplicate_review import (  # noqa: F401 (CHANGED, OUTSIDE_SOURCE: public)
+    CHANGED,
+    DELETE_FAILED,
+    FIND_CANCELLED,
+    FIND_ERROR_EVENT,
+    FIND_FAILED,
+    GONE,
+    KEEPER,
+    KEEPER_CHANGED,
+    KEEPER_GONE,
+    KEEPER_WENT,
+    KEPT_COPY,
+    LOCKED_FILE,
+    MAYBE_SAME_FILE,
+    NO_FILE_ID_ROW,
+    NOT_ATTEMPTED,
+    OUTSIDE_SOURCE,
+    RECORDS_FAILED,
+    ROW_DETAILS,
+    SAME_FILE,
+    SOURCES_CHANGED,
+    SOURCES_CHANGED_REVIEW,
+    HeldFind,
+    SourcesChanged,
+    _fingerprint,
+    _group_order,
+    _identity,
+    _parse_options,
+    _plural,
+    _public_plan,
+    _select,
+    file_problem,
+    human_bytes,
+    still_kept,
+)
+from .duplicate_spellings import Spellings, copy_move
+from .images import _favorite_root_key, _output_image_entry
+from .job_routes import busy_response
+from .jobs import (
+    DUPLICATE_CLEANUP_GROUP_NAMES,
+    DUPLICATE_CLEANUP_JOB_KIND,
+    DUPLICATE_GROUP_NAMES,
+    DUPLICATES_JOB_KIND,
+    JobBusy,
+)
+from .maintenance import DUPLICATE_FILES, _fail_job, _failure, _invalid, backup_note
+
+try:
+    from ...database.duplicate_copies import FILE_LINK_SQL
+except ImportError:
+    from database.duplicate_copies import FILE_LINK_SQL
+
+DUPLICATE_FIND_TITLE = "Find duplicate image files"
+DUPLICATE_CLEANUP_TITLE = "Remove duplicate image files"
+MAX_GROUP_PAGE = 10  # groups a page lists (D69: ten at a time)
+MAX_HELD_FINDS = 3  # finds whose groups the server keeps for review
+favorite_root_key = _favorite_root_key
+
+
+def query_plan_sql():
+    """The link reconciliation's statements (EXPLAIN QUERY PLAN checks)."""
+    return list(FILE_LINK_SQL)
+
+
+def _unlink(path):
+    """Remove one file (the caller closed every handle on it)."""
+    os.remove(path)
+
+
+class DuplicateFilesMixin:
+    """Mixin providing the duplicate-file find, its groups and its removal."""
+
+    def _register_duplicate_routes(self, routes):
+        @routes.post("/prompt_manager/jobs/duplicates")
+        async def start_duplicate_find_route(request):
+            return await self.start_duplicate_find(request)
+
+        @routes.get("/prompt_manager/maintenance/duplicates/{job_id}")
+        async def duplicate_groups_route(request):
+            return await self.duplicate_groups(request)
+
+    # ── find ──────────────────────────────────────────────────────────
+
+    async def start_duplicate_find(self, request):
+        """Start the duplicate find (202), or 409 naming the job in the way."""
+        try:
+            job = self._start_duplicate_find(origin="workspace")
+        except JobBusy as busy:
+            return busy_response(busy)
+        return web.json_response({"success": True, "job": job.summary()}, status=202)
+
+    def _start_duplicate_find(self, origin):
+        """Register a find job (on the loop); raises JobBusy.
+
+        A workspace find walks every effective source; a gallery one the
+        gallery's output folder, as the legacy scan did. Both resolve their
+        folders when the job begins, so a source change later cannot move it.
+        """
+        cleanup = self._jobs.active(DUPLICATE_CLEANUP_JOB_KIND)
+        if cleanup is not None:
+            raise JobBusy(cleanup)
+        held = HeldFind(origin)
+        if origin == "workspace":
+            title = DUPLICATE_FIND_TITLE
+        else:
+            title = DUPLICATE_FIND_LEGACY_TITLE
+        job = self._jobs.start(
+            DUPLICATES_JOB_KIND,
+            title,
+            admission="read",
+            cancellable=True,
+            groups=DUPLICATE_GROUP_NAMES,
+            runner=lambda job: self._run_duplicate_find(job, held),
+        )
+        held.job_id = job.id
+        self._remember_find(held)
+        return job
+
+    def _remember_find(self, held):
+        finds = self._duplicate_finds
+        finds[held.job_id] = held
+        for job_id in list(finds):
+            if self._jobs.get(job_id) is None:
+                del finds[job_id]
+        while len(finds) > MAX_HELD_FINDS:
+            del finds[next(iter(finds))]
+
+    def _find_covers(self, job, legacy_dir):
+        held = self._duplicate_finds.get(job.id)
+        return held is not None and held.covers(legacy_dir)
+
+    def _latest_duplicate_find(self):
+        """The running find, else the newest finished one this session."""
+        for job in self._jobs.list():
+            if job.kind == DUPLICATES_JOB_KIND and job.id in self._duplicate_finds:
+                return job
+        return None
+
+    def _resolve_find_roots(self, held):
+        """Fill in where a find looks (blocking): sources and the gallery folder."""
+        listed = [os.path.abspath(str(d)) for d in self._get_all_output_dirs()]
+        held.legacy_dir = self._find_comfyui_output_dir()
+        if held.origin == "workspace":
+            held.roots = list(enumerate(listed))
+            return
+        if not held.legacy_dir or not os.path.isdir(held.legacy_dir):
+            self.logger.warning("ComfyUI output directory not found")
+            held.roots = []
+            return
+        legacy = os.path.abspath(held.legacy_dir)
+        index = next(
+            (i for i, d in enumerate(listed) if _identity(d) == _identity(legacy)),
+            None,
+        )
+        held.roots = [(index, legacy)]
+
+    def _find_progress(self, job, done, total):
+        if done < job.processed:
+            return  # hashing threads report out of order; keep the count rising
+        pct = int(done / total * 100) if total else 100
+        job.update(
+            processed=done,
+            total=total,
+            status_line=f"Hashing file {done:,} of {total:,}",
+        )
+        job.publish(
+            {
+                "type": "progress",
+                "progress": pct,
+                "status": f"Hashing file {done}/{total}...",
+                "processed": done,
+                "found": 0,
+            }
+        )
+
+    async def _run_duplicate_find(self, job, held):
+        loop = asyncio.get_running_loop()
+
+        def report(done, total):  # from the hashing threads
+            loop.call_soon_threadsafe(self._find_progress, job, done, total)
+
+        try:
+            job.update(status_line="Looking for media files")
+            job.publish(
+                {
+                    "type": "progress",
+                    "progress": 0,
+                    "status": "Collecting media files...",
+                    "processed": 0,
+                    "found": 0,
+                }
+            )
+            await self._run_in_executor(self._resolve_find_roots, held)
+            job.scope["output_folders"] = len(held.roots)
+            job.log_line(f"Searching {_plural(len(held.roots), 'source')}")
+            groups = await self._run_in_executor(
+                self._hash_media_groups,
+                [root for _index, root in held.roots],
+                report,
+                lambda: job.cancelled,
+            )
+            if job.cancelled:
+                job.update(status_line="Cancelled; nothing was changed.")
+                job.publish({"type": "error", "message": FIND_CANCELLED})
+                job.finish("cancelled")
+                return
+            await self._finish_find(job, held, groups)
+        except Exception:
+            self.logger.exception("Duplicate search failed")
+            job.update(status_line=FIND_FAILED)
+            job.publish({"type": "error", "message": FIND_ERROR_EVENT})
+            job.finish("failed")
+
+    async def _finish_find(self, job, held, groups):
+        for group in groups:
+            group["group_id"] = group["hash"]
+            for record in group["files"]:
+                index, root = held.roots[record.pop("slot")]
+                record["root_index"] = index
+                record["root"] = root
+        groups.sort(key=_group_order)
+        held.groups = groups
+        labels = {root: self._public_path(root) for _index, root in held.roots}
+        rows = [
+            {
+                "path": group["files"][0]["rel"],
+                "root": labels[group["files"][0]["root"]],
+                "root_index": group["files"][0]["root_index"],
+                "detail": f"{len(group['files'])} identical files · "
+                f"{human_bytes(group['size'])} each",
+            }
+            for group in groups
+        ]
+        job.record_counted("groups_found", len(groups), rows)
+        line = f"Found {_plural(len(groups), 'group')} of duplicate files"
+        job.log_line(line)
+        legacy = []
+        if held.covers(held.legacy_dir) and held.legacy_dir:
+            legacy = await self._run_in_executor(
+                self._legacy_duplicate_groups, groups, held.legacy_dir
+            )
+        # Every file was read by now (the last reports may still be queued).
+        job.update(progress=100, status_line=line, processed=job.total)
+        job.publish(
+            {
+                "type": "complete",
+                "processed": job.processed,
+                "found": len(legacy),
+                "duplicates": legacy,
+                "message": f"Found {len(legacy)} groups of duplicate images",
+            }
+        )
+
+    # ── groups ────────────────────────────────────────────────────────
+
+    def _held_result(self, job_id):
+        """``(held, None)`` for a finished find's groups, else an error reply."""
+        held = self._duplicate_finds.get(job_id)
+        job = self._jobs.get(job_id)
+        if held is None or job is None or job.kind != DUPLICATES_JOB_KIND:
+            return None, _failure(
+                "No duplicate search with this id. Search again.", 404, "not-found"
+            )
+        if job.running:
+            return None, _failure(
+                "The duplicate search is still running.", 409, "job-running"
+            )
+        if held.groups is None:
+            return None, _failure(
+                "This search did not finish. Search again.", 409, "no-groups"
+            )
+        return held, None
+
+    async def duplicate_groups(self, request):
+        """One page of a finished find's groups (at most MAX_GROUP_PAGE)."""
+        try:
+            offset = int(request.query.get("offset", "0"))
+            limit = int(request.query.get("limit", str(MAX_GROUP_PAGE)))
+        except ValueError:
+            return _invalid("offset and limit must be whole numbers")
+        if offset < 0 or not 0 < limit <= MAX_GROUP_PAGE:
+            return _invalid(f"limit must be 1 to {MAX_GROUP_PAGE}; offset 0 or more")
+        job_id = request.match_info["job_id"]
+        held, error = self._held_result(job_id)
+        if error is not None:
+            return error
+        page = held.groups[offset : offset + limit]
+        groups = await self._run_in_executor(self._describe_groups, page)
+        job = self._jobs.get(job_id)
+        return web.json_response(
+            {
+                "success": True,
+                "job": {"id": job.id, "state": job.state},
+                "total": len(held.groups),
+                "offset": offset,
+                "limit": limit,
+                "groups": groups,
+            },
+            headers={"Cache-Control": "no-store"},
+        )
+
+    def _describe_groups(self, groups):
+        """Public group rows (blocking: one stat per file)."""
+        return [
+            {
+                "group_id": group["group_id"],
+                "size": group["size"],
+                "count": len(group["files"]),
+                "files": [self._describe_copy(record) for record in group["files"]],
+            }
+            for group in groups
+        ]
+
+    def _describe_copy(self, record):
+        label = self._public_path(record["root"])
+        base = {
+            "filename": os.path.basename(record["rel"]),
+            "path": record["rel"],
+            "root": label,
+            "root_index": record["root_index"],
+            "url": None,
+            "thumbnail_url": None,
+            "size": record["size"],
+            "modified_time": record["mtime"],
+            "media_type": "image",
+            "missing": False,
+        }
+        if record["root_index"] is None:
+            return base
+        try:
+            entry = _output_image_entry(
+                Path(record["abs"]), Path(record["root"]), record["root_index"], label
+            )
+        except (OSError, ValueError):
+            return {**base, "missing": True}
+        return {
+            **base,
+            "url": entry["url"],
+            "thumbnail_url": entry["thumbnail_url"],
+            "size": entry["size"],
+            "modified_time": entry["modified_time"],
+            "media_type": entry["media_type"],
+        }
+
+    # ── review ────────────────────────────────────────────────────────
+
+    def _listed_sources(self):
+        """``{identity: index}`` of the effective sources now (blocking)."""
+        return {
+            _identity(d): index for index, d in enumerate(self._get_all_output_dirs())
+        }
+
+    @staticmethod
+    def _source_listed(listed, index, root):
+        return index is not None and listed.get(_identity(root)) == index
+
+    async def preview_duplicate_files(self, options):
+        """The ``duplicate_files`` plan for one finished find and its keepers."""
+        job_id, keep, message = _parse_options(options)
+        if message:
+            return _invalid(message)
+        held, error = self._held_result(job_id)
+        if error is not None:
+            return error
+        selection, message = _select(held, keep)
+        if message:
+            return _invalid(message)
+        db = self.db
+        identity = self._database_identity(db)
+        try:
+            review = await self._run_in_executor(
+                self._review_duplicate_files, db, selection
+            )
+        except SourcesChanged:
+            return _failure(SOURCES_CHANGED_REVIEW, 409, "sources-changed")
+        except Exception:
+            self.logger.exception("Duplicate file review failed")
+            return _failure("Failed to review maintenance", 500)
+        return await self._plan_response(
+            db,
+            identity,
+            DUPLICATE_FILES,
+            review["fingerprint"],
+            review["public"],
+            {
+                "targets": review["targets"],
+                "kept": review["kept"],
+                "protected": review["public"]["protected"]["count"],
+            },
+        )
+
+    def _review_duplicate_files(self, db, selection):
+        """Check every keeper and removal now (blocking); the plan's parts.
+
+        No file is read: each copy is bound to what the search saw by its
+        size, mtime and inode, and never to its kept copy itself. The removal
+        job hashes each file again right before it unlinks it.
+        """
+        listed = self._listed_sources()
+        targets, kept, protected = [], [], {}
+        for group, keepers in selection.values():
+            removals = [r for r in group["files"] if r not in keepers]
+            for record in removals:
+                if record["root_index"] is not None and not self._source_listed(
+                    listed, record["root_index"], record["root"]
+                ):
+                    raise SourcesChanged(record["root"])
+            keeper = next(
+                (k for k in keepers if _bound_problem(k, group) is None), None
+            )
+            reviewed = []
+            for record in removals:
+                if keeper is None:
+                    problem = KEEPER
+                elif record["root_index"] is None:
+                    problem = OUTSIDE_SOURCE
+                else:
+                    problem = _bound_problem(record, group, keeper)
+                if problem:
+                    protected[problem] = protected.get(problem, 0) + 1
+                    continue
+                reviewed.append(self._target(group, record, keeper))
+            if reviewed:
+                targets.extend(reviewed)
+                kept.append(self._copy_row(keeper, KEPT_COPY))
+        records = {}
+        if targets:
+            spellings = self._source_spellings()
+            records = db.count_copy_records(
+                {
+                    t["abs"]: {
+                        "spellings": list(spellings.of(t["abs"]).values()),
+                        "locator": t["locator"],
+                        "sha256": t["hash"],
+                    }
+                    for t in targets
+                }
+            )
+        return {
+            "targets": targets,
+            "kept": kept,
+            "fingerprint": _fingerprint(targets),
+            "public": _public_plan(targets, protected, records),
+        }
+
+    def _source_spellings(self):
+        """The folder spellings records may name files by (blocking).
+
+        The search walks each source's real path, while a record names a file
+        as it was seen when written: under the folder the image monitor
+        watches, the configured spelling, ComfyUI's output folder, a symlink
+        to it, or a parent of a source added later. ``Spellings`` matches
+        them to each file by containment (``py/api/duplicate_spellings.py``).
+        """
+        from ..config import GalleryConfig
+
+        listed = [os.path.abspath(str(d)) for d in self._get_all_output_dirs()]
+        candidates = list(self._monitored_directories())
+        candidates += list(GalleryConfig.MONITORING_DIRECTORIES)
+        try:
+            import folder_paths
+
+            candidates.append(folder_paths.get_output_directory())
+        except Exception:  # nosec B110 - outside ComfyUI there is none
+            pass
+        candidates.append(self._find_comfyui_output_dir())
+        folders = [
+            os.path.abspath(str(c)) for c in candidates if c and os.path.isabs(str(c))
+        ]
+        return Spellings(folders + listed, listed)
+
+    def _target(self, group, record, keeper):
+        return {
+            "group_id": group["group_id"],
+            "hash": group["hash"],
+            "size": record["size"],
+            "root_index": record["root_index"],
+            "root": record["root"],
+            "root_label": self._public_path(record["root"]),
+            "abs": record["abs"],
+            "rel": record["rel"],
+            "keeper_abs": keeper["abs"],
+            "keeper_rel": keeper["rel"],
+            "keeper_root": keeper["root"],
+            "keeper_size": keeper["size"],
+            "locator": os.path.normcase(record["abs"]),
+            "keeper_locator": os.path.normcase(keeper["abs"]),
+        }
+
+    def _copy_row(self, record, detail):
+        return {
+            "path": record["rel"],
+            "root": self._public_path(record["root"]),
+            "root_index": record["root_index"],
+            "detail": detail,
+        }
+
+    # ── removal ───────────────────────────────────────────────────────
+
+    def _start_duplicate_cleanup(self, db, plan, backup_on):
+        """Register the removal job for a reviewed plan (on the loop)."""
+        find = self._jobs.active(DUPLICATES_JOB_KIND)
+        if find is not None:
+            raise JobBusy(find)
+        return self._jobs.start(
+            DUPLICATE_CLEANUP_JOB_KIND,
+            DUPLICATE_CLEANUP_TITLE,
+            admission="records",
+            cancellable=False,
+            scope={"operation": DUPLICATE_FILES},
+            groups=DUPLICATE_CLEANUP_GROUP_NAMES,
+            runner=lambda job: self._run_duplicate_cleanup(job, db, plan, backup_on),
+        )
+
+    def _sources_unchanged(self, targets):
+        listed = self._listed_sources()
+        return all(
+            self._source_listed(listed, t["root_index"], t["root"]) for t in targets
+        )
+
+    async def _run_duplicate_cleanup(self, job, db, plan, backup_on):
+        targets = plan.payload["targets"]
+        unchanged = await self._run_in_executor(self._sources_unchanged, targets)
+        if not unchanged:
+            return _fail_job(job, "sources-changed", SOURCES_CHANGED)
+        ok, backup = await self._safety_backup_step(job, db, backup_on)
+        if not ok:
+            return
+        try:
+            await self._remove_duplicates(job, db, plan, backup)
+        finally:
+            if backup:
+                await self._prune_backup_pool(job, db)
+
+    async def _remove_duplicates(self, job, db, plan, backup):
+        targets = plan.payload["targets"]
+        for row in plan.payload["kept"]:
+            job.record("kept", row, f"= {row['root']}/{row['path']} · {KEPT_COPY}")
+        job.update(total=len(targets), status_line="Removing duplicate files")
+        spellings = await self._run_in_executor(self._source_spellings)
+        removed, failure, keepers = 0, None, {}
+        totals = dict.fromkeys(
+            ("moved", "merged", "favorites", "tags", "review", "collections"), 0
+        )
+        totals["categories"] = 0  # image categories kept on the kept copies (#332)
+        for done, target in enumerate(targets, start=1):
+            if failure is not None:
+                self._record_target(job, "kept", target, NOT_ATTEMPTED)
+                continue
+            outcome, detail, counts = await self._run_in_executor(
+                self._remove_one_duplicate, db, target, keepers, spellings
+            )
+            for key, value in (counts or {}).items():
+                totals[key] += value
+            if outcome == "removed":
+                removed += 1
+                text = f"A copy of {target['keeper_rel']}"
+                self._record_target(job, "removed", target, text, "-")
+            elif outcome == "kept":
+                self._record_target(job, "kept", target, ROW_DETAILS[detail])
+            else:
+                failure = detail
+                self._record_target(job, "errors", target, detail, "!")
+            job.update(processed=done, progress=int(done / len(targets) * 100))
+        if removed:
+            self.invalidate_gallery_cache()
+        self._log_moved_records(job, totals)
+        line = f"Removed {_plural(removed, 'duplicate file')}"
+        if removed < len(targets):
+            line += f", {len(targets) - removed:,} not removed"
+        job.log_line(line)
+        if failure is not None:
+            line = f"Stopped after an error · {line}"
+        job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
+        if failure is not None:
+            job.publish({"type": "error", "code": "delete-failed", "message": line})
+            job.finish("failed")
+
+    @staticmethod
+    def _log_moved_records(job, totals):
+        links = totals["moved"] + totals["merged"]
+        if links:
+            job.log_line(f"Moved {_plural(links, 'database link')} to the kept copies")
+        if totals["favorites"]:
+            job.log_line(
+                f"Moved {_plural(totals['favorites'], 'favorite')} to the kept copies"
+            )
+        if totals["collections"]:
+            job.log_line(
+                f"Moved {_plural(totals['collections'], 'collection membership')} "
+                "to the kept copies"
+            )
+        if totals["tags"]:
+            job.log_line(
+                f"Added {_plural(totals['tags'], 'image tag')} to the kept copies"
+            )
+        if totals.get("categories"):
+            kept = _plural(totals["categories"], "image category", "image categories")
+            job.log_line(f"Kept {kept} on the kept copies")
+        if totals["review"]:
+            job.log_line(
+                f"Settled {_plural(totals['review'], 'Needs review item')} "
+                "of removed copies"
+            )
+
+    @staticmethod
+    def _record_target(job, group, target, detail, mark="="):
+        row = {
+            "path": target["rel"],
+            "root": target["root_label"],
+            "root_index": target["root_index"],
+            "detail": detail,
+        }
+        job.record(
+            group, row, f"{mark} {target['root_label']}/{target['rel']} · {detail}"
+        )
+
+    def _remove_one_duplicate(self, db, target, keepers, spellings):
+        """``(outcome, detail, counts)`` for one reviewed removal (blocking).
+
+        The kept copy (once per group; cached in ``keepers``) and the file
+        are checked again, hashes included, and the file must not be its kept
+        copy. Then the file's records move to the kept copy in one short
+        transaction, and only then is the file unlinked: a crash or a failed
+        unlink leaves its records on the identical kept copy, never on a
+        deleted file. A failed move leaves the file and its records alone.
+        """
+        hasher = self._calculate_file_hash
+        group_id = target["group_id"]
+        if group_id not in keepers:
+            keepers[group_id] = _kept_copy_state(target, hasher)
+        if keepers[group_id] is None:
+            return "kept", KEEPER, None
+        problem = file_problem(
+            target["abs"],
+            target["root"],
+            target["hash"],
+            target["size"],
+            hasher,
+            keeper=target["keeper_abs"],
+            keeper_root=target["keeper_root"],
+        )
+        if problem:
+            return "kept", problem, None
+        try:
+            # Held open until the unlink: on Windows, deleting the kept copy
+            # itself through another path then fails instead of removing it.
+            hold = open(target["keeper_abs"], "rb")
+        except OSError:
+            return "kept", KEEPER, None
+        with hold:
+            return self._move_then_unlink(
+                db, target, keepers[group_id], spellings, hold
+            )
+
+    def _move_then_unlink(self, db, target, kept_stat, spellings, hold):
+        try:
+            counts = db.move_copy_records(copy_move(target, kept_stat, spellings))
+        except Exception:
+            self.logger.exception(f"Could not move the records of {target['rel']}")
+            return "error", RECORDS_FAILED, None
+        if not still_kept(target, hold):
+            # The kept copy went away or changed while the records moved:
+            # keep this file and give its records back to it.
+            return "kept", KEEPER_GONE, self._move_back(db, target, spellings)
+        try:
+            _unlink(target["abs"])
+        except FileNotFoundError:
+            return "removed", None, counts
+        except PermissionError:
+            self.logger.warning(
+                f"Duplicate file is locked: {target['rel']}; "
+                "its records now name the kept copy"
+            )
+            return "error", LOCKED_FILE, counts
+        except OSError:
+            self.logger.warning(
+                f"Could not delete duplicate file {target['rel']}; "
+                "its records now name the kept copy",
+                exc_info=True,
+            )
+            return "error", DELETE_FAILED, counts
+        self.logger.info(f"Deleted duplicate file: {target['rel']}")
+        _remove_thumbnail(Path(target["root"]), Path(target["rel"]), self.logger)
+        return "removed", None, counts
+
+    def _move_back(self, db, target, spellings):
+        """Give the kept copy's records to the file that stays (blocking)."""
+        swapped = {
+            **target,
+            "abs": target["keeper_abs"],
+            "root": target["keeper_root"],
+            "rel": target["keeper_rel"],
+            "locator": target["keeper_locator"],
+            "keeper_abs": target["abs"],
+            "keeper_root": target["root"],
+            "keeper_rel": target["rel"],
+            "keeper_locator": target["locator"],
+        }
+        try:
+            return db.move_copy_records(
+                copy_move(swapped, os.stat(target["abs"]), spellings)
+            )
+        except Exception:
+            self.logger.exception(f"Could not give the records back to {target['rel']}")
+            return None
+
+
+def _bound_problem(record, group, keeper=None):
+    """``file_problem`` without reading the file: size, mtime and inode.
+
+    ``keeper`` is the kept copy's record, never offered as a copy of itself.
+    """
+    return file_problem(
+        record["abs"],
+        record["root"],
+        group["hash"],
+        record["size"],
+        hash_check=False,
+        mtime_ns=record.get("mtime_ns"),
+        inode=record.get("inode"),
+        keeper=keeper["abs"] if keeper else None,
+        keeper_root=keeper["root"] if keeper else None,
+    )
+
+
+def _kept_copy_state(target, hasher):
+    """The kept copy's ``os.stat`` when it is still the reviewed bytes, else None."""
+    problem = file_problem(
+        target["keeper_abs"],
+        target["keeper_root"],
+        target["hash"],
+        target["keeper_size"],
+        hasher,
+    )
+    if problem:
+        return None
+    try:
+        return os.stat(target["keeper_abs"])
+    except OSError:
+        return None

py/api/duplicate_review.py

@@ -0,0 +1,460 @@
+"""Reviewing duplicate image files: the checks and the plan (#249 S6, T17).
+
+Pure helpers of ``py/api/duplicate_files.py``: the keep-list parsing, the
+per-file check ``file_problem`` (containment decided as text before any
+filesystem access, then a regular file of the found size and hash) and the
+public ``duplicate_files`` plan.
+"""
+
+import hashlib
+import json
+import os
+import stat as stat_module
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from .admin import DELETABLE_MEDIA_EXTENSIONS
+from .maintenance import DUPLICATE_FILES
+
+try:
+    from ...utils.media_folders import in_trash_below_roots, is_trash_folder
+except ImportError:
+    from utils.media_folders import in_trash_below_roots, is_trash_folder
+
+# Keep entries one review takes; with paths of a few hundred characters the
+# list stays inside the preview body cap (DUPLICATE_PREVIEW_BODY_MAX, 1 MiB).
+MAX_KEEP = 2000
+MAX_DETAILS = 50
+MAX_JOB_ID = 64
+HASH_CHUNK = 1024 * 1024
+
+# What ``file_problem`` finds, and how a plan counts it as protected.
+OUTSIDE_SOURCE = "outside"
+LINK = "link"
+CHANGED_CONTENT = "changed"
+GONE = "gone"
+UNREADABLE = "unreadable"
+KEEPER = "keeper"
+SAME = "same"
+MAYBE_SAME = "maybe-same"
+NO_FILE_ID = "no-file-id"
+# Two mounts of one share may round mtimes differently (SMB keeps 100 ns,
+# FAT 2 s): within this, equal sizes on another device may be one file.
+MOUNT_MTIME_SLACK_NS = 2_000_000_000
+PROTECTED_REASONS = {
+    MAYBE_SAME: (
+        "file may be the same file reached through another mount or share",
+        "files may be the same file reached through another mount or share",
+    ),
+    NO_FILE_ID: (
+        "file: can't confirm this is a separate file on this drive",
+        "files: can't confirm these are separate files on this drive",
+    ),
+    SAME: (
+        "file is the same file as its kept copy",
+        "files are the same file as their kept copy",
+    ),
+    OUTSIDE_SOURCE: (
+        "file is not inside a configured source",
+        "files are not inside a configured source",
+    ),
+    LINK: ("file is a link, not a regular file", "files are links, not regular files"),
+    CHANGED_CONTENT: (
+        "file changed since the search",
+        "files changed since the search",
+    ),
+    GONE: ("file no longer exists", "files no longer exist"),
+    UNREADABLE: ("file could not be read", "files could not be read"),
+    KEEPER: (
+        "file kept: its kept copy changed or is gone",
+        "files kept: their kept copy changed or is gone",
+    ),
+}
+# Job outcome rows (never an OS error text).
+CHANGED = "The file changed since the review; it was kept."
+KEEPER_CHANGED = "The kept copy changed or is gone; this file was kept."
+SAME_FILE = "This is the same file as its kept copy; it was kept."
+MAYBE_SAME_FILE = (
+    "May be the same file reached through another mount or share; it was kept."
+)
+NO_FILE_ID_ROW = "Can't confirm this is a separate file on this drive; it was kept."
+KEEPER_GONE = "keeper-gone"
+KEEPER_WENT = (
+    "The kept copy changed or went away during the removal; this file was kept "
+    "and its links stay with it."
+)
+ROW_DETAILS = {
+    KEEPER_GONE: None,  # filled below
+    MAYBE_SAME: MAYBE_SAME_FILE,
+    NO_FILE_ID: NO_FILE_ID_ROW,
+    SAME: SAME_FILE,
+    OUTSIDE_SOURCE: "The file is no longer inside its source; it was kept.",
+    LINK: "The file is now a link, not a regular file; it was kept.",
+    CHANGED_CONTENT: CHANGED,
+    GONE: "The file was already gone.",
+    UNREADABLE: "The file could not be read; it was kept.",
+    KEEPER: KEEPER_CHANGED,
+}
+# An unlink fails only after the file's records moved to its kept copy.
+LOCKED_FILE = (
+    "The file is open in another program or read-only; it was not deleted, "
+    "and its links now name the kept copy."
+)
+DELETE_FAILED = "The file could not be deleted; its links now name the kept copy."
+NOT_ATTEMPTED = "Not attempted: the job stopped after an error."
+KEPT_COPY = "Kept copy"
+SOURCES_CHANGED = (
+    "The sources changed since the review; nothing was deleted. Review again."
+)
+RECORDS_FAILED = (
+    "The database could not be updated, so the file was not deleted. "
+    "Its links and tags are unchanged."
+)
+FIND_FAILED = "The duplicate search failed."
+FIND_ERROR_EVENT = "An internal error occurred. Check server logs for details."
+FIND_CANCELLED = "Duplicate scan cancelled"
+
+
+class SourcesChanged(Exception):
+    """A source a review or removal names is no longer listed at its index."""
+
+
+ROW_DETAILS[KEEPER_GONE] = KEEPER_WENT
+
+
+def still_kept(target, hold):
+    """Right before an unlink: the held kept copy is still the file at its
+    path, and the file is still not the kept copy itself (``keeper_problem``).
+    """
+    try:
+        if not os.path.samestat(os.fstat(hold.fileno()), os.stat(target["keeper_abs"])):
+            return False
+        info = os.lstat(target["abs"])
+        problem = keeper_problem(
+            target["abs"],
+            target["root"],
+            info,
+            target["keeper_abs"],
+            target["keeper_root"],
+        )
+        return problem is None
+    except OSError:
+        return False
+
+
+def _plural(count, one, many=None):
+    return f"{count:,} {one if count == 1 else (many or one + 's')}"
+
+
+def human_bytes(size):
+    """``10 B``, ``1.5 KB``, ``2.0 MB``…"""
+    if size < 1024:
+        return f"{size:,} B"
+    for unit in ("KB", "MB", "GB", "TB"):
+        size /= 1024
+        if size < 1024 or unit == "TB":
+            return f"{size:,.1f} {unit}"
+    return f"{size:,.1f} TB"  # pragma: no cover - unreachable
+
+
+def _hash_file(path):
+    digest = hashlib.sha256()
+    with open(path, "rb") as handle:
+        for chunk in iter(lambda: handle.read(HASH_CHUNK), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def file_problem(
+    path,
+    root,
+    digest,
+    size,
+    hasher=None,
+    *,
+    hash_check=True,
+    mtime_ns=None,
+    inode=None,
+    keeper=None,
+    keeper_root=None,
+):
+    """Why ``path`` may not be treated as a copy in ``root``, or None (blocking).
+
+    Containment is decided as text first, with no filesystem access: the
+    normalized path must lie below the normalized root, outside
+    ``thumbnails``, with a media extension. Only then is the file looked at:
+    a regular file, not a link, whose resolved path is still inside the
+    resolved root and that is not ``keeper`` itself (``keeper_problem``: a
+    hardlink, one file reached through two spellings or two mounts, or a
+    volume that cannot tell files apart), of ``size`` bytes.
+    ``mtime_ns`` and ``inode`` (``[st_dev, st_ino]``), when given, bind it to
+    what the search saw; ``hash_check`` compares its content hash with
+    ``digest`` (the removal job, right before an unlink). Every handle is
+    closed when this returns.
+    """
+    # py.config imports ComfyUI's server module: only when it is in use.
+    from ..config import _canonical_path, _lexical_path, _lexically_inside
+
+    lexical = _lexical_path(str(path))
+    base = _lexical_path(str(root))
+    if lexical == base or not _lexically_inside(lexical, [base]):
+        return OUTSIDE_SOURCE
+    parts = Path(os.path.relpath(lexical, base)).parts
+    if "thumbnails" in parts or ".." in parts:
+        return OUTSIDE_SOURCE
+    if any(is_trash_folder(part) for part in parts[:-1]):
+        return OUTSIDE_SOURCE  # never removed from a trash folder (#543)
+    if os.path.splitext(lexical)[1].lower() not in DELETABLE_MEDIA_EXTENSIONS:
+        return OUTSIDE_SOURCE
+    try:
+        info = os.lstat(path)
+    except FileNotFoundError:
+        return GONE
+    except OSError:
+        return UNREADABLE
+    if stat_module.S_ISLNK(info.st_mode):
+        return LINK
+    if not stat_module.S_ISREG(info.st_mode):
+        return LINK
+    try:
+        if not _lexically_inside(_canonical_path(path), [_canonical_path(root)]):
+            return OUTSIDE_SOURCE  # a linked folder on the way
+        if in_trash_below_roots(path, [root]):
+            return OUTSIDE_SOURCE  # a linked folder into a trash folder
+        if keeper is not None:
+            same = keeper_problem(path, root, info, keeper, keeper_root)
+            if same:
+                return same
+        if info.st_size != size:
+            return CHANGED_CONTENT
+        if mtime_ns is not None and info.st_mtime_ns != mtime_ns:
+            return CHANGED_CONTENT
+        if inode is not None and [info.st_dev, info.st_ino] != list(inode):
+            return CHANGED_CONTENT
+        if hash_check and (hasher or _hash_file)(path) != digest:
+            return CHANGED_CONTENT
+    except FileNotFoundError:
+        return GONE
+    except OSError:
+        return UNREADABLE
+    return None
+
+
+def keeper_problem(path, root, info, keeper, keeper_root=None):
+    """Why ``path`` (its ``lstat`` is ``info``) may be ``keeper`` itself, or None.
+
+    - ``NO_FILE_ID``: a volume reports no file id (``st_ino`` 0), so the two
+      cannot be told apart;
+    - ``SAME``: ``os.path.samestat`` (a hardlink, or one file reached through
+      two spellings of one mount);
+    - ``MAYBE_SAME``: another device, the same size, mtimes within
+      ``MOUNT_MTIME_SLACK_NS``, and the same ``st_ino`` (NFS and SMB report
+      the server's file id) or the same path within their sources, or one
+      ending with the other on a folder name (one share mounted twice, a
+      FUSE pool over a listed disk, sources listed at different depths).
+      Protect, never guess: a true copy on another disk at the same relative
+      path is kept too.
+
+    Raises OSError when the kept copy cannot be read (the caller's GONE or
+    UNREADABLE).
+    """
+    kept = os.stat(keeper)
+    if not info.st_ino or not kept.st_ino:
+        return NO_FILE_ID
+    if os.path.samestat(info, kept):
+        return SAME
+    if info.st_dev == kept.st_dev or info.st_size != kept.st_size:
+        return None
+    if abs(info.st_mtime_ns - kept.st_mtime_ns) > MOUNT_MTIME_SLACK_NS:
+        return None
+    if info.st_ino == kept.st_ino:
+        return MAYBE_SAME
+    if keeper_root is not None and _same_tail(
+        _relative_key(path, root), _relative_key(keeper, keeper_root)
+    ):
+        return MAYBE_SAME
+    return None
+
+
+def _same_tail(one, other):
+    """Whether one relative path is the other, or ends with it on a folder name.
+
+    Two mounts of one tree may be listed at different depths (``2024/x.png``
+    in one source, ``x.png`` in the other); ``ax.png`` never matches ``x.png``.
+    """
+    if one is None or other is None:
+        return False
+    return one == other or one.endswith("/" + other) or other.endswith("/" + one)
+
+
+def _relative_key(path, root):
+    try:
+        relative = os.path.relpath(str(path), str(root))
+    except ValueError:  # another drive (Windows)
+        return None
+    return os.path.normcase(relative).replace("\\", "/")
+
+
+@dataclass
+class HeldFind:
+    """One find: where it looked and, once it finished, what it found."""
+
+    origin: str  # "workspace" or "gallery"
+    job_id: str = ""
+    roots: list = field(default_factory=list)  # [(source index or None, root)]
+    legacy_dir: str = None
+    groups: list = None  # [{group_id, hash, size, files:[record]}]
+
+    def covers(self, legacy_dir):
+        """Whether its events answer the legacy gallery for ``legacy_dir``."""
+        if self.origin == "gallery":
+            return True
+        if not legacy_dir:
+            return False
+        wanted = _identity(legacy_dir)
+        return any(_identity(root) == wanted for _index, root in self.roots)
+
+
+def _identity(path):
+    return os.path.normcase(os.path.realpath(str(path)))
+
+
+def _group_order(group):
+    return (-(len(group["files"]) - 1) * group["size"], group["hash"])
+
+
+def _keep_entry(entry):
+    """``((group_id, root_index, path), None)`` or ``(None, message)``."""
+    if not isinstance(entry, dict) or set(entry) != {"group_id", "root_index", "path"}:
+        return None, "Each keep entry is {group_id, root_index, path}"
+    group_id, index, path = entry["group_id"], entry["root_index"], entry["path"]
+    if not isinstance(group_id, str) or not 0 < len(group_id) <= 128:
+        return None, "group_id must be a group's id"
+    if index is not None and (
+        isinstance(index, bool) or not isinstance(index, int) or index < 0
+    ):
+        return None, "root_index must be a source index"
+    if not isinstance(path, str) or not 0 < len(path) <= 4096:
+        return None, "path must be a file's path in its group"
+    return (group_id, index, path), None
+
+
+def _parse_options(options):
+    """``(job_id, keep, None)`` for duplicate_files options, else a message."""
+    if not isinstance(options, dict) or set(options) != {"job_id", "keep"}:
+        return None, None, "duplicate_files takes options {job_id, keep}"
+    job_id, keep = options["job_id"], options["keep"]
+    if not isinstance(job_id, str) or not 0 < len(job_id) <= MAX_JOB_ID:
+        return None, None, "job_id must be a duplicate search's job id"
+    if not isinstance(keep, list) or not 0 < len(keep) <= MAX_KEEP:
+        return None, None, f"keep must list 1 to {MAX_KEEP:,} files to keep"
+    parsed = []
+    for entry in keep:
+        key, message = _keep_entry(entry)
+        if message:
+            return None, None, message
+        parsed.append(key)
+    return job_id, parsed, None
+
+
+def _select(held, keep):
+    """``({group_id: (group, [keeper records])}, None)`` or ``(None, message)``."""
+    groups = {group["group_id"]: group for group in held.groups}
+    selection = {}
+    for group_id, index, path in keep:
+        group = groups.get(group_id)
+        if group is None:
+            return None, "A kept file names a group this search did not find"
+        match = [
+            record
+            for record in group["files"]
+            if record["root_index"] == index and record["rel"] == path
+        ]
+        if not match:
+            return None, "A kept file is not in its group"
+        _group, keepers = selection.setdefault(group_id, (group, []))
+        if match[0] not in keepers:
+            keepers.append(match[0])
+    return selection, None
+
+
+SOURCES_CHANGED_REVIEW = (
+    "The sources changed since the search. Search for duplicates again."
+)
+
+
+def _fingerprint(targets):
+    digest = hashlib.sha256()
+    for target in sorted(targets, key=lambda t: _identity(t["abs"])):
+        part = [
+            os.path.normcase(target["abs"]),
+            target["hash"],
+            target["size"],
+            os.path.normcase(target["keeper_abs"]),
+        ]
+        digest.update(json.dumps(part).encode("utf-8") + b"\n")
+    return "sha256:" + digest.hexdigest()
+
+
+def _record_parts(records, target):
+    """What moves with one removal: its links, tags and Needs review items."""
+    found = records.get(target["abs"], {})
+    links, tags, review = (found.get(key, 0) for key in ("links", "tags", "review"))
+    parts = [
+        (
+            (
+                f"{_plural(links, 'database link')} "
+                f"{'moves' if links == 1 else 'move'} to the kept copy"
+            )
+            if links
+            else "no database links"
+        )
+    ]
+    if tags:
+        parts.append("its image tags join the kept copy's")
+    if review:
+        item = "Needs review item is" if review == 1 else "Needs review items are"
+        parts.append(f"{review:,} {item} settled")
+    return parts
+
+
+def _public_plan(targets, protected, records):
+    count = len(targets)
+    groups = len({t["group_id"] for t in targets})
+    size = sum(t["size"] for t in targets)
+    reasons = [
+        f"{n:,} {PROTECTED_REASONS[code][0 if n == 1 else 1]}"
+        for code, n in sorted(protected.items())
+    ]
+    details = [
+        {
+            "label": target["rel"],
+            "detail": f"{target['root_label']} · a copy of {target['keeper_rel']}"
+            + "".join(f" · {part}" for part in _record_parts(records, target)),
+        }
+        for target in targets[:MAX_DETAILS]
+    ]
+    label = f"{_plural(count, 'duplicate file')} in {_plural(groups, 'group')}"
+    media = (
+        f"Deletes {_plural(count, 'file')} ({human_bytes(size)}) from disk; the "
+        "kept copies stay. The safety backup covers the database, not these files."
+    )
+    tagged = sum(1 for t in targets if records.get(t["abs"], {}).get("tags"))
+    review = sum(records.get(t["abs"], {}).get("review", 0) for t in targets)
+    if tagged:
+        media += (
+            f" The image tags of {_plural(tagged, 'removed copy', 'removed copies')}"
+            " join their kept copy's."
+        )
+    if review:
+        media += (
+            f" {_plural(review, 'pending Needs review item')} of the removed "
+            f"{'copy is' if review == 1 else 'copies are'} settled."
+        )
+    return {
+        "operation": DUPLICATE_FILES,
+        "scope": {"count": count, "label": label},
+        "protected": {"count": sum(protected.values()), "reasons": reasons},
+        "media": {"deletes_files": True, "label": media},
+        "details": details,
+    }

py/api/duplicate_spellings.py

@@ -0,0 +1,157 @@
+"""Every path a duplicate copy is recorded under (#249 System S6, Task 17).
+
+The search walks each source's real path, but a record names a file the way
+it was seen when it was written: under the folder the image monitor watches,
+the configured spelling, ComfyUI's output folder, a symlink to it, or a
+parent of a source listed later. ``Spellings`` finds them by containment:
+for each known folder spelling ``S`` (resolving to the folder ``R``) and a
+file ``P`` inside ``R``, the file is also recorded as ``S + relpath(P, R)``.
+
+Folders are compared by ``os.stat`` identity (``st_dev``, ``st_ino``), not
+by text, so a case variant on a case-insensitive filesystem (macOS, Windows)
+or a symlinked spelling matches. A volume that reports no file id falls
+back to ``normcase(realpath)``. Favorites are keyed by a listed root and the
+path below it, so ``favorites`` lists every listed root that holds a file.
+Blocking: each lookup stats the file's folders once (cached).
+"""
+
+import os
+from pathlib import Path
+
+from .images import _favorite_relative_key, _favorite_root_key, image_category_key
+
+
+def _identity(path):
+    try:
+        info = os.stat(path)
+    except OSError:
+        return None
+    if info.st_ino:
+        return ("inode", info.st_dev, info.st_ino)
+    return ("path", os.path.normcase(os.path.realpath(str(path))))
+
+
+class Spellings:
+    """Folder spellings and listed roots, matched to files by containment."""
+
+    def __init__(self, folders, listed_roots):
+        self._cache = {}
+        self._folders = []
+        for spelled in dict.fromkeys(str(f) for f in folders if f):
+            identity = self._identity(spelled)
+            if identity is not None:
+                self._folders.append((spelled, identity))
+        self._listed = []
+        for root in dict.fromkeys(str(r) for r in listed_roots if r):
+            identity = self._identity(root)
+            if identity is not None:
+                self._listed.append((root, identity))
+
+    def _identity(self, path):
+        if path not in self._cache:
+            self._cache[path] = _identity(path)
+        return self._cache[path]
+
+    def _ancestors(self, path):
+        """``{identity: folder}`` of every folder above ``path``, nearest first."""
+        found = {}
+        folder = os.path.dirname(os.path.abspath(str(path)))
+        while True:
+            identity = self._identity(folder)
+            if identity is not None:
+                found.setdefault(identity, folder)
+            parent = os.path.dirname(folder)
+            if parent == folder:
+                return found
+            folder = parent
+
+    def of(self, path):
+        """``{folder spelling: path spelled under it}``; ``""`` is ``path`` itself."""
+        path = os.path.abspath(str(path))
+        spelled = {"": path}
+        ancestors = self._ancestors(path)
+        for folder, identity in self._folders:
+            inside = ancestors.get(identity)
+            if inside is not None:
+                spelled.setdefault(
+                    folder, os.path.join(folder, os.path.relpath(path, inside))
+                )
+        return spelled
+
+    @property
+    def listed_roots(self):
+        """The listed roots (the output sources), in order."""
+        return [root for root, _identity in self._listed]
+
+    def category_key(self, path):
+        """*path*'s image category file key (#332), or None outside every source."""
+        return image_category_key(self.listed_roots, path)
+
+    def favorites(self, path):
+        """``{listed root: path below it (POSIX)}`` for each listed root holding it."""
+        path = os.path.abspath(str(path))
+        ancestors = self._ancestors(path)
+        held = {}
+        for root, identity in self._listed:
+            inside = ancestors.get(identity)
+            if inside is not None:
+                held[root] = Path(os.path.relpath(path, inside)).as_posix()
+        return held
+
+
+def copy_move(target, kept_stat, spellings):
+    """What ``move_copy_records`` moves for one removal.
+
+    A record found under a folder spelling moves to the kept copy under
+    the same spelling when that folder holds it too, else to its real
+    path. Its favorite and its collection memberships move under every
+    listed root holding the file.
+    """
+    removed = spellings.of(target["abs"])
+    kept = spellings.of(target["keeper_abs"])
+    targets = {
+        spelled: kept.get(folder, target["keeper_abs"])
+        for folder, spelled in removed.items()
+    }
+    kept_favorites = spellings.favorites(target["keeper_abs"])
+    favorites = []
+    for root, relative in spellings.favorites(target["abs"]).items():
+        kept_root, kept_rel = (
+            (root, kept_favorites[root])
+            if root in kept_favorites
+            else (target["keeper_root"], target["keeper_rel"])
+        )
+        favorites.append(
+            (
+                _favorite_root_key(root),
+                _favorite_relative_key(relative),
+                _favorite_root_key(kept_root),
+                _favorite_relative_key(kept_rel),
+            )
+        )
+    return {
+        "removed": list(removed.values()),
+        "kept": list(kept.values()),
+        "targets": targets,
+        "kept_path": target["keeper_abs"],
+        "favorites": favorites,
+        "categories": _category_pairs(target, spellings),
+        "removed_locator": target["locator"],
+        "kept_locator": target["keeper_locator"],
+        "sha256": target["hash"],
+        "kept_size": kept_stat.st_size,
+        "kept_mtime_ns": kept_stat.st_mtime_ns,
+    }
+
+
+def _category_pairs(target, spellings):
+    """``[(removed file key, kept file key)]`` for the image category (#332).
+
+    From the two absolute paths, not the favorites tuples: those follow the
+    listed root, which differs from the file key when sources nest.
+    """
+    removed = spellings.category_key(target["abs"])
+    kept = spellings.category_key(target["keeper_abs"])
+    if removed is None or kept is None:
+        return []
+    return [(removed, kept)]

py/api/gallery_listing.py

@@ -0,0 +1,337 @@
+"""One shared listing per output root, and the pure helpers behind it (#348).
+
+The API keeps one ``RootListing`` per output root (``_root_listings`` in
+``py/api/__init__.py``). The record is read and written only on the event
+loop; the walk itself, the comparison with the previous listing and both
+folder sets run in the executor through ``walk_root``. Routes live in
+``py/api/images.py`` (``_listing``); this module has no aiohttp import so
+tests can exercise it without a server.
+
+A settled output the image monitor reports is added to its root's stored
+listing without a walk (``output_settled``, ``insert_entry``), in the place
+a walk would list it. When that cannot be exact, the root is marked dirty
+(``mark_root_dirty``) instead; a dirty root gets one background walk, no
+sooner than ``rewalk_interval`` after its previous walk started (PR 2).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from bisect import bisect_left
+from dataclasses import dataclass, field
+from pathlib import PurePath
+from typing import Callable, NamedTuple
+
+# A listing older than this (seconds) is served as it is, and one background
+# walk replaces it. Read as ``gallery_listing.REFRESH_AFTER`` so tests can
+# patch it.
+REFRESH_AFTER = 30
+
+# The stale read is served at once; its refresh walk starts this many seconds
+# later, so the walk does not share the GIL with the page's own requests
+# (#348 live test: reads during the walk took 2.2 s instead of 0.07 s).
+REFRESH_DELAY = 1.0
+
+# No background walk of a root starts sooner than rewalk_interval(last walk's
+# duration) after its previous walk started: FACTOR times that duration, at
+# least FLOOR and at most CAP seconds. Walks up to CAP / FACTOR (3 s) then
+# hold the GIL at most ~20% of the time; the cap keeps a new output listed
+# within ~20 s, at the cost of a higher share (back to back from 15 s) on
+# slower walks. Read through this module so tests can patch them.
+REWALK_FLOOR = 5
+REWALK_FACTOR = 5
+REWALK_CAP = 15
+
+# A deferred walk is never armed sooner than this: no zero-delay loop.
+MIN_REWALK_DELAY = 0.5
+
+# A reader with no listing gives up (500) after this many walks in a row were
+# discarded by invalidations made while they ran.
+MAX_DISCARDED_WALKS = 3
+
+WALK_STOPPED = "The listing walk was stopped"
+
+STORED = "stored"
+DISCARDED = "discarded"
+FAILED = "failed"
+
+
+class WalkOutcome(NamedTuple):
+    """What one walk ended with; a walk's flight future resolves to this."""
+
+    kind: str
+    error: BaseException | None = None
+
+    @classmethod
+    def stored(cls) -> "WalkOutcome":
+        return cls(STORED)
+
+    @classmethod
+    def discarded(cls) -> "WalkOutcome":
+        return cls(DISCARDED)
+
+    @classmethod
+    def failed(cls, error: BaseException) -> "WalkOutcome":
+        return cls(FAILED, error)
+
+
+@dataclass
+class RootListing:
+    """The shared listing of one output root.
+
+    A record is never deleted: ``drop()`` clears the listing and bumps
+    ``generation``, so a walk that started before a drop can never be stored
+    after it.
+    """
+
+    # (path, mtime) newest first, as _scan_gallery_files_sync returns it.
+    files: list | None = None
+    folders: frozenset | None = None
+    folders_with_ancestors: frozenset | None = None
+    # Monotonic time the last walk finished, stored or failed.
+    walked_at: float | None = None
+    # Number of the last walk started; a running walk is always that one.
+    walk_seq: int = 0
+    # Number of the walk whose files are stored (Rescan waits on it).
+    stored_seq: int = 0
+    generation: int = 0
+    # The running walk's future (resolves to a WalkOutcome), or None.
+    flight: asyncio.Future | None = None
+    # True while walks keep failing: one warning per streak.
+    failing: bool = False
+    # The one deferred background walk not started yet (loop.call_later), or
+    # None: a stale refresh or a dirty root's rewalk, whichever is due first.
+    rewalk_handle: asyncio.TimerHandle | None = None
+    # normcase(realpath(root)), computed by the walk in the executor: monitor
+    # paths are matched against it without file I/O on the loop.
+    resolved_root: str | None = None
+    # Monotonic start and duration of the last walk (rewalk_interval).
+    last_started_at: float | None = None
+    last_duration: float | None = None
+    # A settled output was reported under this root; cleared when a walk starts.
+    dirty: bool = False
+    # Dropped because the output roots changed: no background walk until a
+    # reader (or the warm-up) asks for this root again. The monitor keeps
+    # watching a removed root until ComfyUI restarts.
+    retired: bool = False
+    # Relative paths of settled outputs waiting to be added to ``files``
+    # without a walk, and the one task adding them (``output_settled``).
+    pending_adds: list = field(default_factory=list)
+    add_task: asyncio.Task | None = None
+    # Settled outputs added to the stored listing since the last walk stored.
+    patched: int = 0
+
+    def drop(self) -> None:
+        self.files = None
+        self.folders = None
+        self.folders_with_ancestors = None
+        self.walked_at = None
+        self.pending_adds = []
+        self.patched = 0
+        self.generation += 1
+
+
+def folder_sets(root, files) -> tuple[frozenset, frozenset]:
+    """``(folders, folders_with_ancestors)`` of the files listed under *root*.
+
+    Folders keep the spelling ``/gallery/subfolders`` has always answered:
+    ``str(relative.parent)`` (native separators) for the folders holding
+    files, plus every ancestor joined by ``/``.
+    """
+    # From the path strings: relative_to per file was half of a large walk.
+    sep, prefix = _root_prefix(root)
+    folders = set()
+    for path, _mtime in files:
+        rel_dir = _folder_of(root, path, sep, prefix)
+        if rel_dir:
+            folders.add(rel_dir)
+    with_ancestors = set(folders)
+    for folder in folders:
+        with_ancestors.update(_ancestors(folder))
+    return frozenset(folders), frozenset(with_ancestors)
+
+
+def add_folder(root, path, folders, with_ancestors) -> tuple[frozenset, frozenset]:
+    """Both folder sets once *path* is listed too (the same sets when it adds none)."""
+    sep, prefix = _root_prefix(root)
+    rel_dir = _folder_of(root, path, sep, prefix)
+    if not rel_dir or rel_dir in folders:
+        return folders, with_ancestors
+    return folders | {rel_dir}, with_ancestors | {rel_dir, *_ancestors(rel_dir)}
+
+
+def _root_prefix(root) -> tuple[str, str]:
+    # The separator is the path flavour's own (tests use PureWindowsPath).
+    sep = str(type(root)("a", "b"))[1]
+    prefix = str(root)
+    if not prefix.endswith(sep):
+        prefix += sep
+    return sep, prefix
+
+
+def _folder_of(root, path, sep, prefix) -> str:
+    """The folder ``/gallery/subfolders`` names for *path*; "" for the root."""
+    text = str(path)
+    if text.startswith(prefix):
+        rel_dir = text[len(prefix) :].rpartition(sep)[0]
+    else:
+        rel_dir = str(path.relative_to(root).parent)
+    return "" if rel_dir == "." else rel_dir
+
+
+def _ancestors(folder: str) -> list[str]:
+    parts = folder.replace("\\", "/").split("/")
+    return ["/".join(parts[:i]) for i in range(1, len(parts))]
+
+
+def relative_part(resolved_root: str, path, pathmod=os.path) -> str | None:
+    """The part of the resolved *path* below *resolved_root*, as a walk spells it.
+
+    Kept in *path*'s own case (``realpath`` gives a file's on-disk case on
+    Windows) with the native separator. None when *path* is not below the
+    root, or when the case folding changed the length so the slice would
+    not be exact.
+    """
+    sep = pathmod.sep
+    text = str(path)
+    if sep == "\\":
+        text = _plain(text.replace("/", sep), pathmod)
+    folded = pathmod.normcase(text)
+    prefix = _plain(resolved_root, pathmod)
+    if not prefix.endswith(sep):
+        prefix += sep
+    if len(folded) != len(text) or not folded.startswith(prefix):
+        return None
+    relative = text[len(prefix) :]
+    if any(part in ("", ".", "..") for part in relative.split(sep)):
+        return None
+    return relative
+
+
+def insert_entry(files: list, entry: tuple, max_files: int) -> list | None:
+    """A new listing with *entry* ``(path, mtime)`` where a walk would list it.
+
+    *files* is newest first; an entry for the same path is replaced. Returns
+    *files* itself when it already holds *entry*, and None when the place is
+    not exact: another file has the same mtime (a walk orders those by its
+    own visit order), or the listing is at the walk's *max_files* cap.
+    Blocking (one pass over *files*): run it in the executor.
+    """
+    path, mtime = entry
+    text = str(path)
+    kept = files
+    for index, (listed, listed_mtime) in enumerate(files):
+        if str(listed) == text:
+            if listed_mtime == mtime:
+                return files
+            kept = files[:index] + files[index + 1 :]
+            break
+    else:
+        if len(files) >= max_files:
+            return None
+    index = bisect_left(kept, -mtime, key=lambda item: -item[1])
+    if index < len(kept) and kept[index][1] == mtime:
+        return None
+    return [*kept[:index], entry, *kept[index:]]
+
+
+# A listing that differs from the cached one by more entries than this is
+# rebuilt in full by the workspace caches (listing_delta).
+MAX_DELTA = 256
+
+
+def listing_delta(old: list, new: list, limit: int = MAX_DELTA):
+    """``(added, removed)`` entries from *old* to *new*, or None past *limit*.
+
+    A listing patched by ``insert_entry`` shares every unchanged entry object
+    with the one it came from, so entries are told apart by identity: a walk
+    builds new tuples, shares none and is rebuilt in full. *old* must still
+    be alive (the
+    caller's cache holds it), so no identity was reused.
+    """
+    if abs(len(new) - len(old)) > limit:
+        return None
+    old_ids = set(map(id, old))
+    added = []
+    for entry in new:
+        if id(entry) not in old_ids:
+            added.append(entry)
+            if len(added) > limit:
+                return None
+    if new and len(added) == len(new):
+        return None  # nothing shared: a walk
+    if len(old) + len(added) == len(new):
+        return added, []
+    new_ids = set(map(id, new))
+    return added, [entry for entry in old if id(entry) not in new_ids]
+
+
+def rewalk_interval(last_duration) -> float:
+    """Seconds from a walk's start to the next background walk of its root."""
+    scaled = REWALK_FACTOR * (last_duration or 0.0)
+    return min(REWALK_CAP, max(REWALK_FLOOR, scaled))
+
+
+def root_contains(resolved_root: str, path, pathmod=os.path) -> bool:
+    """True when the resolved *path* lies under *resolved_root* (no file I/O).
+
+    *resolved_root* is already ``normcase``d; *path* is folded the same way,
+    so Windows matches case-insensitively and with either separator. The
+    match ends at a separator: ``/out2/x.png`` is not under ``/out``.
+    """
+    target = _plain(pathmod.normcase(str(path)), pathmod)
+    prefix = _plain(resolved_root, pathmod)
+    if not prefix.endswith(pathmod.sep):
+        prefix += pathmod.sep
+    return target.startswith(prefix)
+
+
+def _plain(path: str, pathmod) -> str:
+    """*path* (normcased or not) without a Windows extended-length prefix.
+
+    ``realpath`` may keep ``\\\\?\\`` on a file past MAX_PATH while its shorter
+    root resolves without it.
+    """
+    if pathmod.sep != "\\":
+        return path
+    if path[:8].lower() == "\\\\?\\unc\\":
+        return "\\\\" + path[8:]
+    if path.startswith("\\\\?\\"):
+        return path[4:]
+    return path
+
+
+def resolve_root(root) -> str:
+    """``normcase(realpath(root))``, the spelling monitor paths are matched in.
+
+    File I/O (realpath follows links): call it in the executor.
+    """
+    return os.path.normcase(os.path.realpath(str(root)))
+
+
+def same_listing(old, new) -> bool:
+    """True when two listings hold the same ``(path, mtime)`` pairs in order."""
+    if old is None or new is None:
+        return False
+    return old is new or old == new
+
+
+def walk_root(
+    scan_fn: Callable[[PurePath], list], root, previous_files
+) -> tuple[list, frozenset, frozenset, str]:
+    """The single executor job of a walk.
+
+    Returns ``(files, folders, with_ancestors, resolved_root)``.
+
+    Opens the root first, so a missing or unreadable root raises ``OSError``
+    instead of listing nothing. Returns *previous_files* itself when nothing
+    changed, which keeps the identity-keyed caches built on it warm.
+    """
+    with os.scandir(root):
+        pass
+    files = scan_fn(root)
+    if same_listing(previous_files, files):
+        files = previous_files
+    folders, with_ancestors = folder_sets(root, files)
+    return files, folders, with_ancestors, resolve_root(root)

py/api/history.py

@@ -5,6 +5,7 @@ import base64
 import binascii
 from functools import wraps
 import json
+import os
 import re
 import sqlite3
 from typing import Callable
@@ -20,16 +21,23 @@ from .history_current import (
     _OpenedCurrentMedia,
     _current_media_response,
     _prepare_current_media,
+    accepts_jxl,
     _reauthorize_current_target,
     candidate_locator,
     resolve_current_candidate,
     snapshot_current_output_scope,
 )
-from .inspection_service import CurrentFileChanged, inspect_current_file
+from .inspection_service import (
+    CurrentFileChanged,
+    image_tags_reader,
+    inspect_current_file,
+)
+from .images import image_category_key
 
 try:
     from ...database.execution_history import HistoryEpochChanged
     from ...database.history_gallery import read_history_gallery
+    from ...database.run_attribution import recorded_output_lookup
     from ...database.history_reads import (
         HistoryCaptureChanged,
         OutputPosition,
@@ -37,10 +45,15 @@ try:
         WindowPosition,
     )
     from ...utils.history_read_fence import HistoryReadFence
-    from ...utils.inspection_response import encode_history_inspection_response
+    from ...utils.inspection_response import (
+        OWN_CATEGORY_UNREAD,
+        apply_own_category,
+        encode_history_inspection_response,
+    )
 except ImportError:
     from database.execution_history import HistoryEpochChanged
     from database.history_gallery import read_history_gallery
+    from database.run_attribution import recorded_output_lookup
     from database.history_reads import (
         HistoryCaptureChanged,
         OutputPosition,
@@ -48,7 +61,11 @@ except ImportError:
         WindowPosition,
     )
     from utils.history_read_fence import HistoryReadFence
-    from utils.inspection_response import encode_history_inspection_response
+    from utils.inspection_response import (
+        OWN_CATEGORY_UNREAD,
+        apply_own_category,
+        encode_history_inspection_response,
+    )
 
 
 _RUN_MAX_BYTES = 8 * 1024 * 1024
@@ -142,6 +159,18 @@ def _query(request: web.Request) -> None:
         raise ValueError("Duplicate history query parameter")
 
 
+def _jxl_flag(request: web.Request) -> bool:
+    """The cover's ``jxl`` flag (#317): ``1`` when the workspace decoded a
+    JPEG XL image (a browser may not list ``image/jxl`` in Accept), ``0`` or
+    absent otherwise; anything else is a bad request."""
+    value = request.query.get("jxl")
+    if value is None:
+        return False
+    if value not in ("0", "1"):
+        raise ValueError("Invalid jxl flag")
+    return value == "1"
+
+
 def _page_limit(request: web.Request) -> int:
     limit = _decimal(request.query.get("limit", "40"), minimum=1)
     if limit > 100:
@@ -662,6 +691,32 @@ class HistoryRoutesMixin:
     ) -> web.Response:
         return await self._history_current(request, "inspection")
 
+    def _history_own_category(self, reference: dict, scope):
+        """The recorded output's own category (#332), inside the snapshot.
+
+        The path is the symlink-resolved output folder plus the receipt's
+        subfolder and filename, keyed by ``image_category_key`` over the
+        configured output sources as Images keys it. A temp output or a path
+        no source holds has none. A failed read is OWN_CATEGORY_UNREAD.
+        """
+        if (
+            reference.get("folder_type") != "output"
+            or scope.canonical_output is None
+            or candidate_locator(reference, scope) is None
+        ):
+            return None
+        path = os.path.join(
+            str(scope.canonical_output), reference["subfolder"], reference["filename"]
+        )
+        try:
+            key = image_category_key(self._get_all_output_dirs(), path)
+            return None if key is None else self.db.image_category(*key)
+        except Exception:
+            self.logger.warning(
+                "History image category could not be read", exc_info=True
+            )
+            return OWN_CATEGORY_UNREAD
+
     async def _history_current(
         self, request: web.Request, kind: str
     ) -> web.StreamResponse:
@@ -686,6 +741,7 @@ class HistoryRoutesMixin:
                     if scope.current_scope != current_scope:
                         raise _current_stale()
                     result = db.get_history_occurrence(run_id, node, ordinal)
+                    own = None
                     if result is not None:
                         receipt = result["receipt"]
                         path = candidate_locator(receipt["reference"], scope)
@@ -693,6 +749,10 @@ class HistoryRoutesMixin:
                             db.model.get_connection(),
                             (path,) if path is not None else (),
                         )
+                        if kind == "inspection":
+                            own = self._history_own_category(
+                                receipt["reference"], scope
+                            )
             finally:
                 db.model.close()
             if result is None:
@@ -711,9 +771,27 @@ class HistoryRoutesMixin:
                         resolved.canonical_target,
                         source_label=resolved.public["source_label"],
                         saved=saved.get("saved"),
-                        library=saved.get(
-                            "library",
-                            {"status": "unavailable" if saved_unavailable else "none"},
+                        library=apply_own_category(
+                            saved.get(
+                                "library",
+                                {
+                                    "status": (
+                                        "unavailable" if saved_unavailable else "none"
+                                    )
+                                },
+                            ),
+                            own,
+                        ),
+                        # The file's own image tags, keyed as the Tag dialog
+                        # keys them: the normcased realpath (#457).
+                        image_tags=image_tags_reader(
+                            db,
+                            str(resolved.canonical_target),
+                            self.logger,
+                            close=True,
+                        ),
+                        recorded=recorded_output_lookup(
+                            db, close=True, reference=receipt["reference"]
                         ),
                     )
                     _reauthorize_current_target(resolved, scope)
@@ -734,7 +812,10 @@ class HistoryRoutesMixin:
                         raise HistoryReadError(422, "response-limit") from None
                 else:
                     prepared = _prepare_current_media(
-                        resolved, scope, thumbnail=kind == "thumbnail"
+                        resolved,
+                        scope,
+                        thumbnail=kind == "thumbnail",
+                        **media_options,
                     )
                 _check_current(scope, self, fence, epoch, generation)
                 return prepared
@@ -761,25 +842,45 @@ class HistoryRoutesMixin:
                     _check_current(scope, self, fence, epoch, generation)
                 raise
 
-        pending = asyncio.create_task(self._run_in_executor(project))
-        try:
-            prepared = await asyncio.shield(pending)
-        except asyncio.CancelledError:
-            pending.add_done_callback(_close_prepared_result)
-            raise
-        except (
-            HistoryReadError,
-            HistoryEpochChanged,
-            sqlite3.Error,
-            OSError,
-            ValueError,
-        ):
-            raise
-        except Exception as error:
-            self.logger.warning(
-                "Current history media unavailable (%s)", type(error).__name__
-            )
-            raise _current_unavailable("temporary_unavailable") from None
+        async def execute():
+            pending = asyncio.create_task(self._run_in_executor(project))
+            try:
+                return await asyncio.shield(pending)
+            except asyncio.CancelledError:
+                pending.add_done_callback(_close_prepared_result)
+                raise
+            except (
+                HistoryReadError,
+                HistoryEpochChanged,
+                sqlite3.Error,
+                OSError,
+                ValueError,
+            ):
+                raise
+            except Exception as error:
+                self.logger.warning(
+                    "Current history media unavailable (%s)", type(error).__name__
+                )
+                raise _current_unavailable("temporary_unavailable") from None
+
+        # A missing preview (#249) is noted on the first read, made through
+        # the serve route's admission and single flight, then read again. A
+        # video whose preview cannot be made is the placeholder frame.
+        needed = []
+        media_options = {}
+        if kind == "thumbnail":
+            media_options = {
+                "ensure_preview": lambda *preview: needed.append(preview),
+                "placeholder": self._video_placeholder_bytes(),
+                "accepts_jxl": _jxl_flag(request) or accepts_jxl(request),
+            }
+        prepared = await execute()
+        if needed:
+            if not isinstance(prepared, bytes):
+                prepared.close()
+            await self._admit_preview(*needed[0])
+            media_options["ensure_preview"] = None
+            prepared = await execute()
         if isinstance(prepared, bytes):
             return _success_response(prepared)
         return await _current_media_response(request, prepared)

py/api/history_current.py

@@ -5,6 +5,7 @@ from dataclasses import dataclass
 import errno
 import hashlib
 import hmac
+import io
 import json
 import mimetypes
 import os
@@ -14,7 +15,10 @@ from typing import BinaryIO
 
 from aiohttp import web
 
+from . import thumbnail_settings
 from .images import _canonical, _public_path, SERVABLE_EXTENSIONS, VIDEO_EXTENSIONS
+from .images import in_trash_below_roots
+from .images import existing_media_thumbnail, media_thumbnail_rel
 
 try:
     from ...utils.execution_outputs import (
@@ -195,6 +199,9 @@ def resolve_current_candidate(
         if namespace is None or not namespace.is_dir():
             return _unavailable("root_unavailable")
         target = _canonical(lexical)
+        if in_trash_below_roots(target, [namespace]):
+            # Every kind and the outputs listing come through here (#543).
+            return _unavailable("missing")
         if not _authorized(target, namespace, scope):
             return _unavailable("not_authorized")
         if target.suffix.lower() not in SERVABLE_EXTENSIONS:
@@ -238,14 +245,23 @@ def resolve_current_candidate(
 
 
 def _open_current_target(
-    target: Path, namespace: Path, scope: CurrentOutputScope
+    target: Path, namespace: Path, scope: CurrentOutputScope, preview: bool = False
 ) -> BinaryIO:
-    """Open only the canonical target and verify authority against the actual handle."""
+    """Open only the canonical target and verify authority against the actual handle.
+
+    A *preview* (a file under ``thumbnails/``) may also be JPEG XL (#317)."""
     if _canonical(target) != target:
         raise CurrentLocationUnavailable("not_authorized", stale=True)
+    if in_trash_below_roots(target, [namespace]):
+        # Also at the open: the cover chooses its preview after the candidate
+        # check, and a preview can be a link into a trash folder (#543).
+        raise CurrentLocationUnavailable("missing")
     if not _authorized(target, namespace, scope):
         raise CurrentLocationUnavailable("not_authorized")
-    if target.suffix.lower() not in SERVABLE_EXTENSIONS:
+    suffix = target.suffix.lower()
+    if suffix not in SERVABLE_EXTENSIONS and not (
+        preview and suffix in thumbnail_settings.THUMBNAIL_EXTENSIONS
+    ):
         raise CurrentLocationUnavailable("unsupported_media")
     flags = (
         os.O_RDONLY
@@ -309,34 +325,107 @@ class _OpenedCurrentMedia:
         self.handle.close()
 
 
-def _prepare_current_media(
-    resolved: ResolvedCurrentCandidate, scope: CurrentOutputScope, *, thumbnail: bool
-) -> _OpenedCurrentMedia:
+def _preview_writable(root: Path, preview: Path) -> bool:
+    """A preview may be written: ``thumbnails`` and every link segment of the
+    preview's folder resolve inside *root* (the serve route's rule, #249)."""
+    folder = root / "thumbnails"
+    return _canonical(folder).is_relative_to(_canonical(root)) and _canonical(
+        preview
+    ).is_relative_to(_canonical(folder))
+
+
+def _is_jxl(path) -> bool:
+    return Path(path).suffix.lower() == ".jxl"
+
+
+def _preview_without_jxl(root: Path, relative: Path) -> Path | None:
+    """The first existing preview *relative* owns that is not JPEG XL."""
+    return next(
+        (
+            path
+            for path in thumbnail_settings.owned_thumbnails(root, relative)
+            if not _is_jxl(path)
+        ),
+        None,
+    )
+
+
+def accepts_jxl(request) -> bool:
+    """Whether the request's ``Accept`` lists ``image/jxl``."""
+    accept = request.headers.get("Accept", "")
+    return any(
+        part.split(";", 1)[0].strip().lower() == "image/jxl"
+        for part in accept.split(",")
+    )
+
+
+def _placeholder_media(placeholder: bytes) -> "_OpenedCurrentMedia":
+    return _OpenedCurrentMedia(io.BytesIO(placeholder), len(placeholder), "image/jpeg")
+
+
+def _cover_target(resolved, ensure_preview, placeholder, accepts_jxl):
+    """What the cover opens: a preview path, the placeholder media, or None
+    for the original (see ``_prepare_current_media``)."""
     target = resolved.canonical_target
+    relative = target.relative_to(resolved.gallery_root)
+    is_video = target.suffix.lower() in VIDEO_EXTENSIONS
+    # #317: any preview the file owns (the current format first); a missing
+    # one is made under the current format's name.
+    preview = existing_media_thumbnail(resolved.gallery_root, relative)
+    if preview is not None and not accepts_jxl and _is_jxl(preview):
+        preview = _preview_without_jxl(resolved.gallery_root, relative)
+        if preview is None:
+            if is_video and placeholder is not None:
+                return _placeholder_media(placeholder)
+            return None
+        return _canonical(preview)
+    if preview is not None:
+        return _canonical(preview)
+    preview = resolved.gallery_root / media_thumbnail_rel(relative)
+    if ensure_preview is not None and _preview_writable(resolved.gallery_root, preview):
+        ensure_preview(target, preview, is_video)
+    if is_video and placeholder is not None:
+        return _placeholder_media(placeholder)
+    return None
+
+
+def _prepare_current_media(
+    resolved: ResolvedCurrentCandidate,
+    scope: CurrentOutputScope,
+    *,
+    thumbnail: bool,
+    ensure_preview=None,
+    placeholder: bytes | None = None,
+    accepts_jxl: bool = True,
+) -> _OpenedCurrentMedia:
+    """Open the current file, or its preview for ``thumbnail``.
+
+    A missing preview that may be written is handed to
+    ``ensure_preview(src, dst, is_video)`` (the route makes it and reads
+    again, #249). Without one an image's original is opened and a video is
+    the ``placeholder`` JPEG, never the video inside an ``<img>``.
+
+    The cover's URL names no format, so a JPEG XL preview goes only to a
+    browser whose ``Accept`` lists ``image/jxl`` (*accepts_jxl*, review M3).
+    Another one gets the file's preview in another format when there is one,
+    else the original image or a video's placeholder: never a broken image.
+    """
+    target = resolved.canonical_target
+    is_preview = False
     if thumbnail:
-        relative = target.relative_to(resolved.gallery_root)
-        extension = (
-            ".jpg"
-            if target.suffix.lower() in VIDEO_EXTENSIONS
-            else target.suffix.lower()
-        )
-        preview = (
-            resolved.gallery_root
-            / "thumbnails"
-            / f"{relative.with_suffix('').as_posix()}_thumb{extension}"
-        )
-        try:
-            preview.stat()
-        except FileNotFoundError:
-            pass
-        else:
-            target = _canonical(preview)
-    handle = _open_current_target(target, resolved.namespace, scope)
+        chosen = _cover_target(resolved, ensure_preview, placeholder, accepts_jxl)
+        if isinstance(chosen, _OpenedCurrentMedia):
+            return chosen
+        if chosen is not None:
+            target, is_preview = chosen, True
+    handle = _open_current_target(target, resolved.namespace, scope, is_preview)
     try:
         return _OpenedCurrentMedia(
             handle,
             os.fstat(handle.fileno()).st_size,
-            mimetypes.guess_type(target.name)[0] or "application/octet-stream",
+            thumbnail_settings.content_type(target)
+            or mimetypes.guess_type(target.name)[0]
+            or "application/octet-stream",
         )
     except BaseException:
         handle.close()

py/api/image_categories.py

@@ -0,0 +1,376 @@
+"""Image category routes (#332): an image file's own category.
+
+``PUT /prompt_manager/images/category`` sets the own category of up to 500
+listed files; ``DELETE`` removes it (listed or not, so a missing file's
+category can still be removed); ``GET /prompt_manager/images/categories``
+lists the own-category values of files the sources list. Targets are the
+Images listing identity (``{root_index, root_scope, path}``, checked by
+``media_target``) of an output source; LoRA folders are not sources here.
+Instead of ``items`` a request may name ``filter``, the Images listing's own
+query ("Select all N matching", #392), resolved now up to 500 images.
+Each file is stored under its file key (``images.image_category_key``).
+These routes never read or write prompts, recipes or image links.
+"""
+
+import os
+from pathlib import Path
+
+from aiohttp import web
+
+from .image_tag_jobs import MAX_JOB_TARGETS, _parse_targets
+from .images import (
+    _favorite_relative_key,
+    _favorite_root_key,
+    _forbidden,
+    _json_body_or_error,
+    image_category_key,
+)
+from .prompts import bad_request
+
+try:
+    from ...database.image_categories import MAX_CATEGORY_LENGTH, MAX_KEY_LENGTH
+    from ...database.models import normalize_image_path
+    from ...utils.validators import validate_category
+except ImportError:
+    from database.image_categories import MAX_CATEGORY_LENGTH, MAX_KEY_LENGTH
+    from database.models import normalize_image_path
+    from utils.validators import validate_category
+
+CATEGORY_MEDIA_MESSAGE = "Only media files can have a category"
+_PUT_FIELDS = frozenset({"category"})
+_DELETE_FIELDS = frozenset()
+TOO_MANY = f"Select at most {MAX_JOB_TARGETS} images."
+SOURCE_CHANGED_MESSAGE = "The sources changed. Refresh the images and try again."
+WRITE_FAILED = "The category could not be changed."
+READ_FAILED = "Image categories could not be read."
+
+
+def _coded(status: int, code: str, message: str) -> web.Response:
+    return web.json_response(
+        {"success": False, "code": code, "error": message},
+        status=status,
+        headers={"Cache-Control": "no-store"},
+    )
+
+
+def _category_value(raw) -> tuple[str | None, web.Response | None]:
+    """The stripped category, or a 400 (an empty value is a 400: use DELETE)."""
+    if not isinstance(raw, str):
+        return None, bad_request("category must be a string")
+    value = raw.strip()
+    if not value:
+        return None, bad_request("category must not be empty")
+    if len(value) > MAX_CATEGORY_LENGTH:
+        return None, bad_request(
+            f"Category cannot exceed {MAX_CATEGORY_LENGTH} characters"
+        )
+    try:
+        value.encode("utf-8")
+        validate_category(value)
+    except UnicodeEncodeError:
+        return None, bad_request("category must be valid text")
+    except ValueError as exc:
+        return None, bad_request(str(exc))
+    return value, None
+
+
+class _KeyRefused(Exception):
+    """A target has no storable file key (outside every source, or too long)."""
+
+    def __init__(self, message: str = "Access denied"):
+        super().__init__(message)
+        self.message = message
+
+
+def _file_key(roots, root, relative) -> tuple[str, str]:
+    """The target's file key (blocking: resolves the sources)."""
+    key = image_category_key(roots, Path(root) / relative)
+    if key is None:
+        raise _KeyRefused()
+    if max(len(key[0]), len(key[1])) > MAX_KEY_LENGTH:
+        raise _KeyRefused("Path is too long")
+    return key
+
+
+def _nested_sources(root: Path, sources) -> list[tuple[str, str]]:
+    """``[(relative key prefix, source root key)]`` of sources inside *root*.
+
+    Deepest first, so the first prefix that matches is the file's source.
+    """
+    base = os.path.join(os.path.normcase(os.path.realpath(str(root))), "")
+    nested = []
+    for source in sources:
+        real = os.path.realpath(str(source))
+        if not os.path.normcase(real).startswith(base):
+            continue
+        relative = os.path.relpath(real, os.path.realpath(str(root)))
+        nested.append(
+            (
+                _favorite_relative_key(relative.replace(os.sep, "/")),
+                _favorite_root_key(source),
+            )
+        )
+    nested.sort(key=lambda item: item[0].count("/"), reverse=True)
+    return nested
+
+
+def _source_prefix(relative_key: str, nested) -> str | None:
+    """The prefix of the deepest nested source holding *relative_key*."""
+    for prefix, _source_key in nested:
+        if relative_key.startswith(prefix + "/"):
+            return prefix
+    return None
+
+
+class ImageCategoryRoutesMixin:
+    """Routes for images' own categories (#332)."""
+
+    def _register_image_category_routes(self, routes):
+        # Fixed paths beside DELETE /prompt_manager/images/{image_id}, as the
+        # favorite routes are. Registered after it, they still win: aiohttp
+        # (3.10+) looks fixed paths up before dynamic resources
+        # (test_delete_category_never_reaches_delete_image).
+        @routes.put("/prompt_manager/images/category")
+        async def set_image_category_route(request):
+            return await self.set_image_category(request)
+
+        @routes.delete("/prompt_manager/images/category")
+        async def clear_image_category_route(request):
+            return await self.clear_image_category(request)
+
+        @routes.get("/prompt_manager/images/categories")
+        async def list_image_categories_route(request):
+            return await self.list_image_categories(request)
+
+    # -- reading ---------------------------------------------------------------
+
+    def _own_category_map(self, sources, listings) -> dict:
+        """``{normalized listed path: own category}`` for the listed files (blocking).
+
+        One primary-key range read per source; each stored row is matched to
+        the listing of every selected root that holds its source, so the work
+        follows the stored rows, not the files. A file under a nested source
+        is keyed by the deepest one (``image_category_key``). A row whose file
+        is gone matches nothing. A failed read raises.
+        """
+        sources = list(sources)
+        rows = self.db.own_categories_of(_favorite_root_key(s) for s in sources)
+        if not rows:
+            return {}
+        by_source: dict[str, dict[str, str]] = {}
+        for (root_key, relative), category in rows.items():
+            by_source.setdefault(root_key, {})[relative] = category
+        found = {}
+        for _index, root, files in listings:
+            listing = self._favorite_listing(root, files)
+            root_key = _favorite_root_key(root)
+            # normalize_image_path(root / listed) without a call per file: the
+            # listed spellings are clean relative POSIX paths. Windows abspath
+            # also drops a component's trailing dots and spaces, so it keeps
+            # the call there.
+            base = os.path.join(normalize_image_path(str(root)), "")
+
+            def normalized(listed, base=base, root=root):
+                if os.name == "nt":
+                    return normalize_image_path(str(root / listed))
+                return base + os.path.normcase(listed.replace("/", os.sep))
+
+            nested = [
+                item for item in _nested_sources(root, sources) if item[1] != root_key
+            ]
+            for relative, category in by_source.get(root_key, {}).items():
+                listed = listing.get(relative)
+                if listed is not None and _source_prefix(relative, nested) is None:
+                    found[normalized(listed)] = category
+            for prefix, source_key in nested:
+                for relative, category in by_source.get(source_key, {}).items():
+                    full = f"{prefix}/{relative}"
+                    listed = listing.get(full)
+                    if listed is not None and _source_prefix(full, nested) == prefix:
+                        found[normalized(listed)] = category
+        return found
+
+    async def _source_listings(self, roots) -> list[tuple]:
+        return [
+            (index, root, await self._get_gallery_files(root))
+            for index, root in enumerate(roots)
+        ]
+
+    async def list_image_categories(self, request):
+        """GET: the own-category values of listed files, sorted."""
+        try:
+            roots = list(self._get_all_output_dirs())
+            listings = await self._source_listings(roots)
+            found = await self._run_in_executor(self._own_category_map, roots, listings)
+        except Exception:
+            self.logger.exception("Image categories read failed")
+            return _coded(500, "categories-failed", READ_FAILED)
+        return web.json_response(
+            {"success": True, "categories": sorted(set(found.values()))},
+            headers={"Cache-Control": "no-store"},
+        )
+
+    # -- writing ---------------------------------------------------------------
+
+    async def _filter_targets(self, raw):
+        """``(targets, None)`` for an Images listing filter, else ``(None, error)``.
+
+        "Select all N matching" (#392) sends the listing's own query; it is
+        resolved now, up to 500 images (a 400 ``too-many-images`` above).
+        """
+        from .images import ListingSourceChanged, parse_listing_filter
+
+        try:
+            selection = await self.resolve_listing_filter(
+                parse_listing_filter(raw), MAX_JOB_TARGETS
+            )
+        except ListingSourceChanged:
+            return None, _coded(409, "source-changed", SOURCE_CHANGED_MESSAGE)
+        except ValueError as exc:
+            return None, bad_request(str(exc))
+        if selection.truncated:
+            return None, _coded(400, "too-many-images", TOO_MANY)
+        return tuple(selection.targets), None
+
+    async def _category_request(self, request, fields: frozenset, data=None):
+        """``(data, targets, roots, None)`` or ``(None, None, None, error)``.
+
+        The images are ``items`` (1 to 500 targets) or ``filter`` (all images
+        an Images listing filter admits), besides *fields*. *data* is the body
+        when the caller has already read it (a body is read once).
+        """
+        if data is None:
+            data, error = await _json_body_or_error(request)
+            if error is not None:
+                return None, None, None, error
+        names = set(data) - fields
+        if set(data) & fields != fields or names not in ({"items"}, {"filter"}):
+            return (
+                None,
+                None,
+                None,
+                bad_request(
+                    "Request must contain "
+                    + ", ".join(sorted(fields))
+                    + (" and " if fields else "")
+                    + "either items or filter"
+                ),
+            )
+        if "filter" in data:
+            targets, error = await self._filter_targets(data["filter"])
+            if error is not None:
+                return None, None, None, error
+            return data, targets, list(self._get_all_output_dirs()), None
+        items = data["items"]
+        if not isinstance(items, list) or not 1 <= len(items) <= MAX_JOB_TARGETS:
+            return (
+                None,
+                None,
+                None,
+                bad_request(f"items must list 1 to {MAX_JOB_TARGETS} images"),
+            )
+        targets, error = _parse_targets(items, media_message=CATEGORY_MEDIA_MESSAGE)
+        if error is not None:
+            return None, None, None, error
+        try:
+            for _index, _scope, relative in targets:
+                relative.encode("utf-8")
+        except UnicodeEncodeError:
+            return None, None, None, bad_request("path must be valid text")
+        roots = list(self._get_all_output_dirs())
+        for index, scope in dict.fromkeys((t[0], t[1]) for t in targets):
+            if index >= len(roots):
+                return None, None, None, bad_request("Invalid gallery root")
+            error = self._gallery_scope_error(
+                {"root_scope": scope, "root": str(index)}, roots
+            )
+            if error is not None:
+                if error.status == 409:
+                    error = _coded(409, "source-changed", SOURCE_CHANGED_MESSAGE)
+                return None, None, None, error
+        return data, targets, roots, None
+
+    def _listed_file_keys(self, roots, targets, files) -> tuple[list, list]:
+        """``(listed keys, unlisted keys)`` of the targets (blocking).
+
+        A target counts as listed when its source's listing holds it; its key
+        then follows the listed spelling. Targets folding to one key count
+        once, and a key that is listed through any target is listed.
+        """
+        listings, listed, missing = {}, {}, {}
+        for index, _scope, relative in targets:
+            root = roots[index]
+            if index not in listings:
+                listings[index] = self._favorite_listing(root, files[index])
+            spelled = listings[index].get(_favorite_relative_key(relative))
+            key = _file_key(roots, root, spelled or relative)
+            (missing if spelled is None else listed)[key] = True
+        return list(listed), [key for key in missing if key not in listed]
+
+    async def set_image_category(self, request):
+        """PUT ``{items, category}``: set the own category of listed files."""
+        data, targets, roots, error = await self._category_request(request, _PUT_FIELDS)
+        if error is not None:
+            return error
+        category, error = _category_value(data["category"])
+        if error is not None:
+            return error
+        try:
+            files = {}
+            for index in dict.fromkeys(t[0] for t in targets):
+                files[index] = await self._get_gallery_files(roots[index])
+            listed_keys, missing_keys = await self._run_in_executor(
+                self._listed_file_keys, roots, targets, files
+            )
+        except _KeyRefused as refused:
+            return _forbidden(refused.message)
+        except Exception:
+            self.logger.exception("Image category update failed")
+            return _coded(500, "category-failed", WRITE_FAILED)
+        try:
+            if not listed_keys:
+                return _coded(
+                    404,
+                    "files-not-found",
+                    "None of the selected images could be found.",
+                )
+            counts = await self._run_in_executor(
+                self.db.set_image_categories, listed_keys, category
+            )
+        except Exception:
+            self.logger.exception("Image category update failed")
+            return _coded(500, "category-failed", WRITE_FAILED)
+        return web.json_response(
+            {
+                "success": True,
+                "category": category,
+                "updated": counts["updated"],
+                "unchanged": counts["unchanged"],
+                "not_found": len(missing_keys),
+            },
+            headers={"Cache-Control": "no-store"},
+        )
+
+    async def clear_image_category(self, request):
+        """DELETE ``{items}``: remove the own category, listed or not."""
+        _data, targets, roots, error = await self._category_request(
+            request, _DELETE_FIELDS
+        )
+        if error is not None:
+            return error
+        try:
+            keys = await self._run_in_executor(
+                lambda: list(
+                    dict.fromkeys(_file_key(roots, roots[i], r) for i, _s, r in targets)
+                )
+            )
+            cleared = await self._run_in_executor(self.db.clear_image_categories, keys)
+        except _KeyRefused as refused:
+            return _forbidden(refused.message)
+        except Exception:
+            self.logger.exception("Image category removal failed")
+            return _coded(500, "category-failed", WRITE_FAILED)
+        return web.json_response(
+            {"success": True, "cleared": cleared},
+            headers={"Cache-Control": "no-store"},
+        )

py/api/image_delete.py

@@ -0,0 +1,521 @@
+"""Delete selected images: remove them from the library, or trash them (#480).
+
+``POST /prompt_manager/images/delete/preview {items | filter}`` reviews the
+selection for both modes at once and answers a reviewed plan (the System ›
+Maintenance contract, ``py/api/maintenance.py``): the token, the counts of
+the records each file loses, the refused files with a coded reason (both
+modes, and the trash-only ones), the prompts that lose their last image and
+the safety backup. It writes nothing. ``items`` are Images listing identities
+(at most 500); ``filter`` is "Select all N matching". Each target is matched
+to its entry in the current listing (``not-listed`` otherwise), so every
+later step uses the walked spelling, never the request's text.
+
+``POST /prompt_manager/maintenance/apply {token, mode}`` runs the plan as the
+``image_delete`` job (admission ``records``, not cancellable): the sources
+are checked, the promised safety backup is taken, then each file is checked
+again (containment and fingerprint) and
+
+- ``mode="library"``: its records are deleted and its removal mark written
+  in one transaction; the file stays on disk, hidden;
+- ``mode="trash"``: it is moved to the trash first (``utils/trash.py``); a
+  refused move leaves the file and its records alone; after a move its
+  records are deleted.
+
+No path here unlinks a media file. A failed records write stops the job; the
+files after it are not attempted.
+"""
+
+import hashlib
+import json
+import os
+import stat as stat_module
+from pathlib import Path
+
+from .admin import DELETABLE_MEDIA_EXTENSIONS, _remove_thumbnail
+from .duplicate_review import (
+    CHANGED_CONTENT,
+    GONE,
+    LINK,
+    OUTSIDE_SOURCE,
+    UNREADABLE,
+    file_problem,
+)
+from .images import _favorite_relative_key, _favorite_root_key
+from .jobs import IMAGE_DELETE_GROUP_NAMES, IMAGE_DELETE_JOB_KIND, JobBusy
+from .job_routes import busy_response
+from .maintenance import MAINTENANCE_PREVIEW_LEASE, _fail_job, _failure, backup_note
+
+try:
+    from ...database import image_removal_schema
+    from ...database.image_removal import mark_hides
+    from ...database.annotation_schema import MAX_LOCATOR_LENGTH
+    from ...utils import trash
+    from ...utils.media_folders import is_trash_folder
+except ImportError:
+    from database import image_removal_schema
+    from database.image_removal import mark_hides
+    from database.annotation_schema import MAX_LOCATOR_LENGTH
+    from utils import trash
+    from utils.media_folders import is_trash_folder
+
+IMAGE_DELETE = "delete_images"
+MODES = ("library", "trash")
+TITLES = {
+    "library": "Delete images (remove from the library)",
+    "trash": "Delete images (move to the trash)",
+}
+MAX_DETAIL_ROWS = 50
+# The system moved the file but its place in a trash could not be shown (#499).
+TRASH_UNCONFIRMED = "trash-unconfirmed"
+RECORDS_FAILED_CODE = "records-failed"
+
+# Reason codes (design §4.4) and the sentence each shows.
+REASONS = {
+    "not-listed": "No longer listed in Images.",
+    "outside": "Outside the output folders.",
+    "link": "A link, not a file.",
+    "gone": "The file no longer exists.",
+    "unreadable": "The file can't be read.",
+    "thumbnail": "A generated preview, not an image.",
+    "not-media": "Not a media file.",
+    "trash-folder": "Inside a trash folder.",
+    "path-too-long": "The path is too long.",
+    "changed": "The file changed after the review.",
+    trash.NO_TRASH: "This drive has no trash.",
+    trash.NETWORK: "Network drives can't use the trash.",
+    trash.TRASH_IN_ROOT: "This drive's trash is inside an output folder.",
+    trash.LOCKED: "The file is open in another program. Close it and try again.",
+    trash.FAILED: "The file could not be moved to the trash.",
+    trash.NO_PERMISSION: (
+        "PromptManager isn't allowed to move this file (no permission, or it is "
+        "locked in Finder)."
+    ),
+    trash.CLOUD_SYNC: (
+        "Files in iCloud Drive or a cloud-synced folder can't be moved to the "
+        "trash here."
+    ),
+    TRASH_UNCONFIRMED: (
+        "Moved by macOS, but its place in the trash could not be confirmed."
+    ),
+    trash.WINDOWS_RECYCLE: (
+        "Moving files to the Recycle Bin isn't available on Windows: files larger "
+        "than the bin's capacity would be deleted permanently."
+    ),
+    "not-attempted": "Not attempted: the job stopped after an error.",
+}
+_PROBLEM_CODES = {
+    OUTSIDE_SOURCE: "outside",
+    LINK: "link",
+    GONE: "gone",
+    UNREADABLE: "unreadable",
+    CHANGED_CONTENT: "changed",
+}
+SOURCES_CHANGED = "The sources changed since the review; nothing was changed."
+RECORDS_FAILED = "The image records could not be removed; the job stopped."
+
+
+def lexical_problem(relative: str) -> str | None:
+    """The refusal code a root-relative POSIX path earns before any file access."""
+    parts = relative.split("/")
+    if "thumbnails" in parts[:-1]:
+        return "thumbnail"
+    if any(is_trash_folder(part) for part in parts[:-1]):
+        return "trash-folder"
+    if os.path.splitext(parts[-1])[1].lower() not in DELETABLE_MEDIA_EXTENSIONS:
+        return "not-media"
+    return None
+
+
+def file_check(path, root, relative, *, expected=None):
+    """``(code, lstat)``: why *path* may not be deleted, or ``(None, lstat)``.
+
+    Text first (inside the root, not a preview or trash folder, a media
+    file), then ``duplicate_review.file_problem``: a regular file, not a
+    link, whose resolved path is still inside the resolved root. With
+    *expected* (``{size, mtime_ns, inode}`` from the review) the file must
+    still be those bytes. Every handle is closed when this returns.
+    """
+    code = lexical_problem(relative)
+    if code:
+        return code, None
+    try:
+        info = os.lstat(path)
+    except FileNotFoundError:
+        return "gone", None
+    except OSError:
+        return "unreadable", None
+    if getattr(info, "st_file_attributes", 0) & getattr(
+        stat_module, "FILE_ATTRIBUTE_REPARSE_POINT", 0
+    ):
+        return "link", info
+    if expected is None:
+        problem = file_problem(path, root, None, info.st_size, hash_check=False)
+    else:
+        problem = file_problem(
+            path,
+            root,
+            None,
+            expected["size"],
+            hash_check=False,
+            mtime_ns=expected["mtime_ns"],
+            inode=expected["inode"],
+        )
+    if problem:
+        return _PROBLEM_CODES.get(problem, "outside"), info
+    if len(os.path.normcase(os.path.realpath(path))) > MAX_LOCATOR_LENGTH:
+        return "path-too-long", info
+    return None, info
+
+
+def _row(target, code):
+    return {
+        "path": target["rel"],
+        "root_label": target["root_label"],
+        "code": code,
+        "detail": REASONS[code],
+    }
+
+
+def _summary(rows):
+    counts = {}
+    for row in rows:
+        counts[row["code"]] = counts.get(row["code"], 0) + 1
+    return {"count": len(rows), "counts": counts, "rows": rows[:MAX_DETAIL_ROWS]}
+
+
+def _fingerprint(targets) -> str:
+    material = [
+        (t["root_index"], t["rel"], t["size"], t["mtime_ns"], t["inode"])
+        for t in targets
+    ]
+    return hashlib.sha256(json.dumps(material).encode("utf-8")).hexdigest()
+
+
+class ImageDeleteMixin:
+    """The Images Delete… review and its apply job (#480)."""
+
+    def _register_image_delete_routes(self, routes):
+        # A fixed path beside the dynamic /images/{image_id} routes: aiohttp
+        # (3.10+) looks fixed paths up first (see image_categories.py).
+        @routes.post("/prompt_manager/images/delete/preview")
+        async def preview_image_delete_route(request):
+            return await self.preview_image_delete(request)
+
+    # ── review ────────────────────────────────────────────────────────
+
+    async def preview_image_delete(self, request):
+        """Review the selection under the maintenance preview lease."""
+        try:
+            async with self._jobs.lease(MAINTENANCE_PREVIEW_LEASE):
+                return await self._preview_image_delete(request)
+        except JobBusy as busy:
+            return busy_response(busy)
+
+    async def _preview_image_delete(self, request):
+        _data, targets, roots, error = await self._category_request(
+            request, frozenset()
+        )
+        if error is not None:
+            return error
+        db = self.db
+        identity = self._database_identity(db)
+        try:
+            files = {}
+            for index in dict.fromkeys(t[0] for t in targets):
+                files[index] = await self._get_gallery_files(roots[index])
+            review = await self._run_in_executor(
+                self._review_image_delete, db, roots, targets, files
+            )
+        except Exception:
+            self.logger.exception("Image delete review failed")
+            return _failure("The review failed. Check server logs for details.", 500)
+        return await self._plan_response(
+            db,
+            identity,
+            IMAGE_DELETE,
+            _fingerprint(review["targets"]),
+            review["public"],
+            {"targets": review["targets"]},
+        )
+
+    def _review_image_delete(self, db, roots, targets, files):
+        """Every target checked now (blocking); ``{targets, public}``."""
+        listings, seen = {}, set()
+        acted, refused, trash_refused = [], [], []
+        spellings = self._source_spellings()
+        root_texts = [str(root) for root in roots]
+        for index, _scope, relative in targets:
+            root = roots[index]
+            if index not in listings:
+                listings[index] = self._favorite_listing(root, files[index])
+            spelled = listings[index].get(_favorite_relative_key(relative))
+            target = {
+                "root_index": index,
+                "root_scope": _scope,
+                "root": str(root),
+                "root_label": self._public_path(root),
+                "rel": spelled or relative,
+            }
+            if spelled is None:
+                refused.append(_row(target, "not-listed"))
+                continue
+            absolute = os.path.join(str(root), *spelled.split("/"))
+            code, info = file_check(absolute, str(root), spelled)
+            if code:
+                refused.append(_row(target, code))
+                continue
+            identity = (info.st_dev, info.st_ino)
+            if identity in seen:
+                continue  # one physical file named twice (nested roots, hardlinks)
+            seen.add(identity)
+            target.update(self._removal_of(absolute, info, spellings))
+            target["trash"] = trash.trash_available(absolute, root_texts)
+            if target["trash"]:
+                trash_refused.append(_row(target, target["trash"]))
+            acted.append(target)
+        counted = db.count_removal_records([t["removal"] for t in acted])
+        library_count = len(acted)
+        public = {
+            "operation": IMAGE_DELETE,
+            "scope": {"count": library_count, "label": "images"},
+            "library": {"count": library_count},
+            "trash": {"count": library_count - len(trash_refused)},
+            "protected": {"count": len(refused), "reasons": []},
+            "refused": _summary(refused),
+            "trash_refused": _summary(trash_refused),
+            "records": counted["records"],
+            "orphaned_prompts": counted["orphaned_prompts"],
+            "media": {"deletes_files": False, "label": "Chosen in the review"},
+            "details": [],
+        }
+        return {"targets": acted, "public": public}
+
+    @staticmethod
+    def _removal_of(absolute, info, spellings):
+        favorites = [
+            (_favorite_root_key(root), _favorite_relative_key(relative))
+            for root, relative in spellings.favorites(absolute).items()
+        ]
+        return {
+            "abs": absolute,
+            "size": info.st_size,
+            "mtime_ns": info.st_mtime_ns,
+            "inode": [info.st_dev, info.st_ino],
+            "removal": {
+                "spellings": list(spellings.of(absolute).values()),
+                "locator": os.path.normcase(os.path.realpath(absolute)),
+                "favorites": favorites,
+                "category": spellings.category_key(absolute),
+            },
+        }
+
+    # ── apply ─────────────────────────────────────────────────────────
+
+    def _start_image_delete(self, db, plan, mode, backup_on):
+        """Register the delete job for a reviewed plan (on the loop)."""
+        return self._jobs.start(
+            IMAGE_DELETE_JOB_KIND,
+            TITLES[mode],
+            admission="records",
+            cancellable=False,
+            scope={"operation": IMAGE_DELETE, "mode": mode},
+            groups=IMAGE_DELETE_GROUP_NAMES,
+            runner=lambda job: self._run_image_delete(job, db, plan, mode, backup_on),
+        )
+
+    async def _run_image_delete(self, job, db, plan, mode, backup_on):
+        targets = plan.payload["targets"]
+        unchanged = await self._run_in_executor(self._sources_unchanged, targets)
+        if not unchanged:
+            return _fail_job(job, "sources-changed", SOURCES_CHANGED)
+        ok, backup = await self._safety_backup_step(job, db, backup_on)
+        if not ok:
+            return
+        try:
+            await self._delete_images(job, db, targets, mode, backup)
+        finally:
+            if backup:
+                await self._prune_backup_pool(job, db)
+
+    async def _delete_images(self, job, db, targets, mode, backup):
+        job.update(total=len(targets), status_line=TITLES[mode])
+        roots = [str(root) for root in self._get_all_output_dirs()]
+        done_group = "trashed" if mode == "trash" else "removed"
+        changed, failed = 0, None
+        for done, target in enumerate(targets, start=1):
+            if failed:
+                self._record_delete(job, "not_changed", target, "not-attempted")
+                continue
+            group, code, label = await self._run_in_executor(
+                self._delete_one_image, db, target, mode, roots
+            )
+            if group == "errors":
+                failed = code or RECORDS_FAILED_CODE
+                text = REASONS[code] if code else RECORDS_FAILED
+                job.record(
+                    "errors",
+                    {
+                        "path": target["rel"],
+                        "root": target["root_label"],
+                        "root_index": target["root_index"],
+                        "root_scope": target["root_scope"],
+                        "detail": text,
+                    },
+                    f"! {target['root_label']}/{target['rel']} · {text}",
+                )
+            elif group == done_group:
+                changed += 1
+                detail = (
+                    f"Moved to the {label}" if label else "Removed from the library"
+                )
+                self._record_delete(job, group, target, None, detail)
+            else:
+                self._record_delete(job, group, target, code)
+            job.update(processed=done, progress=int(done / len(targets) * 100))
+        if changed:
+            self.invalidate_gallery_cache()
+        verb = "Moved" if mode == "trash" else "Removed"
+        where = "to the trash" if mode == "trash" else "from the library"
+        line = f"{verb} {changed:,} of {len(targets):,} images {where}"
+        job.log_line(line)
+        job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
+        if failed:
+            # "records-failed" names every stop at an errors row (the page's
+            # "Stopped after N of M" toast); the row says which error it was.
+            job.scope["failure"] = RECORDS_FAILED_CODE
+            job.publish({"type": "error", "code": failed, "message": line})
+            job.finish("failed")
+
+    @staticmethod
+    def _record_delete(job, group, target, code, detail=None):
+        text = detail or REASONS[code]
+        row = {
+            "path": target["rel"],
+            "root": target["root_label"],
+            "root_index": target["root_index"],
+            "root_scope": target["root_scope"],
+            "code": code,
+            "detail": text,
+        }
+        mark = {"removed": "-", "trashed": "-"}.get(group, "=")
+        job.record(
+            group, row, f"{mark} {target['root_label']}/{target['rel']} · {text}"
+        )
+
+    def _delete_one_image(self, db, target, mode, roots):
+        """``(group, code, trash label)`` for one reviewed file (blocking)."""
+        code, _info = file_check(
+            target["abs"],
+            target["root"],
+            target["rel"],
+            expected={
+                "size": target["size"],
+                "mtime_ns": target["mtime_ns"],
+                "inode": target["inode"],
+            },
+        )
+        if code:
+            return "not_changed", code, None
+        label = None
+        if mode == "trash":
+            try:
+                label = trash.move_to_trash(target["abs"], roots)
+            except trash.TrashRefused as refused:
+                code = refused.code if refused.code in REASONS else trash.FAILED
+                self.logger.info(
+                    "Not moved to the trash (%s): %s %s",
+                    code,
+                    target["rel"],
+                    refused.detail,
+                )
+                return "not_changed", code, None
+            except trash.TrashUnconfirmed as unconfirmed:
+                # The file left its folder: keep its records, stop the job.
+                self.logger.warning(
+                    "Trash move not confirmed: %s %s", target["rel"], unconfirmed.detail
+                )
+                return "errors", TRASH_UNCONFIRMED, None
+            mark = None
+        else:
+            mark = (target["size"], target["mtime_ns"])
+        try:
+            db.remove_image_records(target["removal"], mark=mark)
+        except Exception:
+            self.logger.exception("Removing an image's records failed")
+            return "errors", None, label
+        _remove_thumbnail(Path(target["root"]), Path(target["rel"]), self.logger)
+        return ("trashed" if mode == "trash" else "removed"), None, label
+
+    # ── the workspace listing ─────────────────────────────────────────
+
+    def _removal_marks(self) -> dict:
+        """The removal marks, cached until a mark changes or the database does.
+
+        A listing page reads no table while nothing changed: the cache follows
+        this process's marks generation and the database identity (a restore).
+        """
+        key = (self._database_identity(self.db), image_removal_schema.generation())
+        cached = getattr(self, "_marks_cache", None)
+        if cached is None or cached[0] != key:
+            cached = (key, self.db.removal_marks())
+            self._marks_cache = cached
+        return cached[1]
+
+    def _hidden_entries(self, listings) -> set:
+        """Normalized paths of listed files a removal mark hides (blocking).
+
+        Only marked locators are looked at, and only those files are
+        ``lstat``'ed: a mark hides a file while its size and ``mtime_ns`` are
+        still the ones it was removed with.
+        """
+        marks = self._removal_marks()
+        if not marks:
+            return set()
+        located = self._workspace_locators(listings)
+        return {
+            entry.normalized
+            for locator, entry in ((locator, located.get(locator)) for locator in marks)
+            if entry is not None and mark_hides(marks, locator, entry.path)
+        }
+
+    def _removed_paths(self, paths) -> set:
+        """The absolute paths among *paths* a removal mark hides (blocking)."""
+        marks = self._removal_marks()
+        if not marks:
+            return set()
+        return {
+            str(path)
+            for path in paths
+            if mark_hides(marks, os.path.normcase(os.path.realpath(str(path))), path)
+        }
+
+    def _removal_check_under(self, root):
+        """``hidden(path) -> bool`` for files a walk of *root* found (blocking).
+
+        A walk never follows a link below *root*, so a file's locator is
+        ``normcase(join(realpath(root), relative))``: no ``realpath`` per
+        file, and an ``lstat`` only for a marked one. Without marks it
+        answers False at once.
+        """
+        marks = self._removal_marks()
+        if not marks:
+            return lambda _path: False
+        root = os.fspath(root)
+        real = os.path.realpath(root)
+
+        def hidden(path):
+            relative = os.path.relpath(os.fspath(path), root)
+            locator = os.path.normcase(os.path.join(real, relative))
+            return mark_hides(marks, locator, path)
+
+        return hidden
+
+    def _without_removed(self, root, paths) -> list:
+        """*paths* found by a walk of *root*, less the hidden ones (blocking)."""
+        removed = self._removal_check_under(root)
+        return [path for path in paths if not removed(path)]
+
+
+def parse_mode(value):
+    """The apply's ``mode``, or None when it is not one of MODES."""
+    return value if isinstance(value, str) and value in MODES else None

py/api/image_return.py

@@ -0,0 +1,148 @@
+"""Return removed images to the library (#498).
+
+``POST /prompt_manager/images/removed/return {items | filter}`` names files a
+removal mark hides (#480, "Remove from the library only"): ``items`` are 1 to
+500 listing identities, ``filter`` the removed view's listing query
+(``removed: "1"`` plus the view's other parameters, at most 500 images). Only
+this route takes ``removed``: every other bulk route refuses it, because an
+explicit write there would show a file again without re-reading it.
+
+The route answers 202 ``{job, returned, not_removed}`` after starting a
+``scan`` job titled "Return images to the library" (admission ``records``:
+409 ``busy`` beside a scan, restore or delete, nothing changed). The job
+first deletes the marks that still hide their file (same size and
+``mtime_ns``), in one transaction, then scans exactly those files with the
+scan's own pipeline: prompts, links, pairs and Needs review items come back
+from the files' embedded metadata. Image tags, auto-tag marks, the own
+category, collection memberships and the favorite were deleted by the remove
+and do not come back. No file is moved, written or deleted.
+"""
+
+import os
+from pathlib import Path
+
+from aiohttp import web
+
+from .image_categories import TOO_MANY, _coded
+from .images import (
+    ListingSourceChanged,
+    _favorite_relative_key,
+    _json_body_or_error,
+    parse_listing_filter,
+)
+from .image_tag_jobs import MAX_JOB_TARGETS
+from .jobs import JobBusy
+from .job_routes import busy_response
+
+try:
+    from ...database.image_removal import mark_hides
+except ImportError:
+    from database.image_removal import mark_hides
+
+NONE_REMOVED = "None of the selected images are removed from the library."
+SOURCE_CHANGED = "The image sources changed. Reload Images and try again."
+
+
+def _bad(message):
+    return web.json_response({"success": False, "error": message}, status=400)
+
+
+class ImageReturnMixin:
+    """Return to library (#498): clear the marks, then scan those files."""
+
+    def _register_image_return_routes(self, routes):
+        # A fixed path beside the dynamic /images/{image_id} routes.
+        @routes.post("/prompt_manager/images/removed/return")
+        async def return_removed_images_route(request):
+            return await self.return_removed_images(request)
+
+    async def return_removed_images(self, request):
+        data, error = await _json_body_or_error(request)
+        if error is not None:
+            return error
+        if set(data) == {"filter"}:
+            targets, roots, error = await self._removed_filter_targets(data["filter"])
+        else:
+            _data, targets, roots, error = await self._category_request(
+                request, frozenset(), data=data
+            )
+        if error is not None:
+            return error
+        files = {}
+        for index in dict.fromkeys(t[0] for t in targets):
+            files[index] = await self._get_gallery_files(roots[index])
+        returned, not_removed = await self._run_in_executor(
+            self._hidden_targets, roots, targets, files
+        )
+        if not returned:
+            return _coded(400, "no-images", NONE_REMOVED)
+        try:
+            job = self._start_scan_job(
+                files="all", output_dirs=list(roots), returned=returned
+            )
+        except JobBusy as busy:
+            return busy_response(busy)
+        return web.json_response(
+            {
+                "success": True,
+                "job": job.summary(),
+                "returned": len(returned),
+                "not_removed": not_removed,
+            },
+            status=202,
+        )
+
+    async def _removed_filter_targets(self, raw):
+        """``(targets, roots, None)`` for the removed view's filter, else an error."""
+        if not isinstance(raw, dict) or raw.get("removed") != "1":
+            return None, None, _bad('filter must hold removed: "1"')
+        rest = {key: value for key, value in raw.items() if key != "removed"}
+        try:
+            parse_listing_filter(rest)
+            selection = await self.resolve_listing_filter(
+                {**rest, "removed": "1"}, MAX_JOB_TARGETS
+            )
+        except ListingSourceChanged:
+            return None, None, _coded(409, "source-changed", SOURCE_CHANGED)
+        except ValueError as exc:
+            return None, None, _bad(str(exc))
+        if selection.truncated:
+            return None, None, _coded(400, "too-many-images", TOO_MANY)
+        if not selection.targets:
+            return None, None, _coded(400, "no-images", NONE_REMOVED)
+        return tuple(selection.targets), list(self._get_all_output_dirs()), None
+
+    def _hidden_targets(self, roots, targets, files):
+        """``([(path, locator)], not_removed)``: the targets a mark hides now.
+
+        Each target is matched to its listed spelling (the walk's). One
+        physical file named twice (nested roots) is returned once; a target
+        that is not listed or not hidden counts in ``not_removed``.
+        """
+        marks = self._removal_marks()
+        listings, seen, returned, not_removed = {}, set(), [], 0
+        for index, _scope, relative in targets:
+            root = roots[index]
+            if index not in listings:
+                listings[index] = self._favorite_listing(root, files[index])
+            spelled = listings[index].get(_favorite_relative_key(relative))
+            if spelled is None:
+                not_removed += 1
+                continue
+            path = os.path.join(str(root), *spelled.split("/"))
+            locator = os.path.normcase(os.path.realpath(path))
+            if locator in seen:
+                continue
+            if not mark_hides(marks, locator, path):
+                not_removed += 1
+                continue
+            seen.add(locator)
+            returned.append((Path(path), locator))
+        return returned, not_removed
+
+    def _clear_returned_marks(self, returned) -> list:
+        """The paths whose marks this deleted (blocking; one transaction)."""
+        cleared = self.db.clear_hiding_marks(
+            [(locator, str(path)) for path, locator in returned]
+        )
+        return [path for path, locator in returned if locator in cleared]

py/api/image_tag_jobs.py

@@ -2,7 +2,9 @@
 
 ``POST /prompt_manager/jobs/image-tags`` starts one ``image-tags`` job over
 1..500 listed images, named by the listing identity ``{root_index,
-root_scope, path}``. The job is record-changing (it writes image tag sets and
+root_scope, path}``, or over every image a listing ``filter`` admits (at most
+10,000, resolved at start in the listing's order; "Select all N matching",
+#392). The job is record-changing (it writes image tag sets and
 new vocabulary names), so it never runs beside the metadata scan.
 
 - **Add tags** (``mode: "add"``) hashes each current file and unions a fixed
@@ -18,13 +20,21 @@ the suggestion worker and the storage call ``add_image_tags``. Files are
 handled one at a time and the job stops at a file boundary when cancelled,
 keeping what it already wrote. Rows and log lines carry root-relative paths
 and public root labels only.
+
+Auto-tag marks each image it ran the model on as auto-tagged (#379), with the
+union of its tags. The ``autotag`` job ("Tag output images", started by
+``POST /jobs/autotag``) runs the same per-file loop over every still image the
+output folders list (:meth:`ImageTagJobsMixin._run_autotag_batch`): image
+tags only, never a prompt tag or prompt text.
 """
 
 import asyncio
+import dataclasses
 import functools
 import os
 import time
 from dataclasses import dataclass, field
+from typing import NamedTuple
 
 from aiohttp import web
 
@@ -32,7 +42,7 @@ from .. import model_gate
 from .admin import _plural, _time_left
 from .annotation_suggest import (
     OptionError,
-    unavailable_message,
+    model_refusal,
     validate_options,
 )
 from .annotations import (
@@ -45,10 +55,26 @@ from .annotations import (
     _submit,
     _target,
 )
-from .autotag_routes import model_busy_response
-from .images import _public_path
+from .autotag_routes import BATCH_CANCELLED, MODEL_BUSY_ERROR, model_busy_response
+from .images import (
+    MAX_LISTING_SELECTION,
+    CollectionMissing,
+    ListingSourceChanged,
+    collection_not_found,
+    _current_fingerprint,
+    _listing_posix,
+    _public_path,
+    parse_listing_filter,
+)
 from .job_routes import busy_response
-from .jobs import IMAGE_TAG_GROUP_NAMES, IMAGE_TAG_JOB_KIND, JobBusy
+from .jobs import (
+    AUTOTAG_BATCH_JOB_KIND,
+    AUTOTAG_GROUP_NAMES,
+    IMAGE_TAG_GROUP_NAMES,
+    IMAGE_TAG_JOB_KIND,
+    JobBusy,
+)
+from .maintenance import disk_plan, short_space_message
 from .prompts import bad_request
 
 try:
@@ -75,6 +101,11 @@ except ImportError:
     from utils.video_metadata import is_video_path
 
 MAX_JOB_TARGETS = 500
+# A filter start resolves on the server, so the request body does not bound it.
+MAX_FILTER_TARGETS = MAX_LISTING_SELECTION
+NO_IMAGES = "No images match this view."
+FILTER_LISTING_FAILED = "The matching images could not be listed. Try again."
+TOO_MANY_IMAGES = "Narrow the view to at most {cap:,} images."
 # 500 maximal targets plus the options: a path of MAX_TARGET_PATH code points
 # takes at most 6 bytes each on the wire (a \uXXXX escape; raw UTF-8 is at most
 # 4), and the root index and scope fit in 256. The annotation routes' 64 KiB
@@ -88,9 +119,10 @@ ADD_TITLE = "Add tags to images"
 AUTO_TITLE = "Auto-tag images"
 GATE_OWNER_KIND = "image-tag-job"
 
+# Each mode's fields besides the images, named by ``targets`` or ``filter``.
 _FIELDS = {
-    "add": frozenset({"mode", "targets", "tags"}),
-    "auto": frozenset({"mode", "targets", "model_type", "options"}),
+    "add": frozenset({"mode", "tags"}),
+    "auto": frozenset({"mode", "model_type", "options"}),
 }
 _VERBS = {"add": "Add tags", "auto": "Auto-tag"}
 _LOG_PREFIXES = {"tagged": "+", "unchanged": "=", "skipped": "-", "errors": "!"}
@@ -107,6 +139,14 @@ READ_FAILED = "The file could not be read."
 SOURCE_CHANGED = "The source changed. Refresh the image collection."
 STORAGE_RESTORED = "Prompt storage was restored; start the job again."
 BUSY_RETRY = "The file could not be read now. Run the job again for it."
+# Tag output images (#379): the per-image batch.
+ALREADY_AUTOTAGGED = "Already auto-tagged"
+LISTING_FAILED = "The output folders could not be listed. Try again."
+LISTING_STATUS = "Listing output images…"
+_INTERNAL_ERROR = "An internal error occurred. Check server logs for details."
+# The pre-pass stats and looks up this many targets at a time, checking for a
+# cancel between chunks.
+PREPASS_CHUNK = 500
 
 _monotonic = time.monotonic  # the job's clock (a test can fake it)
 
@@ -141,6 +181,8 @@ class _Spec:
     tags: tuple = ()
     model_type: str | None = None
     options: dict | None = None
+    # A listing filter the start resolves into ``targets`` (#392).
+    filter: dict | None = None
 
     @property
     def title(self) -> str:
@@ -153,13 +195,17 @@ class _Spec:
         return scope
 
 
-def _parse_targets(raw):
-    """Unique validated targets in request order, or ``(None, error)``."""
+def _parse_targets(raw, **target_options):
+    """Unique validated targets in request order, or ``(None, error)``.
+
+    ``target_options`` reach the target check (the image category routes
+    pass their own ``media_message``, #332).
+    """
     if not isinstance(raw, list) or not 1 <= len(raw) <= MAX_JOB_TARGETS:
         return None, bad_request(f"targets must list 1 to {MAX_JOB_TARGETS} images")
     seen, targets = set(), []
     for item in raw:
-        target, error = _target(item)
+        target, error = _target(item, **target_options)
         if error is not None:
             return None, error
         if target not in seen:
@@ -199,21 +245,35 @@ def parse_image_tag_request(data: dict):
     """A :class:`_Spec` for a start body, or ``(None, error response)``.
 
     ``{mode:"add", targets, tags}`` or ``{mode:"auto", targets, model_type,
-    options}`` exactly; options follow the Tag dialog's Generate rules.
+    options}`` exactly; options follow the Tag dialog's Generate rules. A
+    ``filter`` (a listing query object, parse_listing_filter) may stand in for
+    ``targets``; the start resolves it.
     """
     fields = _FIELDS.get(data.get("mode"))
     if fields is None:
         return None, bad_request('mode must be "add" or "auto"')
+    fields = fields | {"filter" if "filter" in data else "targets"}
     if set(data) != fields:
         return None, bad_request(
             "Request must contain exactly " + ", ".join(sorted(fields))
         )
-    targets, error = _parse_targets(data["targets"])
-    if error is not None:
-        return None, error
+    listing_filter, targets = None, ()
+    if "filter" in data:
+        try:
+            listing_filter = parse_listing_filter(data["filter"])
+        except ValueError as exc:
+            return None, bad_request(str(exc))
+    else:
+        targets, error = _parse_targets(data["targets"])
+        if error is not None:
+            return None, error
     if data["mode"] == "add":
-        return _parse_add(targets, data)
-    return _parse_auto(targets, data)
+        spec, error = _parse_add(targets, data)
+    else:
+        spec, error = _parse_auto(targets, data)
+    if spec is not None and listing_filter is not None:
+        spec = dataclasses.replace(spec, filter=listing_filter)
+    return spec, error
 
 
 def _capture_outcome(exc: CaptureError, root, relative: str):
@@ -246,6 +306,92 @@ async def _capacity_retried(job, attempt):
     return await attempt()
 
 
+class Refusal(NamedTuple):
+    """Why a Tag output images start was refused.
+
+    ``response`` is the native route's JSON answer; ``message`` the sentence
+    a legacy stream frame carries.
+    """
+
+    status: int
+    code: str
+    message: str
+    response: web.Response
+
+
+@dataclass(frozen=True)
+class AutotagRun:
+    """What one Tag output images job runs with (validated at its start)."""
+
+    model_type: str
+    existing: str
+    options: dict
+    keep_in_memory: bool
+    backup_on: bool
+    owner: object
+
+
+@dataclass(frozen=True)
+class _BatchTarget:
+    """One still image of the batch: the job target and its private keys."""
+
+    target: tuple  # (root_index, root_scope, relative): the image-tags target
+    path: str  # the listed file, for the pre-pass stat (never sent out)
+    locator: str  # the image tag key: normcase(join(realpath(root), relative))
+    label: str  # the root's public label
+
+
+def _still_targets(api, index, root, files, seen: set) -> list:
+    """Worker thread: one root's listed still images as _BatchTargets.
+
+    Newest first (the listing's order). A locator an earlier root already
+    produced (overlapping roots) is dropped; ``seen`` collects them.
+    """
+    scope = api._gallery_root_scope(root, index)
+    label = _public_path(root)
+    real = os.path.realpath(root)
+    prefix = os.path.join(str(root), "")
+    found = []
+    for path, _mtime in files:
+        relative = _listing_posix(root, prefix, path)
+        if is_video_path(relative):
+            continue
+        locator = os.path.normcase(os.path.join(real, *relative.split("/")))
+        if locator in seen:
+            continue
+        seen.add(locator)
+        found.append(_BatchTarget((index, scope, relative), str(path), locator, label))
+    return found
+
+
+def _prepass_chunk(db, chunk, skip_marked: bool, max_file: int):
+    """Worker thread: ``(remaining, skips)`` for a chunk of _BatchTargets.
+
+    Stats each file (no open, no hash): gone is "File not found", over the
+    capture limit "Too large to verify"; under "Skip already tagged" a file
+    whose current bytes are marked is "Already auto-tagged". ``skips`` are
+    ``(target, detail)`` pairs in target order.
+    """
+    stats, skips, alive = {}, [], []
+    for item in chunk:
+        fingerprint = _current_fingerprint(item.path)
+        if fingerprint is None:
+            skips.append((item, FILE_NOT_FOUND))
+        elif fingerprint[0] > max_file:
+            skips.append((item, TOO_LARGE))
+        else:
+            stats[item.locator] = fingerprint
+            alive.append(item)
+    marked = db.autotag_state(stats) if skip_marked and stats else {}
+    remaining = []
+    for item in alive:
+        if marked.get(item.locator):
+            skips.append((item, ALREADY_AUTOTAGGED))
+        else:
+            remaining.append(item)
+    return remaining, skips
+
+
 def _load_model(service, model_type: str, use_gpu: bool) -> None:
     """Worker thread: load (or swap to) *model_type*; never downloads."""
     if service.is_model_loaded() and service.get_loaded_model_type() == model_type:
@@ -254,6 +400,25 @@ def _load_model(service, model_type: str, use_gpu: bool) -> None:
         raise RuntimeError(f"{model_type} could not be loaded")
 
 
+def _refusal(status: int, code: str, message: str) -> Refusal:
+    response = web.json_response(
+        {"success": False, "code": code, "error": message}, status=status
+    )
+    return Refusal(status, code, message, response)
+
+
+def _busy_refusal(busy: JobBusy) -> Refusal:
+    title = getattr(busy.running, "title", "") or "Another job"
+    return Refusal(409, "busy", f"{title} already running", busy_response(busy))
+
+
+def _batch_cancelled(job) -> str:
+    """Say the batch was cancelled; its end state (the runner ends the job)."""
+    job.update(status_line=BATCH_CANCELLED)
+    job.log_line(BATCH_CANCELLED)
+    return "cancelled"
+
+
 class ImageTagJobsMixin:
     """The ``image-tags`` job kind: start route and runner."""
 
@@ -270,6 +435,10 @@ class ImageTagJobsMixin:
         spec, error = parse_image_tag_request(data)
         if error is not None:
             return error
+        if spec.filter is not None:
+            spec, error = await self._resolve_job_filter(spec)
+            if error is not None:
+                return error
         roots = await self._run_in_executor(self._job_roots, spec.targets)
         if any(root is None for root, _label in roots.values()):
             return _coded("source-changed")
@@ -301,6 +470,250 @@ class ImageTagJobsMixin:
             job.task.add_done_callback(lambda _task: model_gate.release(owner))
         return web.json_response({"success": True, "job": job.summary()}, status=202)
 
+    # -- Tag output images (#379) ---------------------------------------------
+
+    async def _autotag_targets(self) -> list:
+        """Every still image the output folders list, as _BatchTargets.
+
+        Root by root in configured order, each newest first (the shared
+        listing, ``_get_gallery_files``, used as it is even when stale). Videos
+        are not targets; a file under two overlapping roots is one target.
+        """
+        seen: set = set()
+        found = []
+        for index, root in enumerate(list(self._get_all_output_dirs())):
+            files = await self._get_gallery_files(root)
+            found.extend(
+                await self._run_in_executor(
+                    _still_targets, self, index, root, files, seen
+                )
+            )
+        # #480: a file removed from the library is never tagged by a batch.
+        hidden = await self._run_in_executor(
+            self._removed_paths, [target.path for target in found]
+        )
+        return [target for target in found if target.path not in hidden]
+
+    async def _start_autotag(
+        self, *, model_type, existing, options, keep_in_memory, title, owner_kind
+    ):
+        """Register a Tag output images job: ``(job, None)`` or ``(None, Refusal)``.
+
+        ``options`` are already validated (``validate_options``). The checks
+        run in this order: 409 ``model-unavailable``; 507
+        ``insufficient-space`` (only when the safety backup is on); 409
+        ``busy`` (another records job); 409 ``model-busy``. No await follows
+        the busy check. The gate is taken before the job registers and
+        released when the task ends, also before its runner ever starts.
+        """
+        from ..autotag import MODELS, get_autotag_service
+
+        backup_on = self._safety_backup_policy() == "on"
+        status = await self._run_in_executor(get_autotag_service().get_models_status)
+        _disk, short, _size = await self._run_in_executor(
+            functools.partial(
+                disk_plan, self.db.model.db_path, backup=backup_on, vacuum=False
+            )
+        )
+        label = MODELS[model_type]["label"]
+        message = model_refusal(status, model_type, label)
+        if message is not None:
+            return None, _refusal(409, "model-unavailable", message)
+        if backup_on and short is not None:
+            return None, _refusal(507, "insufficient-space", short_space_message(short))
+        busy = self._jobs.blocker(AUTOTAG_BATCH_JOB_KIND, admission="records")
+        if busy is not None:
+            return None, _busy_refusal(JobBusy(busy))
+        owner = model_gate.new_owner(owner_kind)
+        if not model_gate.try_acquire(owner):
+            response = model_busy_response()
+            return None, Refusal(409, "model-busy", MODEL_BUSY_ERROR, response)
+        run = AutotagRun(
+            model_type, existing, options, keep_in_memory, backup_on, owner
+        )
+        try:
+            job = self._jobs.start(
+                AUTOTAG_BATCH_JOB_KIND,
+                title,
+                admission="records",
+                cancellable=True,
+                scope={"model_type": model_type, "existing": existing, "label": label},
+                groups=AUTOTAG_GROUP_NAMES,
+                runner=functools.partial(self._run_autotag_batch, run),
+            )
+        except BaseException as refused:
+            model_gate.release(owner)
+            if isinstance(refused, JobBusy):
+                return None, _busy_refusal(refused)
+            raise
+        job.task.add_done_callback(lambda _task: model_gate.release(owner))
+        return job, None
+
+    async def _run_autotag_batch(self, run: AutotagRun, job) -> None:
+        """The ``autotag`` job: backup, model, listing, pre-pass, per-image loop.
+
+        ``job.cancelled`` is checked between phases (and per pre-pass chunk
+        and per image). Each phase publishes ``{"type": "phase", "phase"}``.
+        With ``keep_in_memory`` false the model is unloaded under the gate.
+        The job ends only after that cleanup and the gate release, so a job
+        that reads as ended never still holds the gate.
+        """
+        state = "failed"
+        try:
+            state = await self._autotag_batch_phases(run, job)
+        except Exception:
+            self.logger.exception("Tag output images failed")
+            job.update(status_line=_INTERNAL_ERROR)
+            job.log_line(f"Tagging failed · {_INTERNAL_ERROR}")
+        finally:
+            try:
+                if not run.keep_in_memory:
+                    await self._unload_batch_model(run.owner)
+                if run.backup_on:
+                    await self._prune_backup_pool(job, self.db)
+            finally:
+                model_gate.release(run.owner)
+                job.publish({"type": state, "progress": job.progress})
+                job.finish(state)
+
+    async def _unload_batch_model(self, owner) -> None:
+        from ..autotag import get_autotag_service
+
+        service = get_autotag_service()
+        try:
+            if service.is_model_loaded():
+                await model_gate.run(owner, service.unload_model)
+        except Exception:
+            self.logger.exception("The tagging model could not be unloaded")
+
+    async def _autotag_batch_phases(self, run: AutotagRun, job) -> str:
+        """Run the phases; the job's end state (the caller ends the job)."""
+        job.publish({"type": "phase", "phase": "backup"})
+        ok, _backup = await self._safety_backup_step(job, self.db, run.backup_on)
+        if not ok:
+            return "failed"
+        if job.cancelled:
+            return _batch_cancelled(job)
+        job.publish({"type": "phase", "phase": "model"})
+        job.update(status_line="Loading the tagging model")
+        spec = _Spec("auto", (), model_type=run.model_type, options=run.options)
+        service = await self._prepare_tagging_model(job, spec, run.owner)
+        if service is None:
+            return "failed"
+        if job.cancelled:
+            return _batch_cancelled(job)
+        job.publish({"type": "phase", "phase": "listing"})
+        job.update(status_line=LISTING_STATUS)
+        epoch = await self._run_in_executor(self.db.get_execution_epoch)
+        try:
+            targets = await self._autotag_targets()
+        except Exception:
+            self.logger.exception("Tag output images could not list the outputs")
+            job.update(status_line=LISTING_FAILED)
+            job.log_line(f"! {LISTING_FAILED}")
+            return "failed"
+        total = len(targets)
+        job.scope["images"] = total
+        job.update(total=total)
+        policy = (
+            "skip images already auto-tagged" if run.existing == "skip" else "merge"
+        )
+        job.log_line(
+            f"Tagging started · {job.scope.get('label') or run.model_type} · "
+            f"{_plural(total, 'image')} · {policy}"
+        )
+        if job.cancelled:
+            return _batch_cancelled(job)
+        remaining = await self._autotag_prepass(job, targets, run.existing == "skip")
+        if remaining is None:
+            return _batch_cancelled(job)
+        done = total - len(remaining)
+        spec = dataclasses.replace(
+            spec, targets=tuple(item.target for item in remaining)
+        )
+        loop_run = _Run(job, spec, epoch, run.owner, service)
+        if not await self._tag_each_file(loop_run, done_before=done):
+            job.update(status_line=STORAGE_RESTORED)
+            job.log_line(f"Tagging stopped · {STORAGE_RESTORED}")
+            return "failed"
+        if job.cancelled:
+            return _batch_cancelled(job)
+        finished = f"Tagged {_plural(job.counters.get('tagged', 0), 'image')}"
+        job.update(status_line=finished, progress=100)
+        job.log_line(finished)
+        return "finished"
+
+    async def _autotag_prepass(self, job, targets, skip_marked: bool):
+        """The targets left for the model, or None when cancelled meanwhile.
+
+        Opens and hashes nothing: one stat per file and one indexed read per
+        chunk. Skips are counted with ``record_counted`` (exact counters, rows
+        up to the cap, no log line each).
+        """
+        max_file = self._capture_store().max_file
+        remaining, skipped = [], 0
+        total = len(targets)
+        for start in range(0, total, PREPASS_CHUNK):
+            if job.cancelled:
+                return None
+            chunk = targets[start : start + PREPASS_CHUNK]
+            left, skips = await self._run_in_executor(
+                _prepass_chunk, self.db, chunk, skip_marked, max_file
+            )
+            remaining.extend(left)
+            if skips:
+                job.record_counted(
+                    "skipped",
+                    len(skips),
+                    [
+                        {
+                            "path": item.target[2],
+                            "root": item.label,
+                            "root_index": item.target[0],
+                            "detail": detail,
+                        }
+                        for item, detail in skips
+                    ],
+                )
+                skipped += len(skips)
+                job.update(processed=skipped, progress=skipped * 100 // total)
+                job.publish({"type": "progress", "progress": job.progress})
+        if skipped:
+            job.log_line(f"Skipped {_plural(skipped, 'image')} before tagging")
+        return remaining
+
+    async def _resolve_job_filter(self, spec: _Spec):
+        """``spec`` with its filter resolved into targets, or ``(None, error)``.
+
+        The listing's own identities, in its order, kept only where a target
+        could have named them (a media path the job can tag).
+        """
+        cap = MAX_FILTER_TARGETS
+        try:
+            selection = await self.resolve_listing_filter(spec.filter, cap)
+        except ListingSourceChanged:
+            return None, _coded("source-changed")
+        except CollectionMissing:
+            return None, collection_not_found()
+        except ValueError as exc:
+            return None, bad_request(str(exc))
+        except Exception:
+            # Never echo it: it can carry SQL or database text.
+            self.logger.error("A tagging filter could not be resolved", exc_info=True)
+            return None, _error(500, "listing-failed", FILTER_LISTING_FAILED)
+        if selection.truncated:
+            return None, _error(400, "too-many-images", TOO_MANY_IMAGES.format(cap=cap))
+        targets = []
+        for index, scope, relative in selection.targets:
+            target, error = _target(
+                {"root_index": index, "root_scope": scope, "path": relative}
+            )
+            if error is None:
+                targets.append(target)
+        if not targets:
+            return None, _error(400, "no-images", NO_IMAGES)
+        return dataclasses.replace(spec, targets=tuple(targets)), None
+
     # -- runner ----------------------------------------------------------------
 
     def _job_roots(self, targets) -> dict:
@@ -331,28 +744,39 @@ class ImageTagJobsMixin:
                     job.finish("failed")
                     return
             if not await self._tag_each_file(run):
+                job.update(status_line=STORAGE_RESTORED)
                 job.log_line(f"{verb} stopped · {STORAGE_RESTORED}")
                 job.finish("failed")
                 return
             state = "cancelled" if job.cancelled else "finished"
-            job.log_line(f"{verb} {state} · {_plural(job.processed, 'file')} processed")
+            # An ended job never keeps the loop's "about X min left" (#379).
+            ended = f"{verb} {state} · {_plural(job.processed, 'file')} processed"
+            job.update(status_line=ended)
+            job.log_line(ended)
             job.publish({"type": state, "progress": job.progress})
             job.finish(state)
         finally:
             if owner is not None:
                 model_gate.release(owner)
 
-    async def _tag_each_file(self, run: _Run) -> bool:
-        """Tag the targets in order; False when the job had to stop."""
-        job, total = run.job, len(run.spec.targets)
+    async def _tag_each_file(self, run: _Run, *, done_before: int = 0) -> bool:
+        """Tag the targets in order; False when the job had to stop.
+
+        ``done_before`` files were already counted (the batch's pre-pass): file
+        n of this loop is "file done_before + n" of the job, and the time left
+        uses this loop's rate only.
+        """
+        job, count = run.job, len(run.spec.targets)
+        total = done_before + count
         run.roots = await self._run_in_executor(self._job_roots, run.spec.targets)
         started = _monotonic()
         for number, target in enumerate(run.spec.targets, 1):
             if job.cancelled:
                 break
             # The scan's estimate (D56), with each finished file as one batch.
-            left = _time_left(number - 1, total, number - 1, _monotonic() - started)
-            job.update(status_line=f"Processing file {number:,} of {total:,}{left}")
+            left = _time_left(number - 1, count, number - 1, _monotonic() - started)
+            shown = done_before + number
+            job.update(status_line=f"Processing file {shown:,} of {total:,}{left}")
             job.publish({"type": "progress", "progress": job.progress})
             stopped = False
             try:
@@ -363,20 +787,20 @@ class ImageTagJobsMixin:
                 self.logger.exception("Image tagging failed for one file")
                 group, detail = "errors", SAVE_FAILED
             self._record_tag_outcome(run, target, group, detail)
-            job.update(processed=number, progress=number * 100 // total)
+            job.update(processed=shown, progress=shown * 100 // total)
             if stopped:
                 return False
         return True
 
     async def _prepare_tagging_model(self, job, spec: _Spec, owner):
         """The loaded service, or None (logged) when the job cannot run."""
-        from ..autotag import MODELS, get_autotag_service
+        from ..autotag import MODELS, MissingBackendPackages, get_autotag_service
 
         service = get_autotag_service()
         status = await self._run_in_executor(service.get_models_status)
-        if not status.get(spec.model_type, {}).get("downloaded"):
-            name = MODELS.get(spec.model_type, {}).get("name", spec.model_type)
-            message = unavailable_message(name)
+        name = MODELS.get(spec.model_type, {}).get("name", spec.model_type)
+        message = model_refusal(status, spec.model_type, name)
+        if message is not None:
             job.update(status_line=message)
             job.log_line(f"! {message}")
             return None
@@ -388,6 +812,11 @@ class ImageTagJobsMixin:
                 spec.model_type,
                 spec.options["use_gpu"],
             )
+        except MissingBackendPackages as exc:
+            # A package gone since the status read (load_model logged it).
+            job.update(status_line=str(exc))
+            job.log_line(f"! {exc}")
+            return None
         except Exception:
             self.logger.exception("The auto-tag job could not load its model")
             job.update(status_line=INFERENCE_FAILED)
@@ -439,9 +868,17 @@ class ImageTagJobsMixin:
             store.release(capture.token)
 
     async def _auto_tag_capture(self, run: _Run, store, capture, root, relative):
-        """Suggest on the capture, then union only onto the same current bytes."""
+        """Suggest on the capture, then union only onto the same current bytes.
+
+        The bytes are marked auto-tagged with the union, also when the model
+        returned no names; captured bytes no model can read are marked too
+        (and stay skipped), so a "Skip already tagged" run passes them by.
+        """
         if not capture.inferable:
-            return "skipped", NOT_A_STILL
+            group, detail = await self._union_image_tags(
+                capture, [], run.epoch, mark_autotagged=True
+            )
+            return (group, detail) if group == "errors" else ("skipped", NOT_A_STILL)
         if store.acquire(capture.token) is None:
             raise _JobStopped  # every capture was invalidated (restore)
         spec = run.spec
@@ -467,21 +904,30 @@ class ImageTagJobsMixin:
         if facts.locator != capture.locator or facts.sha256 != capture.sha256:
             return "skipped", FILE_CHANGED
         names = [str(name) for name, _score in pairs]
-        if not names:
-            return "unchanged", NO_NEW_TAGS
-        return await self._union_image_tags(facts, names, run.epoch)
+        return await self._union_image_tags(
+            facts, names, run.epoch, mark_autotagged=True
+        )
 
-    async def _union_image_tags(self, facts, tags: list, epoch: str):
-        """``add_image_tags`` on the verified bytes, as a (group, detail)."""
+    async def _union_image_tags(
+        self, facts, tags: list, epoch: str, *, mark_autotagged: bool = False
+    ):
+        """``add_image_tags`` on the verified bytes, as a (group, detail).
+
+        ``mark_autotagged`` (a model's result) marks the bytes auto-tagged in
+        the same write; then ``tags`` may be empty.
+        """
         try:
             receipt = await self._run_in_executor(
-                self.db.add_image_tags,
-                locator=facts.locator,
-                sha256=facts.sha256,
-                size=facts.size,
-                mtime_ns=facts.mtime_ns,
-                expected_epoch=epoch,
-                tags=tags,
+                functools.partial(
+                    self.db.add_image_tags,
+                    locator=facts.locator,
+                    sha256=facts.sha256,
+                    size=facts.size,
+                    mtime_ns=facts.mtime_ns,
+                    expected_epoch=epoch,
+                    tags=tags,
+                    mark_autotagged=mark_autotagged,
+                )
             )
         except LibraryCommandError as exc:
             if exc.code == "storage-changed":